walk is now awesomesauce... but still a memory hog

This commit is contained in:
AJ ONeal 2010-12-12 01:58:43 -07:00
parent d7c93fe195
commit e68088526e
9 changed files with 566 additions and 574 deletions

165
README.md
View File

@ -1,7 +1,18 @@
node-walk
====
A not-quite-port of python's `os.walk`.
A port python's `os.walk`, but using Node.JS conventions.
* EventEmitter
* Asynchronous
* Chronological (optionally)
* Built-in flow-control
As few file descriptors are opened at a time as possible.
This is particularly well suited for single hard disks which are not flash or solid state.
Memory usage is high (120mb for 60,000 dirs), but reduction is being investigated.
Patches welcome.
Installation
----
@ -11,91 +22,107 @@ Installation
Usage
====
All of the examples in the folder included in this repository work and have no typos.
without emitter
----
var walk = require('walk').walk,
options,
walker;
options = {
followLinks: false,
};
function fileHandler(err, path, errors, dirs, files, links, blocks, chars, fifos, sockets) {
// handle each path
}
walker = walk("path/to/dir", options);
walk("path/to/dir", options, fileHandler);
// This also works
// walk("path/to/dir", options).whenever(fileHandler);
walker.on("directories", function (root, dirStatsArray, next) {
// dirStatsArray is an array of `stat` objects with the additional attributes
// * type
// * error
// * name
Single Arguments
next();
});
* `err` - Error when reading path (Probably due to permissions).
* `path` - the current path being read
walker.on("file", function (root, fileStats, next) {
fs.readFile(file, function () {
// doStuff
next();
});
});
walker.on("errors", function (root, nodeStatsArray, next) {
next();
});
API
====
Emitted Values
* `root` - the containing the files to be inspected
* *stats[Array]* - a single `stats` object or an array with some added attributes
* type - 'file', 'directory', etc
* error
* name - the name of the file, dir, etc
* next - no more files will be read until this is called
Single Events - fired immediately
* `directoryError` - Error when `fstat` succeeded, but reading path failed (Probably due to permissions).
* `node` - a `stats` object for a node of any type
* `file` - includes links when `followLinks` is `true`
* `directory`
* `blockDevice`
* `characterDevice`
* `symbolicLink` - always empty when `followLinks` is `true`
* `FIFO`
* `socket`
Array Arguments
* `errors` - `fs.stat` encountered on files in the directory
* `dirs` - directories (modification of this array - sorting, removing, etc - affects traversal)
* `files` - regular files (includes links when `followLinks` is `true`)
* `links` - symbolic links (always empty when `followLinks` is `true`)
* `blocks` - block devices
* `chars` - character devices
* `fifos` - FIFOs
* `sockets` - sockets
* `errors` - errors encountered by `fs.stat` when reading ndes in a directory
* `nodes` - an array of `stats` of any type
* `files`
* `directories` - modification of this array - sorting, removing, etc - affects traversal
* `blockDevices`
* `characterDevices`
* `symbolicLinks`
* `FIFOs`
* `sockets`
using emitter
----
**Warning** when following links, an infinite loop is possible
`errors`, `directories`, `files`, `symbolicLinks`
var walk = require('walk').walk,
emitter = walk('./walk-test');
emitter.on("directories", function (path, dirs) {
// the second directory will not be traversed
dirs.splice(1,1);
dirs.forEach(function (dir) {
console.log(dir);
});
});
emitter.on("files", function (path, files) {
files.forEach(function (dir) {
console.log(dir);
});
});
Example
Comparisons
====
mkdir -p walk-test/dir1
touch walk-test/file-1
touch walk-test/file-2
touch walk-test/dir1/file-1
touch walk-test/dir1/file-2
Tested on my `/System` containing 59,490 (+ self) directories (and lots of files).
The size of the text output was 6mb.
node-walk-test
----
`find`:
time bash -c "find /System -type d | wc"
59491 97935 6262916
var walk = require('walk');
real 2m27.114s
user 0m1.193s
sys 0m14.859s
walk('./walk-test', undefined, function (err, path, errors, dirs, files, links) {
if (err) {
console.log('ERROR: ');
console.log(err);
return;
}
`find.js`:
dirs.forEach(function (item, i, arr) {
if (item.name.match(/trash/i)) {
console.log('found a trash');
arr.splice(i,1);
}
});
Note that `find.js` omits the start directory
time bash -c "node examples/find.js /System -type d | wc"
59490 97934 6262908
# Test 1
real 2m52.273s
user 0m20.374s
sys 0m27.800s
# Test 2
real 2m23.725s
user 0m18.019s
sys 0m23.202s
# Test 3
real 2m50.077s
user 0m17.661s
sys 0m24.008s
console.log("PATH: " + path);
console.log("SORTED: ");
console.log(errors, dirs, files, links);
});

35
examples/find-test.js Normal file
View File

@ -0,0 +1,35 @@
(function () {
var walk = require("../lib/walk3.js"),
emit = walk(process.argv[2] || "/tmp"),
util = require('util'),
path = require('path');
// nor the root, nor the node should ever be empty
walk.fnodeTypesPlural.forEach(function (fnodeType) {
emit.on(fnodeType, function (root, nodes, next) {
if (!nodes || !nodes.length || !root) {
console.log(fnodeType, "empty set", root, nodes.length); //JSON.stringify(nodes));
}
next();
});
});
walk.fnodeTypes.forEach(function (fnodeType) {
emit.on(fnodeType, function (root, node, next) {
if (!node || !node.name || !root) {
console.log(fnodeType, "empty item", root, node.name); //JSON.stringify(node));
}
next();
});
});
emit.on('directory', function (root, dir, next) {
console.log(path.join(root, dir.name));
next();
});
/*
emit.on('file', function (root, file, next) {
console.log(path.join(root, file.name));
next();
});
*/
}());

View File

@ -3,13 +3,20 @@
emit = walk(process.argv[2] || "/tmp");
//icount = 0;
emit.on('directories', function (path, dirs, next) {
dirs.forEach(function (dir) {
console.log(path + '/' + dir.name);
});
next();
});
/*
emit.on('directory', function (path, file, next) {
//icount += 1;
console.log(path + '/' + file.name); // + " " + icount);
process.nextTick(next);
next();
//process.nextTick(next);
//setTimeout(next, 100);
});
/*
emit.on('file', function (path, file, next) {
console.log("FILE:", file.name, "\n");
next();

230
lib/walk-0.9.js Normal file
View File

@ -0,0 +1,230 @@
(function () {
var fs = require('fs'),
Futures = require('futures'),
joinPath = require('path').join,
util = require('util'),
ev = require("events"),
emitter = new ev.EventEmitter(),
oneNodeEvent = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"fifo",
"socket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
],
eventsTpl = {
listeners: function () { return []; },
next: function () { return; }
},
events = {},
nexts = {};
function newVersion() {
throw new Error("see README.md at http://github.com/coolaj86/node-walk");
}
function noop() {
}
function remove(arr, obj) {
return arr.splice(arr.indexOf(obj), 1);
}
oneNodeEvent.forEach(function (key) {
var e = events[key] = {}, next;
Object.keys(eventsTpl).forEach(function (k) {
e[k] = eventsTpl[k]();
});
emitter.on("newListener", function (ev, listener) {
var count = 0,
num = e.listeners.length + 1;
e.listeners.push(listener);
e.next = function (cb) {
cb = noop;
return function () {
if (count === num) { cb(); }
count += 1;
};
};
});
// TODO
next = function () {
};
});
function sortNodesByType(path, stats, o, cb) {
if (stats.isFile()) {
o.files.push(stats);
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
} else if (stats.isDirectory()) {
o.dirs.push(stats);
emitter.emit("directory", path, stats, function () {
remove(o.dirs, stats);
}, (nexts["directory"]||noop)(cb));
} else if (stats.isBlockDevice()) {
o.blocks.push(stats);
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
} else if (stats.isCharacterDevice()) {
o.chars.push(stats);
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
} else if (stats.isSymbolicLink()) {
o.links.push(stats);
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
} else if (stats.isFIFO()) {
o.fifos.push(stats);
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
} else if (stats.isSocket()) {
o.sockets.push(stats);
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
} else {
// emitter.emit("error", stats);
util.debug(stats.name + 'is not of any node type');
}
}
/*
import os
from os.path import join, getsize
for root, dirs, files in os.walk('python/Lib/email'):
print root, "consumes",
print sum(getsize(join(root, name)) for name in files),
print "bytes in", len(files), "non-directory files"
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
*/
/*
fs.walk(path, function ({ err, root, dirs, files }) {}, {
// currently ignored
topdown: boolean,
onerror: boolean, // ignored
followLinks: boolean // lstat or stat
});
*/
function walk(firstPath, options, callback) {
options = options || {};
var fstat = options.followLinks ? fs.stat : fs.lstat,
subscription = Futures.subscription();
if (callback) { subscription.subscribe(callback); }
function readDir(path) {
var p = Futures.promise();
fs.readdir(path, function (err, files) {
if (err) {
err.path = path;
subscription.deliver(err, path);
// Signal the completion of this readdir attempt
p.fulfill();
return;
}
// TODO fix futures sequence to not require a first function like this
var s = Futures.sequence(function(n){n();}),
nodes = [],
o = {
errors: [],
dirs: [],
files: [],
links: [],
blocks: [],
chars: [],
fifos: [],
sockets: []
};
files.forEach(function (file) {
// pushes onto the sequence stack without recursion
s.then(function (next) {
fstat(joinPath(path, file), function (err, stats) {
stats = stats || {};
stats.name = file;
nodes.push(stats);
if (err) {
stats.err = err;
o.errors.push(stats);
} else {
sortNodesByType(path, stats, o);
}
next();
});
});
});
s.then(function (next) {
var s2 = Futures.sequence(function(n){n();});
if (nodes.length > 0) {
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
if (o.errors.length > 0) {
emitter.emit("errors", path, o.errors);
}
if (o.dirs.length > 0) {
emitter.emit("directories", path, o.dirs);
}
if (o.files.length > 0) {
emitter.emit("files", path, o.files);
}
if (o.links.length > 0) {
emitter.emit("symbolicLinks", path, o.links);
}
if (o.blocks.length > 0) {
emitter.emit("blockDevices", path, o.blocks);
}
if (o.chars.length > 0) {
emitter.emit("characterDevices", path, o.chars);
}
if (o.fifos.length > 0) {
emitter.emit("fifos", path, o.fifos);
}
if (o.sockets.length > 0) {
emitter.emit("sockets", path, o.fifos);
}
p.fulfill();
o.dirs.forEach(function (dir) {
s2.then(function (next2) {
readDir(joinPath(path, dir.name))
.when(function () { next2(); });
});
});
next();
}
});
});
return p.passable();
}
readDir(firstPath) //.whenever(callback);
emitter.whenever = subscription.subscribe;
return emitter;
}
newVersion.walk = walk;
newVersion.remove = remove;
module.exports = newVersion;
}());

View File

@ -0,0 +1,60 @@
// 2011-11-25 jorge@jorgechamorro.com
function walk (file, cb) {
var fs = require('fs');
var q= [];
var queue= [q];
walk2();
function walk2 () {
cb(file);
fs.lstat(file, function (err, stat) {
if (err || !stat.isDirectory()) return next();
getDirectory(function (files) {
queue.push(q= files);
next();
});
});
}
function next () {
if (q.length) {
file= q.pop();
walk2();
}
else if (queue.length-= 1) {
q= queue[queue.length-1];
next();
}
}
function getDirectory (cb) {
fs.readdir(file, function(err, files) {
if (!files) return;
//if (err) throw Error(err);
files.sort(sort);
files.forEach(fullPath);
cb(files);
});
}
function fullPath (v,i,o) {
o[i]= [file, '/', v].join('');
}
function sort (a,b) {
a= a.toLowerCase();
b= b.toLowerCase();
if (a > b) return -1;
if (a < b) return 1;
else return 0;
}
}
// your callback here
var ctr= 0;
function callBack (file) { console.log( ["[", ++ctr, "] ", file].join('') ) };
process.argv.forEach(function(val, index, array) {
if (index > 1) walk(val, callBack);
});

View File

@ -1,230 +1,157 @@
(function () {
var fs = require('fs'),
Futures = require('futures'),
joinPath = require('path').join,
util = require('util'),
ev = require("events"),
emitter = new ev.EventEmitter(),
oneNodeEvent = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"fifo",
"socket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
],
eventsTpl = {
listeners: function () { return []; },
next: function () { return; }
},
events = {},
nexts = {};
function newVersion() {
throw new Error("see README.md at http://github.com/coolaj86/node-walk");
}
function noop() {
}
function remove(arr, obj) {
return arr.splice(arr.indexOf(obj), 1);
}
oneNodeEvent.forEach(function (key) {
var e = events[key] = {}, next;
Object.keys(eventsTpl).forEach(function (k) {
e[k] = eventsTpl[k]();
});
emitter.on("newListener", function (ev, listener) {
var count = 0,
num = e.listeners.length + 1;
e.listeners.push(listener);
e.next = function (cb) {
cb = noop;
return function () {
if (count === num) { cb(); }
count += 1;
};
};
});
// TODO
next = function () {
// * add types by listener dynamically
// * unroll loops for better readability?
// * should emitted errors wait for `next()`?
(function (undefined) {
var fs = require('fs'),
upath = require('path'),
util = require('util'),
Futures = require('futures'),
events = require('events'),
noop = function () {},
// "FIFO" isn't easy to convert to camelCame and back reliably
isFnodeTypes = [
"isFile", "isDirectory", "isBlockDevice", "isCharacterDevice", "isSymbolicLink", "isFIFO", "isSocket"
],
fnodeTypes = [
"file", "directory", "blockDevice", "characterDevice", "symbolicLink", "FIFO", "socket"
],
fnodeTypesPlural = [
"files", "directories", "blockDevices", "characterDevices", "symbolicLinks", "FIFOs", "sockets"
];
};
});
// Create a new walk instance
function create(path, options) {
var emitter = new events.EventEmitter(),
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
function sortNodesByType(path, stats, o, cb) {
if (stats.isFile()) {
o.files.push(stats);
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
} else if (stats.isDirectory()) {
o.dirs.push(stats);
emitter.emit("directory", path, stats, function () {
remove(o.dirs, stats);
}, (nexts["directory"]||noop)(cb));
} else if (stats.isBlockDevice()) {
o.blocks.push(stats);
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
} else if (stats.isCharacterDevice()) {
o.chars.push(stats);
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
} else if (stats.isSymbolicLink()) {
o.links.push(stats);
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
} else if (stats.isFIFO()) {
o.fifos.push(stats);
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
} else if (stats.isSocket()) {
o.sockets.push(stats);
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
} else {
// emitter.emit("error", stats);
util.debug(stats.name + 'is not of any node type');
}
// Get the current number of listeners (which may change)
// Emit events to each listener
// Wait for all listeners to `next()` before continueing
// (in theory this may avoid disk thrashing)
function emitSingleEvents(path, stats, next) {
var num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
function nextWhenReady() {
num -= 1;
if (0 === num) { next(); }
}
/*
import os
from os.path import join, getsize
for root, dirs, files in os.walk('python/Lib/email'):
print root, "consumes",
print sum(getsize(join(root, name)) for name in files),
print "bytes in", len(files), "non-directory files"
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
*/
emitter.emit(stats.type, path, stats, nextWhenReady);
emitter.emit("node", path, stats, nextWhenReady);
nextWhenReady();
}
/*
fs.walk(path, function ({ err, root, dirs, files }) {}, {
// currently ignored
topdown: boolean,
onerror: boolean, // ignored
followLinks: boolean // lstat or stat
// Since the risk for disk thrashing among anything
// other than files is relatively low, all types are
// emitted at once, but all must complete before advancing
function emitPluralEvents(path, nodes, next) {
var num = 1;
function nextWhenReady() {
num -= 1;
if (0 === num) { next(); }
}
fnodeTypesPlural.concat(["nodes", "errors"]).forEach(function (fnodeType) {
if (0 === nodes[fnodeType].length) { return; }
num += emitter.listeners(fnodeType).length;
emitter.emit(fnodeType, path, nodes[fnodeType], nextWhenReady);
});
*/
nextWhenReady();
}
function walk(firstPath, options, callback) {
options = options || {};
var fstat = options.followLinks ? fs.stat : fs.lstat,
subscription = Futures.subscription();
if (callback) { subscription.subscribe(callback); }
// Determine each file node's type
//
function sortFnodesByType(path, stats, fnodes, nextFile) {
isFnodeTypes.forEach(function (isType, i) {
if (stats[isType]()) {
if (stats.type) { throw new Error("is_" + type + " and " + isType); }
stats.type = fnodeTypes[i];
fnodes[fnodeTypesPlural[i]].push(stats);
// TODO throw to break;
}
});
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
emitSingleEvents(path, stats, nextFile);
}
function readDir(path) {
var p = Futures.promise();
fs.readdir(path, function (err, files) {
if (err) {
err.path = path;
subscription.deliver(err, path);
// Signal the completion of this readdir attempt
p.fulfill();
// Asynchronously get the stats
//
function getStats(path, files, walkDirs) {
var nodeGroups = {};
fnodeTypesPlural.concat("nodes", "errors").forEach(function (fnodeTypePlural) {
nodeGroups[fnodeTypePlural] = [];
});
function nextFile() {
var file = files.pop(), dirs = [], fnames = [];
if (undefined === file) {
emitPluralEvents(path, nodeGroups, function () {
nodeGroups.directories.forEach(function (dir) {
dirs.push(dir.name);
});
walkDirs(dirs);
});
return;
}
// TODO fix futures sequence to not require a first function like this
var s = Futures.sequence(function(n){n();}),
nodes = [],
o = {
errors: [],
dirs: [],
files: [],
links: [],
blocks: [],
chars: [],
fifos: [],
sockets: []
};
files.forEach(function (file) {
// pushes onto the sequence stack without recursion
s.then(function (next) {
fstat(joinPath(path, file), function (err, stats) {
fstat(upath.join(path, file), function (err, stats) {
//console.log("`stat`ed file " + file);
stats = stats || {};
stats.name = file;
nodes.push(stats);
nodeGroups.nodes.push(stats);
if (err) {
stats.err = err;
o.errors.push(stats);
} else {
sortNodesByType(path, stats, o);
stats.error = err;
stats.type = 'error';
nodeGroups.errors.push(stats);
//emitter.emit('fileError', path, stats, noop);
return nextFile();
}
sortFnodesByType(path, stats, nodeGroups, nextFile);
});
}
nextFile();
}
next();
function walk(path, next) {
fs.readdir(path, function (err, nodes) {
if (err) {
emitter.emit('directoryError', path, { error: err, name: path }, noop);
return next(); /*TODO*/ throw err;
}
getStats(path, nodes, function (dirs) {
walkDirs(path, dirs, next);
});
});
}
function walkDirs(path, dirs, cb) {
function nextDir() {
var dir = dirs.pop();
if (undefined === dir) {
delete dirs;
return cb();
}
walk(upath.join(path, dir), nextDir);
}
nextDir();
}
walk(upath.normalize(path), function () {
//throw Error("hey");
//console.log("Utterly Done!");
});
s.then(function (next) {
var s2 = Futures.sequence(function(n){n();});
if (nodes.length > 0) {
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
if (o.errors.length > 0) {
emitter.emit("errors", path, o.errors);
}
if (o.dirs.length > 0) {
emitter.emit("directories", path, o.dirs);
}
if (o.files.length > 0) {
emitter.emit("files", path, o.files);
}
if (o.links.length > 0) {
emitter.emit("symbolicLinks", path, o.links);
}
if (o.blocks.length > 0) {
emitter.emit("blockDevices", path, o.blocks);
}
if (o.chars.length > 0) {
emitter.emit("characterDevices", path, o.chars);
}
if (o.fifos.length > 0) {
emitter.emit("fifos", path, o.fifos);
}
if (o.sockets.length > 0) {
emitter.emit("sockets", path, o.fifos);
}
p.fulfill();
o.dirs.forEach(function (dir) {
s2.then(function (next2) {
readDir(joinPath(path, dir.name))
.when(function () { next2(); });
});
});
next();
}
});
});
return p.passable();
}
readDir(firstPath) //.whenever(callback);
emitter.whenever = subscription.subscribe;
return emitter;
}
newVersion.walk = walk;
newVersion.remove = remove;
module.exports = newVersion;
module.exports = create;
module.exports.isFnodeTypes = isFnodeTypes;
module.exports.fnodeTypes = fnodeTypes;
module.exports.fnodeTypesPlural = fnodeTypesPlural;
}());

View File

@ -1,138 +0,0 @@
(function () {
var fs = require('fs'),
upath = require('path'),
util = require('util'),
Futures = require('futures'),
events = require('events'),
emitter = new events.EventEmitter(),
oneNodeEvents = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"fifo",
"socket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
],
fstat;
function sortNodesByType(path, stats, o, next) {
var type, listeners, num, count;
if (stats.isFile()) {
type = "file";
o.files.push(stats);
} else if (stats.isDirectory()) {
type = "directory";
o.dirs.push(stats);
} else if (stats.isBlockDevice()) {
type = "blockDevice";
o.blocks.push(stats);
} else if (stats.isCharacterDevice()) {
type = "characterDevice";
o.chars.push(stats);
} else if (stats.isSymbolicLink()) {
type = "symbolicLink";
o.links.push(stats);
} else if (stats.isFIFO()) {
type = "fifo";
o.fifos.push(stats);
} else if (stats.isSocket()) {
type = "socket";
o.sockets.push(stats);
} else {
throw new Error(upath.join(path,stats.name) + 'is not of any tested node type');
}
listeners = emitter.listeners(type);
// get the current number of listeners (may change)
num = listeners.length;
if (!num) {
next();
return;
}
// join all; once all listeners have responded, continue
count = 0;
emitter.emit(type, path, stats, function () {
count += 1;
if (num === count) {
next();
}
});
}
function handleFiles(path, files) {
var s = Futures.sequence(),
nodes = [],
o = {
errors: [],
dirs: [],
files: [],
links: [],
blocks: [],
chars: [],
fifos: [],
sockets: []
};
files.forEach(function (file) {
s.then(function (next) {
fstat(upath.join(path, file), function (err, stats) {
if (err) {
util.debug("[Error] " + util.inspect(err));
next();
return;
}
stats.name = file;
sortNodesByType(path, stats, o, next);
});
});
});
s.then(function (next) {
// TODO copycat the emitters above
next();
});
return s;
}
function handlePath(path) {
}
function walk(fullpath, options) {
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
var path, file, len, s;
upath.normalize(fullpath);
len = fullpath.length - 1;
if (len > 1 && '/' === fullpath[len]) {
fullpath = fullpath.substr(0, len);
}
path = upath.dirname(fullpath);
file = upath.basename(fullpath);
s = handleFiles(path, [file]);
s(function (next) { next(); });
return emitter;
}
var walker = walk("/System");
walker.on("directory", function (path, dir, next) {
console.log(path, dir.name);
next();
});
}());

View File

@ -1,156 +0,0 @@
(function (undefined) {
var fs = require('fs'),
upath = require('path'),
util = require('util'),
Futures = require('futures'),
events = require('events');
function create(path, options) {
// TODO add types by listener dynamically
var emitter = new events.EventEmitter(),
oneNodeEvents = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"FIFO",
"socket"
],
oneNodeTypes = [
"isFile",
"isDirectory",
"isBlockDevice",
"isCharacterDevice",
"isSymbolicLink",
"isFIFO",
"isSocket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"FIFOs",
"sockets"
],
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
function normalize(path) {
return path;
var index;
path = upath.normalize(path);
index = path.length - 1;
if (index > 0 && '/' === path[index]) {
path = path.substr(0, index);
}
return path;
}
function sortNodesByType(path, stats, nodes, next) {
// This could easily be unrolled
// but in this case concise was more readable for me
oneNodeTypes.forEach(function (isType, i) {
if (stats[isType]()) {
if (stats.type) { throw new Error("is_" + type + " and " + isType); }
stats.type = oneNodeEvents[i];
nodes[multiNodeEvents[i]].push(stats);
// TODO throw to break;
}
});
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
function emitSingleEvents(stats, next) {
var num;
// get the current number of listeners (may change)
num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
function nextWhenReady() {
num -= 1;
if (0 === num) { next(); }
}
nextWhenReady();
// Total number of listeners
emitter.emit(stats.type, path, stats, function () {
nextWhenReady();
});
emitter.emit("node", path, stats, function () {
nextWhenReady();
});
} // emitSingleEvents
emitSingleEvents(stats, next);
}
function getStats(path, files, walkDirs) {
var nodes = [], o = {};
// TODO unroll for better readability
multiNodeEvents.forEach(function (eventType) {
o[eventType] = [];
});
function nextFile() {
var file = files.pop(), dirs = [], fnames = [];
if (undefined === file) {
o.directories.forEach(function (dir) {
dirs.push(dir.name);
});
return walkDirs(dirs);
}
fstat(upath.join(path, file), function (err, stats) {
//console.log("`stat`ed file " + file);
stats = stats || {};
stats.name = file;
nodes.push(stats);
if (err) {
stats.error = err
// TODO single error emittor
o.errors.push(stats);
util.debug("[Error 1] " + util.inspect(err));
return nextFile();
}
sortNodesByType(path, stats, o, nextFile);
});
}
nextFile();
}
function walk(path, next) {
fs.readdir(path, function (err, nodes) {
if (err) { return next(); /*TODO*/ throw err; }
getStats(path, nodes, function (dirs) {
walkDirs(path, dirs, function () {
next();
});
});
});
}
function walkDirs(path, dirs, cb) {
function nextDir() {
var dir = dirs.pop();
if (undefined === dir) {
delete dirs;
return cb();
}
walk(upath.join(path, dir), nextDir);
}
nextDir();
}
walk(normalize(path), function () {
//throw Error("hey");
//console.log("Utterly Done!");
});
return emitter;
}
module.exports = create;
}());

View File

@ -1,12 +1,12 @@
{
"name" : "walk",
"description" : "A node port of python's os.walk",
"url" : "github.com/coolaj86/walk",
"keywords" : ["util", "os", "fs", "walk"],
"url" : "github.com/coolaj86/node-walk",
"keywords" : ["util", "os", "sys", "fs", "walk", "walkSync"],
"author" : "AJ ONeal <coolaj86@gmail.com>",
"contributors" : [],
"dependencies" : ["futures"],
"dependencies" : [],
"lib" : "lib",
"main" : "./lib/walk.js",
"version" : "0.9.2"
"version" : "1.0.0"
}