walk is now awesomesauce... but still a memory hog

This commit is contained in:
AJ ONeal 2010-12-12 01:58:43 -07:00
parent d7c93fe195
commit e68088526e
9 changed files with 566 additions and 574 deletions

169
README.md
View File

@ -1,7 +1,18 @@
node-walk node-walk
==== ====
A not-quite-port of python's `os.walk`. A port python's `os.walk`, but using Node.JS conventions.
* EventEmitter
* Asynchronous
* Chronological (optionally)
* Built-in flow-control
As few file descriptors are opened at a time as possible.
This is particularly well suited for single hard disks which are not flash or solid state.
Memory usage is high (120mb for 60,000 dirs), but reduction is being investigated.
Patches welcome.
Installation Installation
---- ----
@ -11,91 +22,107 @@ Installation
Usage Usage
==== ====
All of the examples in the folder included in this repository work and have no typos.
without emitter
----
var walk = require('walk').walk, var walk = require('walk').walk,
options = { options,
walker;
options = {
followLinks: false, followLinks: false,
}; };
function fileHandler(err, path, errors, dirs, files, links, blocks, chars, fifos, sockets) { walker = walk("path/to/dir", options);
// handle each path
}
walk("path/to/dir", options, fileHandler); walker.on("directories", function (root, dirStatsArray, next) {
// This also works // dirStatsArray is an array of `stat` objects with the additional attributes
// walk("path/to/dir", options).whenever(fileHandler); // * type
// * error
// * name
Single Arguments next();
});
* `err` - Error when reading path (Probably due to permissions). walker.on("file", function (root, fileStats, next) {
* `path` - the current path being read fs.readFile(file, function () {
// doStuff
next();
});
});
walker.on("errors", function (root, nodeStatsArray, next) {
next();
});
API
====
Emitted Values
* `root` - the containing the files to be inspected
* *stats[Array]* - a single `stats` object or an array with some added attributes
* type - 'file', 'directory', etc
* error
* name - the name of the file, dir, etc
* next - no more files will be read until this is called
Single Events - fired immediately
* `directoryError` - Error when `fstat` succeeded, but reading path failed (Probably due to permissions).
* `node` - a `stats` object for a node of any type
* `file` - includes links when `followLinks` is `true`
* `directory`
* `blockDevice`
* `characterDevice`
* `symbolicLink` - always empty when `followLinks` is `true`
* `FIFO`
* `socket`
Array Arguments Array Arguments
* `errors` - `fs.stat` encountered on files in the directory * `errors` - errors encountered by `fs.stat` when reading ndes in a directory
* `dirs` - directories (modification of this array - sorting, removing, etc - affects traversal) * `nodes` - an array of `stats` of any type
* `files` - regular files (includes links when `followLinks` is `true`) * `files`
* `links` - symbolic links (always empty when `followLinks` is `true`) * `directories` - modification of this array - sorting, removing, etc - affects traversal
* `blocks` - block devices * `blockDevices`
* `chars` - character devices * `characterDevices`
* `fifos` - FIFOs * `symbolicLinks`
* `sockets` - sockets * `FIFOs`
* `sockets`
using emitter **Warning** when following links, an infinite loop is possible
----
`errors`, `directories`, `files`, `symbolicLinks` Comparisons
var walk = require('walk').walk,
emitter = walk('./walk-test');
emitter.on("directories", function (path, dirs) {
// the second directory will not be traversed
dirs.splice(1,1);
dirs.forEach(function (dir) {
console.log(dir);
});
});
emitter.on("files", function (path, files) {
files.forEach(function (dir) {
console.log(dir);
});
});
Example
==== ====
mkdir -p walk-test/dir1 Tested on my `/System` containing 59,490 (+ self) directories (and lots of files).
touch walk-test/file-1 The size of the text output was 6mb.
touch walk-test/file-2
touch walk-test/dir1/file-1
touch walk-test/dir1/file-2
node-walk-test `find`:
---- time bash -c "find /System -type d | wc"
59491 97935 6262916
var walk = require('walk'); real 2m27.114s
user 0m1.193s
sys 0m14.859s
walk('./walk-test', undefined, function (err, path, errors, dirs, files, links) { `find.js`:
if (err) {
console.log('ERROR: ');
console.log(err);
return;
}
dirs.forEach(function (item, i, arr) { Note that `find.js` omits the start directory
if (item.name.match(/trash/i)) {
console.log('found a trash'); time bash -c "node examples/find.js /System -type d | wc"
arr.splice(i,1); 59490 97934 6262908
}
}); # Test 1
real 2m52.273s
user 0m20.374s
sys 0m27.800s
# Test 2
real 2m23.725s
user 0m18.019s
sys 0m23.202s
# Test 3
real 2m50.077s
user 0m17.661s
sys 0m24.008s
console.log("PATH: " + path);
console.log("SORTED: ");
console.log(errors, dirs, files, links);
});

35
examples/find-test.js Normal file
View File

@ -0,0 +1,35 @@
(function () {
var walk = require("../lib/walk3.js"),
emit = walk(process.argv[2] || "/tmp"),
util = require('util'),
path = require('path');
// nor the root, nor the node should ever be empty
walk.fnodeTypesPlural.forEach(function (fnodeType) {
emit.on(fnodeType, function (root, nodes, next) {
if (!nodes || !nodes.length || !root) {
console.log(fnodeType, "empty set", root, nodes.length); //JSON.stringify(nodes));
}
next();
});
});
walk.fnodeTypes.forEach(function (fnodeType) {
emit.on(fnodeType, function (root, node, next) {
if (!node || !node.name || !root) {
console.log(fnodeType, "empty item", root, node.name); //JSON.stringify(node));
}
next();
});
});
emit.on('directory', function (root, dir, next) {
console.log(path.join(root, dir.name));
next();
});
/*
emit.on('file', function (root, file, next) {
console.log(path.join(root, file.name));
next();
});
*/
}());

View File

@ -3,13 +3,20 @@
emit = walk(process.argv[2] || "/tmp"); emit = walk(process.argv[2] || "/tmp");
//icount = 0; //icount = 0;
emit.on('directories', function (path, dirs, next) {
dirs.forEach(function (dir) {
console.log(path + '/' + dir.name);
});
next();
});
/*
emit.on('directory', function (path, file, next) { emit.on('directory', function (path, file, next) {
//icount += 1; //icount += 1;
console.log(path + '/' + file.name); // + " " + icount); console.log(path + '/' + file.name); // + " " + icount);
process.nextTick(next); next();
//process.nextTick(next);
//setTimeout(next, 100); //setTimeout(next, 100);
}); });
/*
emit.on('file', function (path, file, next) { emit.on('file', function (path, file, next) {
console.log("FILE:", file.name, "\n"); console.log("FILE:", file.name, "\n");
next(); next();

230
lib/walk-0.9.js Normal file
View File

@ -0,0 +1,230 @@
(function () {
var fs = require('fs'),
Futures = require('futures'),
joinPath = require('path').join,
util = require('util'),
ev = require("events"),
emitter = new ev.EventEmitter(),
oneNodeEvent = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"fifo",
"socket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
],
eventsTpl = {
listeners: function () { return []; },
next: function () { return; }
},
events = {},
nexts = {};
function newVersion() {
throw new Error("see README.md at http://github.com/coolaj86/node-walk");
}
function noop() {
}
function remove(arr, obj) {
return arr.splice(arr.indexOf(obj), 1);
}
oneNodeEvent.forEach(function (key) {
var e = events[key] = {}, next;
Object.keys(eventsTpl).forEach(function (k) {
e[k] = eventsTpl[k]();
});
emitter.on("newListener", function (ev, listener) {
var count = 0,
num = e.listeners.length + 1;
e.listeners.push(listener);
e.next = function (cb) {
cb = noop;
return function () {
if (count === num) { cb(); }
count += 1;
};
};
});
// TODO
next = function () {
};
});
function sortNodesByType(path, stats, o, cb) {
if (stats.isFile()) {
o.files.push(stats);
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
} else if (stats.isDirectory()) {
o.dirs.push(stats);
emitter.emit("directory", path, stats, function () {
remove(o.dirs, stats);
}, (nexts["directory"]||noop)(cb));
} else if (stats.isBlockDevice()) {
o.blocks.push(stats);
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
} else if (stats.isCharacterDevice()) {
o.chars.push(stats);
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
} else if (stats.isSymbolicLink()) {
o.links.push(stats);
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
} else if (stats.isFIFO()) {
o.fifos.push(stats);
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
} else if (stats.isSocket()) {
o.sockets.push(stats);
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
} else {
// emitter.emit("error", stats);
util.debug(stats.name + 'is not of any node type');
}
}
/*
import os
from os.path import join, getsize
for root, dirs, files in os.walk('python/Lib/email'):
print root, "consumes",
print sum(getsize(join(root, name)) for name in files),
print "bytes in", len(files), "non-directory files"
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
*/
/*
fs.walk(path, function ({ err, root, dirs, files }) {}, {
// currently ignored
topdown: boolean,
onerror: boolean, // ignored
followLinks: boolean // lstat or stat
});
*/
function walk(firstPath, options, callback) {
options = options || {};
var fstat = options.followLinks ? fs.stat : fs.lstat,
subscription = Futures.subscription();
if (callback) { subscription.subscribe(callback); }
function readDir(path) {
var p = Futures.promise();
fs.readdir(path, function (err, files) {
if (err) {
err.path = path;
subscription.deliver(err, path);
// Signal the completion of this readdir attempt
p.fulfill();
return;
}
// TODO fix futures sequence to not require a first function like this
var s = Futures.sequence(function(n){n();}),
nodes = [],
o = {
errors: [],
dirs: [],
files: [],
links: [],
blocks: [],
chars: [],
fifos: [],
sockets: []
};
files.forEach(function (file) {
// pushes onto the sequence stack without recursion
s.then(function (next) {
fstat(joinPath(path, file), function (err, stats) {
stats = stats || {};
stats.name = file;
nodes.push(stats);
if (err) {
stats.err = err;
o.errors.push(stats);
} else {
sortNodesByType(path, stats, o);
}
next();
});
});
});
s.then(function (next) {
var s2 = Futures.sequence(function(n){n();});
if (nodes.length > 0) {
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
if (o.errors.length > 0) {
emitter.emit("errors", path, o.errors);
}
if (o.dirs.length > 0) {
emitter.emit("directories", path, o.dirs);
}
if (o.files.length > 0) {
emitter.emit("files", path, o.files);
}
if (o.links.length > 0) {
emitter.emit("symbolicLinks", path, o.links);
}
if (o.blocks.length > 0) {
emitter.emit("blockDevices", path, o.blocks);
}
if (o.chars.length > 0) {
emitter.emit("characterDevices", path, o.chars);
}
if (o.fifos.length > 0) {
emitter.emit("fifos", path, o.fifos);
}
if (o.sockets.length > 0) {
emitter.emit("sockets", path, o.fifos);
}
p.fulfill();
o.dirs.forEach(function (dir) {
s2.then(function (next2) {
readDir(joinPath(path, dir.name))
.when(function () { next2(); });
});
});
next();
}
});
});
return p.passable();
}
readDir(firstPath) //.whenever(callback);
emitter.whenever = subscription.subscribe;
return emitter;
}
newVersion.walk = walk;
newVersion.remove = remove;
module.exports = newVersion;
}());

View File

@ -0,0 +1,60 @@
// 2011-11-25 jorge@jorgechamorro.com
function walk (file, cb) {
var fs = require('fs');
var q= [];
var queue= [q];
walk2();
function walk2 () {
cb(file);
fs.lstat(file, function (err, stat) {
if (err || !stat.isDirectory()) return next();
getDirectory(function (files) {
queue.push(q= files);
next();
});
});
}
function next () {
if (q.length) {
file= q.pop();
walk2();
}
else if (queue.length-= 1) {
q= queue[queue.length-1];
next();
}
}
function getDirectory (cb) {
fs.readdir(file, function(err, files) {
if (!files) return;
//if (err) throw Error(err);
files.sort(sort);
files.forEach(fullPath);
cb(files);
});
}
function fullPath (v,i,o) {
o[i]= [file, '/', v].join('');
}
function sort (a,b) {
a= a.toLowerCase();
b= b.toLowerCase();
if (a > b) return -1;
if (a < b) return 1;
else return 0;
}
}
// your callback here
var ctr= 0;
function callBack (file) { console.log( ["[", ++ctr, "] ", file].join('') ) };
process.argv.forEach(function(val, index, array) {
if (index > 1) walk(val, callBack);
});

View File

@ -1,230 +1,157 @@
(function () { // TODO
// * add types by listener dynamically
// * unroll loops for better readability?
// * should emitted errors wait for `next()`?
(function (undefined) {
var fs = require('fs'), var fs = require('fs'),
Futures = require('futures'), upath = require('path'),
joinPath = require('path').join,
util = require('util'), util = require('util'),
ev = require("events"), Futures = require('futures'),
emitter = new ev.EventEmitter(), events = require('events'),
oneNodeEvent = [ noop = function () {},
"file", // "FIFO" isn't easy to convert to camelCame and back reliably
"directory", isFnodeTypes = [
"blockDevice", "isFile", "isDirectory", "isBlockDevice", "isCharacterDevice", "isSymbolicLink", "isFIFO", "isSocket"
"characterDevice",
"symbolicLink",
"fifo",
"socket"
], ],
multiNodeEvents = [ fnodeTypes = [
// multiple "file", "directory", "blockDevice", "characterDevice", "symbolicLink", "FIFO", "socket"
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
], ],
eventsTpl = { fnodeTypesPlural = [
listeners: function () { return []; }, "files", "directories", "blockDevices", "characterDevices", "symbolicLinks", "FIFOs", "sockets"
next: function () { return; } ];
},
events = {},
nexts = {};
function newVersion() { // Create a new walk instance
throw new Error("see README.md at http://github.com/coolaj86/node-walk"); function create(path, options) {
} var emitter = new events.EventEmitter(),
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
function noop() {
}
function remove(arr, obj) { // Get the current number of listeners (which may change)
return arr.splice(arr.indexOf(obj), 1); // Emit events to each listener
} // Wait for all listeners to `next()` before continueing
// (in theory this may avoid disk thrashing)
function emitSingleEvents(path, stats, next) {
var num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
oneNodeEvent.forEach(function (key) { function nextWhenReady() {
var e = events[key] = {}, next; num -= 1;
if (0 === num) { next(); }
}
Object.keys(eventsTpl).forEach(function (k) { emitter.emit(stats.type, path, stats, nextWhenReady);
e[k] = eventsTpl[k](); emitter.emit("node", path, stats, nextWhenReady);
}); nextWhenReady();
emitter.on("newListener", function (ev, listener) {
var count = 0,
num = e.listeners.length + 1;
e.listeners.push(listener);
e.next = function (cb) {
cb = noop;
return function () {
if (count === num) { cb(); }
count += 1;
};
};
});
// TODO
next = function () {
};
});
function sortNodesByType(path, stats, o, cb) {
if (stats.isFile()) {
o.files.push(stats);
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
} else if (stats.isDirectory()) {
o.dirs.push(stats);
emitter.emit("directory", path, stats, function () {
remove(o.dirs, stats);
}, (nexts["directory"]||noop)(cb));
} else if (stats.isBlockDevice()) {
o.blocks.push(stats);
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
} else if (stats.isCharacterDevice()) {
o.chars.push(stats);
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
} else if (stats.isSymbolicLink()) {
o.links.push(stats);
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
} else if (stats.isFIFO()) {
o.fifos.push(stats);
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
} else if (stats.isSocket()) {
o.sockets.push(stats);
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
} else {
// emitter.emit("error", stats);
util.debug(stats.name + 'is not of any node type');
} }
}
/*
import os
from os.path import join, getsize
for root, dirs, files in os.walk('python/Lib/email'):
print root, "consumes",
print sum(getsize(join(root, name)) for name in files),
print "bytes in", len(files), "non-directory files"
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
*/
/* // Since the risk for disk thrashing among anything
fs.walk(path, function ({ err, root, dirs, files }) {}, { // other than files is relatively low, all types are
// currently ignored // emitted at once, but all must complete before advancing
topdown: boolean, function emitPluralEvents(path, nodes, next) {
onerror: boolean, // ignored var num = 1;
followLinks: boolean // lstat or stat
});
*/
function walk(firstPath, options, callback) { function nextWhenReady() {
options = options || {}; num -= 1;
var fstat = options.followLinks ? fs.stat : fs.lstat, if (0 === num) { next(); }
subscription = Futures.subscription(); }
if (callback) { subscription.subscribe(callback); } fnodeTypesPlural.concat(["nodes", "errors"]).forEach(function (fnodeType) {
if (0 === nodes[fnodeType].length) { return; }
num += emitter.listeners(fnodeType).length;
emitter.emit(fnodeType, path, nodes[fnodeType], nextWhenReady);
});
nextWhenReady();
}
function readDir(path) {
var p = Futures.promise();
fs.readdir(path, function (err, files) { // Determine each file node's type
if (err) { //
err.path = path; function sortFnodesByType(path, stats, fnodes, nextFile) {
subscription.deliver(err, path); isFnodeTypes.forEach(function (isType, i) {
// Signal the completion of this readdir attempt if (stats[isType]()) {
p.fulfill(); if (stats.type) { throw new Error("is_" + type + " and " + isType); }
stats.type = fnodeTypes[i];
fnodes[fnodeTypesPlural[i]].push(stats);
// TODO throw to break;
}
});
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
emitSingleEvents(path, stats, nextFile);
}
// Asynchronously get the stats
//
function getStats(path, files, walkDirs) {
var nodeGroups = {};
fnodeTypesPlural.concat("nodes", "errors").forEach(function (fnodeTypePlural) {
nodeGroups[fnodeTypePlural] = [];
});
function nextFile() {
var file = files.pop(), dirs = [], fnames = [];
if (undefined === file) {
emitPluralEvents(path, nodeGroups, function () {
nodeGroups.directories.forEach(function (dir) {
dirs.push(dir.name);
});
walkDirs(dirs);
});
return; return;
} }
// TODO fix futures sequence to not require a first function like this fstat(upath.join(path, file), function (err, stats) {
var s = Futures.sequence(function(n){n();}), //console.log("`stat`ed file " + file);
nodes = [], stats = stats || {};
o = { stats.name = file;
errors: [], nodeGroups.nodes.push(stats);
dirs: [], if (err) {
files: [], stats.error = err;
links: [], stats.type = 'error';
blocks: [], nodeGroups.errors.push(stats);
chars: [], //emitter.emit('fileError', path, stats, noop);
fifos: [], return nextFile();
sockets: []
};
files.forEach(function (file) {
// pushes onto the sequence stack without recursion
s.then(function (next) {
fstat(joinPath(path, file), function (err, stats) {
stats = stats || {};
stats.name = file;
nodes.push(stats);
if (err) {
stats.err = err;
o.errors.push(stats);
} else {
sortNodesByType(path, stats, o);
}
next();
});
});
});
s.then(function (next) {
var s2 = Futures.sequence(function(n){n();});
if (nodes.length > 0) {
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
if (o.errors.length > 0) {
emitter.emit("errors", path, o.errors);
}
if (o.dirs.length > 0) {
emitter.emit("directories", path, o.dirs);
}
if (o.files.length > 0) {
emitter.emit("files", path, o.files);
}
if (o.links.length > 0) {
emitter.emit("symbolicLinks", path, o.links);
}
if (o.blocks.length > 0) {
emitter.emit("blockDevices", path, o.blocks);
}
if (o.chars.length > 0) {
emitter.emit("characterDevices", path, o.chars);
}
if (o.fifos.length > 0) {
emitter.emit("fifos", path, o.fifos);
}
if (o.sockets.length > 0) {
emitter.emit("sockets", path, o.fifos);
}
p.fulfill();
o.dirs.forEach(function (dir) {
s2.then(function (next2) {
readDir(joinPath(path, dir.name))
.when(function () { next2(); });
});
});
next();
} }
sortFnodesByType(path, stats, nodeGroups, nextFile);
}); });
}
}); nextFile();
return p.passable();
} }
readDir(firstPath) //.whenever(callback); function walk(path, next) {
fs.readdir(path, function (err, nodes) {
if (err) {
emitter.emit('directoryError', path, { error: err, name: path }, noop);
return next(); /*TODO*/ throw err;
}
getStats(path, nodes, function (dirs) {
walkDirs(path, dirs, next);
});
});
}
emitter.whenever = subscription.subscribe; function walkDirs(path, dirs, cb) {
function nextDir() {
var dir = dirs.pop();
if (undefined === dir) {
delete dirs;
return cb();
}
walk(upath.join(path, dir), nextDir);
}
nextDir();
}
walk(upath.normalize(path), function () {
//throw Error("hey");
//console.log("Utterly Done!");
});
return emitter; return emitter;
} }
module.exports = create;
newVersion.walk = walk; module.exports.isFnodeTypes = isFnodeTypes;
newVersion.remove = remove; module.exports.fnodeTypes = fnodeTypes;
module.exports = newVersion; module.exports.fnodeTypesPlural = fnodeTypesPlural;
}()); }());

View File

@ -1,138 +0,0 @@
(function () {
var fs = require('fs'),
upath = require('path'),
util = require('util'),
Futures = require('futures'),
events = require('events'),
emitter = new events.EventEmitter(),
oneNodeEvents = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"fifo",
"socket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"fifos",
"sockets"
],
fstat;
function sortNodesByType(path, stats, o, next) {
var type, listeners, num, count;
if (stats.isFile()) {
type = "file";
o.files.push(stats);
} else if (stats.isDirectory()) {
type = "directory";
o.dirs.push(stats);
} else if (stats.isBlockDevice()) {
type = "blockDevice";
o.blocks.push(stats);
} else if (stats.isCharacterDevice()) {
type = "characterDevice";
o.chars.push(stats);
} else if (stats.isSymbolicLink()) {
type = "symbolicLink";
o.links.push(stats);
} else if (stats.isFIFO()) {
type = "fifo";
o.fifos.push(stats);
} else if (stats.isSocket()) {
type = "socket";
o.sockets.push(stats);
} else {
throw new Error(upath.join(path,stats.name) + 'is not of any tested node type');
}
listeners = emitter.listeners(type);
// get the current number of listeners (may change)
num = listeners.length;
if (!num) {
next();
return;
}
// join all; once all listeners have responded, continue
count = 0;
emitter.emit(type, path, stats, function () {
count += 1;
if (num === count) {
next();
}
});
}
function handleFiles(path, files) {
var s = Futures.sequence(),
nodes = [],
o = {
errors: [],
dirs: [],
files: [],
links: [],
blocks: [],
chars: [],
fifos: [],
sockets: []
};
files.forEach(function (file) {
s.then(function (next) {
fstat(upath.join(path, file), function (err, stats) {
if (err) {
util.debug("[Error] " + util.inspect(err));
next();
return;
}
stats.name = file;
sortNodesByType(path, stats, o, next);
});
});
});
s.then(function (next) {
// TODO copycat the emitters above
next();
});
return s;
}
function handlePath(path) {
}
function walk(fullpath, options) {
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
var path, file, len, s;
upath.normalize(fullpath);
len = fullpath.length - 1;
if (len > 1 && '/' === fullpath[len]) {
fullpath = fullpath.substr(0, len);
}
path = upath.dirname(fullpath);
file = upath.basename(fullpath);
s = handleFiles(path, [file]);
s(function (next) { next(); });
return emitter;
}
var walker = walk("/System");
walker.on("directory", function (path, dir, next) {
console.log(path, dir.name);
next();
});
}());

View File

@ -1,156 +0,0 @@
(function (undefined) {
var fs = require('fs'),
upath = require('path'),
util = require('util'),
Futures = require('futures'),
events = require('events');
function create(path, options) {
// TODO add types by listener dynamically
var emitter = new events.EventEmitter(),
oneNodeEvents = [
"file",
"directory",
"blockDevice",
"characterDevice",
"symbolicLink",
"FIFO",
"socket"
],
oneNodeTypes = [
"isFile",
"isDirectory",
"isBlockDevice",
"isCharacterDevice",
"isSymbolicLink",
"isFIFO",
"isSocket"
],
multiNodeEvents = [
// multiple
"files",
"directories",
"blockDevices",
"characterDevices",
"symbolicLinks",
"FIFOs",
"sockets"
],
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
function normalize(path) {
return path;
var index;
path = upath.normalize(path);
index = path.length - 1;
if (index > 0 && '/' === path[index]) {
path = path.substr(0, index);
}
return path;
}
function sortNodesByType(path, stats, nodes, next) {
// This could easily be unrolled
// but in this case concise was more readable for me
oneNodeTypes.forEach(function (isType, i) {
if (stats[isType]()) {
if (stats.type) { throw new Error("is_" + type + " and " + isType); }
stats.type = oneNodeEvents[i];
nodes[multiNodeEvents[i]].push(stats);
// TODO throw to break;
}
});
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
function emitSingleEvents(stats, next) {
var num;
// get the current number of listeners (may change)
num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
function nextWhenReady() {
num -= 1;
if (0 === num) { next(); }
}
nextWhenReady();
// Total number of listeners
emitter.emit(stats.type, path, stats, function () {
nextWhenReady();
});
emitter.emit("node", path, stats, function () {
nextWhenReady();
});
} // emitSingleEvents
emitSingleEvents(stats, next);
}
function getStats(path, files, walkDirs) {
var nodes = [], o = {};
// TODO unroll for better readability
multiNodeEvents.forEach(function (eventType) {
o[eventType] = [];
});
function nextFile() {
var file = files.pop(), dirs = [], fnames = [];
if (undefined === file) {
o.directories.forEach(function (dir) {
dirs.push(dir.name);
});
return walkDirs(dirs);
}
fstat(upath.join(path, file), function (err, stats) {
//console.log("`stat`ed file " + file);
stats = stats || {};
stats.name = file;
nodes.push(stats);
if (err) {
stats.error = err
// TODO single error emittor
o.errors.push(stats);
util.debug("[Error 1] " + util.inspect(err));
return nextFile();
}
sortNodesByType(path, stats, o, nextFile);
});
}
nextFile();
}
function walk(path, next) {
fs.readdir(path, function (err, nodes) {
if (err) { return next(); /*TODO*/ throw err; }
getStats(path, nodes, function (dirs) {
walkDirs(path, dirs, function () {
next();
});
});
});
}
function walkDirs(path, dirs, cb) {
function nextDir() {
var dir = dirs.pop();
if (undefined === dir) {
delete dirs;
return cb();
}
walk(upath.join(path, dir), nextDir);
}
nextDir();
}
walk(normalize(path), function () {
//throw Error("hey");
//console.log("Utterly Done!");
});
return emitter;
}
module.exports = create;
}());

View File

@ -1,12 +1,12 @@
{ {
"name" : "walk", "name" : "walk",
"description" : "A node port of python's os.walk", "description" : "A node port of python's os.walk",
"url" : "github.com/coolaj86/walk", "url" : "github.com/coolaj86/node-walk",
"keywords" : ["util", "os", "fs", "walk"], "keywords" : ["util", "os", "sys", "fs", "walk", "walkSync"],
"author" : "AJ ONeal <coolaj86@gmail.com>", "author" : "AJ ONeal <coolaj86@gmail.com>",
"contributors" : [], "contributors" : [],
"dependencies" : ["futures"], "dependencies" : [],
"lib" : "lib", "lib" : "lib",
"main" : "./lib/walk.js", "main" : "./lib/walk.js",
"version" : "0.9.2" "version" : "1.0.0"
} }