walk is now awesomesauce... but still a memory hog
This commit is contained in:
parent
d7c93fe195
commit
e68088526e
169
README.md
169
README.md
|
|
@ -1,7 +1,18 @@
|
|||
node-walk
|
||||
====
|
||||
|
||||
A not-quite-port of python's `os.walk`.
|
||||
A port python's `os.walk`, but using Node.JS conventions.
|
||||
|
||||
* EventEmitter
|
||||
* Asynchronous
|
||||
* Chronological (optionally)
|
||||
* Built-in flow-control
|
||||
|
||||
As few file descriptors are opened at a time as possible.
|
||||
This is particularly well suited for single hard disks which are not flash or solid state.
|
||||
|
||||
Memory usage is high (120mb for 60,000 dirs), but reduction is being investigated.
|
||||
Patches welcome.
|
||||
|
||||
Installation
|
||||
----
|
||||
|
|
@ -11,91 +22,107 @@ Installation
|
|||
Usage
|
||||
====
|
||||
|
||||
All of the examples in the folder included in this repository work and have no typos.
|
||||
|
||||
without emitter
|
||||
----
|
||||
|
||||
var walk = require('walk').walk,
|
||||
options = {
|
||||
options,
|
||||
walker;
|
||||
|
||||
options = {
|
||||
followLinks: false,
|
||||
};
|
||||
};
|
||||
|
||||
function fileHandler(err, path, errors, dirs, files, links, blocks, chars, fifos, sockets) {
|
||||
// handle each path
|
||||
}
|
||||
walker = walk("path/to/dir", options);
|
||||
|
||||
walk("path/to/dir", options, fileHandler);
|
||||
// This also works
|
||||
// walk("path/to/dir", options).whenever(fileHandler);
|
||||
walker.on("directories", function (root, dirStatsArray, next) {
|
||||
// dirStatsArray is an array of `stat` objects with the additional attributes
|
||||
// * type
|
||||
// * error
|
||||
// * name
|
||||
|
||||
Single Arguments
|
||||
next();
|
||||
});
|
||||
|
||||
* `err` - Error when reading path (Probably due to permissions).
|
||||
* `path` - the current path being read
|
||||
walker.on("file", function (root, fileStats, next) {
|
||||
fs.readFile(file, function () {
|
||||
// doStuff
|
||||
next();
|
||||
});
|
||||
});
|
||||
|
||||
walker.on("errors", function (root, nodeStatsArray, next) {
|
||||
next();
|
||||
});
|
||||
|
||||
API
|
||||
====
|
||||
|
||||
Emitted Values
|
||||
|
||||
* `root` - the containing the files to be inspected
|
||||
* *stats[Array]* - a single `stats` object or an array with some added attributes
|
||||
* type - 'file', 'directory', etc
|
||||
* error
|
||||
* name - the name of the file, dir, etc
|
||||
* next - no more files will be read until this is called
|
||||
|
||||
Single Events - fired immediately
|
||||
|
||||
* `directoryError` - Error when `fstat` succeeded, but reading path failed (Probably due to permissions).
|
||||
* `node` - a `stats` object for a node of any type
|
||||
* `file` - includes links when `followLinks` is `true`
|
||||
* `directory`
|
||||
* `blockDevice`
|
||||
* `characterDevice`
|
||||
* `symbolicLink` - always empty when `followLinks` is `true`
|
||||
* `FIFO`
|
||||
* `socket`
|
||||
|
||||
Array Arguments
|
||||
|
||||
* `errors` - `fs.stat` encountered on files in the directory
|
||||
* `dirs` - directories (modification of this array - sorting, removing, etc - affects traversal)
|
||||
* `files` - regular files (includes links when `followLinks` is `true`)
|
||||
* `links` - symbolic links (always empty when `followLinks` is `true`)
|
||||
* `blocks` - block devices
|
||||
* `chars` - character devices
|
||||
* `fifos` - FIFOs
|
||||
* `sockets` - sockets
|
||||
* `errors` - errors encountered by `fs.stat` when reading ndes in a directory
|
||||
* `nodes` - an array of `stats` of any type
|
||||
* `files`
|
||||
* `directories` - modification of this array - sorting, removing, etc - affects traversal
|
||||
* `blockDevices`
|
||||
* `characterDevices`
|
||||
* `symbolicLinks`
|
||||
* `FIFOs`
|
||||
* `sockets`
|
||||
|
||||
using emitter
|
||||
----
|
||||
**Warning** when following links, an infinite loop is possible
|
||||
|
||||
`errors`, `directories`, `files`, `symbolicLinks`
|
||||
|
||||
var walk = require('walk').walk,
|
||||
emitter = walk('./walk-test');
|
||||
|
||||
emitter.on("directories", function (path, dirs) {
|
||||
// the second directory will not be traversed
|
||||
dirs.splice(1,1);
|
||||
dirs.forEach(function (dir) {
|
||||
console.log(dir);
|
||||
});
|
||||
});
|
||||
|
||||
emitter.on("files", function (path, files) {
|
||||
files.forEach(function (dir) {
|
||||
console.log(dir);
|
||||
});
|
||||
});
|
||||
|
||||
Example
|
||||
Comparisons
|
||||
====
|
||||
|
||||
mkdir -p walk-test/dir1
|
||||
touch walk-test/file-1
|
||||
touch walk-test/file-2
|
||||
touch walk-test/dir1/file-1
|
||||
touch walk-test/dir1/file-2
|
||||
Tested on my `/System` containing 59,490 (+ self) directories (and lots of files).
|
||||
The size of the text output was 6mb.
|
||||
|
||||
node-walk-test
|
||||
----
|
||||
`find`:
|
||||
time bash -c "find /System -type d | wc"
|
||||
59491 97935 6262916
|
||||
|
||||
var walk = require('walk');
|
||||
real 2m27.114s
|
||||
user 0m1.193s
|
||||
sys 0m14.859s
|
||||
|
||||
walk('./walk-test', undefined, function (err, path, errors, dirs, files, links) {
|
||||
if (err) {
|
||||
console.log('ERROR: ');
|
||||
console.log(err);
|
||||
return;
|
||||
}
|
||||
`find.js`:
|
||||
|
||||
dirs.forEach(function (item, i, arr) {
|
||||
if (item.name.match(/trash/i)) {
|
||||
console.log('found a trash');
|
||||
arr.splice(i,1);
|
||||
}
|
||||
});
|
||||
Note that `find.js` omits the start directory
|
||||
|
||||
time bash -c "node examples/find.js /System -type d | wc"
|
||||
59490 97934 6262908
|
||||
|
||||
# Test 1
|
||||
real 2m52.273s
|
||||
user 0m20.374s
|
||||
sys 0m27.800s
|
||||
|
||||
# Test 2
|
||||
real 2m23.725s
|
||||
user 0m18.019s
|
||||
sys 0m23.202s
|
||||
|
||||
# Test 3
|
||||
real 2m50.077s
|
||||
user 0m17.661s
|
||||
sys 0m24.008s
|
||||
|
||||
console.log("PATH: " + path);
|
||||
console.log("SORTED: ");
|
||||
console.log(errors, dirs, files, links);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
(function () {
|
||||
var walk = require("../lib/walk3.js"),
|
||||
emit = walk(process.argv[2] || "/tmp"),
|
||||
util = require('util'),
|
||||
path = require('path');
|
||||
|
||||
// nor the root, nor the node should ever be empty
|
||||
walk.fnodeTypesPlural.forEach(function (fnodeType) {
|
||||
emit.on(fnodeType, function (root, nodes, next) {
|
||||
if (!nodes || !nodes.length || !root) {
|
||||
console.log(fnodeType, "empty set", root, nodes.length); //JSON.stringify(nodes));
|
||||
}
|
||||
next();
|
||||
});
|
||||
});
|
||||
walk.fnodeTypes.forEach(function (fnodeType) {
|
||||
emit.on(fnodeType, function (root, node, next) {
|
||||
if (!node || !node.name || !root) {
|
||||
console.log(fnodeType, "empty item", root, node.name); //JSON.stringify(node));
|
||||
}
|
||||
next();
|
||||
});
|
||||
});
|
||||
emit.on('directory', function (root, dir, next) {
|
||||
console.log(path.join(root, dir.name));
|
||||
next();
|
||||
});
|
||||
/*
|
||||
emit.on('file', function (root, file, next) {
|
||||
console.log(path.join(root, file.name));
|
||||
next();
|
||||
});
|
||||
*/
|
||||
}());
|
||||
|
||||
|
|
@ -3,13 +3,20 @@
|
|||
emit = walk(process.argv[2] || "/tmp");
|
||||
//icount = 0;
|
||||
|
||||
emit.on('directories', function (path, dirs, next) {
|
||||
dirs.forEach(function (dir) {
|
||||
console.log(path + '/' + dir.name);
|
||||
});
|
||||
next();
|
||||
});
|
||||
/*
|
||||
emit.on('directory', function (path, file, next) {
|
||||
//icount += 1;
|
||||
console.log(path + '/' + file.name); // + " " + icount);
|
||||
process.nextTick(next);
|
||||
next();
|
||||
//process.nextTick(next);
|
||||
//setTimeout(next, 100);
|
||||
});
|
||||
/*
|
||||
emit.on('file', function (path, file, next) {
|
||||
console.log("FILE:", file.name, "\n");
|
||||
next();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,230 @@
|
|||
(function () {
|
||||
var fs = require('fs'),
|
||||
Futures = require('futures'),
|
||||
joinPath = require('path').join,
|
||||
util = require('util'),
|
||||
ev = require("events"),
|
||||
emitter = new ev.EventEmitter(),
|
||||
oneNodeEvent = [
|
||||
"file",
|
||||
"directory",
|
||||
"blockDevice",
|
||||
"characterDevice",
|
||||
"symbolicLink",
|
||||
"fifo",
|
||||
"socket"
|
||||
],
|
||||
multiNodeEvents = [
|
||||
// multiple
|
||||
"files",
|
||||
"directories",
|
||||
"blockDevices",
|
||||
"characterDevices",
|
||||
"symbolicLinks",
|
||||
"fifos",
|
||||
"sockets"
|
||||
],
|
||||
eventsTpl = {
|
||||
listeners: function () { return []; },
|
||||
next: function () { return; }
|
||||
},
|
||||
events = {},
|
||||
nexts = {};
|
||||
|
||||
function newVersion() {
|
||||
throw new Error("see README.md at http://github.com/coolaj86/node-walk");
|
||||
}
|
||||
|
||||
function noop() {
|
||||
}
|
||||
|
||||
function remove(arr, obj) {
|
||||
return arr.splice(arr.indexOf(obj), 1);
|
||||
}
|
||||
|
||||
oneNodeEvent.forEach(function (key) {
|
||||
var e = events[key] = {}, next;
|
||||
|
||||
Object.keys(eventsTpl).forEach(function (k) {
|
||||
e[k] = eventsTpl[k]();
|
||||
});
|
||||
|
||||
emitter.on("newListener", function (ev, listener) {
|
||||
var count = 0,
|
||||
num = e.listeners.length + 1;
|
||||
|
||||
e.listeners.push(listener);
|
||||
e.next = function (cb) {
|
||||
cb = noop;
|
||||
return function () {
|
||||
if (count === num) { cb(); }
|
||||
count += 1;
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
// TODO
|
||||
next = function () {
|
||||
|
||||
};
|
||||
});
|
||||
|
||||
function sortNodesByType(path, stats, o, cb) {
|
||||
if (stats.isFile()) {
|
||||
o.files.push(stats);
|
||||
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
|
||||
} else if (stats.isDirectory()) {
|
||||
o.dirs.push(stats);
|
||||
emitter.emit("directory", path, stats, function () {
|
||||
remove(o.dirs, stats);
|
||||
}, (nexts["directory"]||noop)(cb));
|
||||
} else if (stats.isBlockDevice()) {
|
||||
o.blocks.push(stats);
|
||||
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
|
||||
} else if (stats.isCharacterDevice()) {
|
||||
o.chars.push(stats);
|
||||
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
|
||||
} else if (stats.isSymbolicLink()) {
|
||||
o.links.push(stats);
|
||||
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
|
||||
} else if (stats.isFIFO()) {
|
||||
o.fifos.push(stats);
|
||||
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
|
||||
} else if (stats.isSocket()) {
|
||||
o.sockets.push(stats);
|
||||
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
|
||||
} else {
|
||||
// emitter.emit("error", stats);
|
||||
util.debug(stats.name + 'is not of any node type');
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
import os
|
||||
from os.path import join, getsize
|
||||
for root, dirs, files in os.walk('python/Lib/email'):
|
||||
print root, "consumes",
|
||||
print sum(getsize(join(root, name)) for name in files),
|
||||
print "bytes in", len(files), "non-directory files"
|
||||
if 'CVS' in dirs:
|
||||
dirs.remove('CVS') # don't visit CVS directories
|
||||
*/
|
||||
|
||||
/*
|
||||
fs.walk(path, function ({ err, root, dirs, files }) {}, {
|
||||
// currently ignored
|
||||
topdown: boolean,
|
||||
onerror: boolean, // ignored
|
||||
followLinks: boolean // lstat or stat
|
||||
});
|
||||
*/
|
||||
|
||||
function walk(firstPath, options, callback) {
|
||||
options = options || {};
|
||||
var fstat = options.followLinks ? fs.stat : fs.lstat,
|
||||
subscription = Futures.subscription();
|
||||
|
||||
if (callback) { subscription.subscribe(callback); }
|
||||
|
||||
function readDir(path) {
|
||||
var p = Futures.promise();
|
||||
|
||||
fs.readdir(path, function (err, files) {
|
||||
if (err) {
|
||||
err.path = path;
|
||||
subscription.deliver(err, path);
|
||||
// Signal the completion of this readdir attempt
|
||||
p.fulfill();
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO fix futures sequence to not require a first function like this
|
||||
var s = Futures.sequence(function(n){n();}),
|
||||
nodes = [],
|
||||
o = {
|
||||
errors: [],
|
||||
dirs: [],
|
||||
files: [],
|
||||
links: [],
|
||||
blocks: [],
|
||||
chars: [],
|
||||
fifos: [],
|
||||
sockets: []
|
||||
};
|
||||
|
||||
files.forEach(function (file) {
|
||||
// pushes onto the sequence stack without recursion
|
||||
s.then(function (next) {
|
||||
fstat(joinPath(path, file), function (err, stats) {
|
||||
stats = stats || {};
|
||||
stats.name = file;
|
||||
nodes.push(stats);
|
||||
|
||||
if (err) {
|
||||
stats.err = err;
|
||||
o.errors.push(stats);
|
||||
} else {
|
||||
sortNodesByType(path, stats, o);
|
||||
}
|
||||
|
||||
next();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
s.then(function (next) {
|
||||
var s2 = Futures.sequence(function(n){n();});
|
||||
if (nodes.length > 0) {
|
||||
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
|
||||
if (o.errors.length > 0) {
|
||||
emitter.emit("errors", path, o.errors);
|
||||
}
|
||||
if (o.dirs.length > 0) {
|
||||
emitter.emit("directories", path, o.dirs);
|
||||
}
|
||||
if (o.files.length > 0) {
|
||||
emitter.emit("files", path, o.files);
|
||||
}
|
||||
if (o.links.length > 0) {
|
||||
emitter.emit("symbolicLinks", path, o.links);
|
||||
}
|
||||
if (o.blocks.length > 0) {
|
||||
emitter.emit("blockDevices", path, o.blocks);
|
||||
}
|
||||
if (o.chars.length > 0) {
|
||||
emitter.emit("characterDevices", path, o.chars);
|
||||
}
|
||||
if (o.fifos.length > 0) {
|
||||
emitter.emit("fifos", path, o.fifos);
|
||||
}
|
||||
if (o.sockets.length > 0) {
|
||||
emitter.emit("sockets", path, o.fifos);
|
||||
}
|
||||
p.fulfill();
|
||||
|
||||
o.dirs.forEach(function (dir) {
|
||||
s2.then(function (next2) {
|
||||
readDir(joinPath(path, dir.name))
|
||||
.when(function () { next2(); });
|
||||
});
|
||||
});
|
||||
|
||||
next();
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
return p.passable();
|
||||
}
|
||||
|
||||
readDir(firstPath) //.whenever(callback);
|
||||
|
||||
emitter.whenever = subscription.subscribe;
|
||||
return emitter;
|
||||
}
|
||||
|
||||
newVersion.walk = walk;
|
||||
newVersion.remove = remove;
|
||||
module.exports = newVersion;
|
||||
}());
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
// 2011-11-25 jorge@jorgechamorro.com
|
||||
|
||||
function walk (file, cb) {
|
||||
var fs = require('fs');
|
||||
var q= [];
|
||||
var queue= [q];
|
||||
walk2();
|
||||
|
||||
function walk2 () {
|
||||
cb(file);
|
||||
fs.lstat(file, function (err, stat) {
|
||||
if (err || !stat.isDirectory()) return next();
|
||||
getDirectory(function (files) {
|
||||
queue.push(q= files);
|
||||
next();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function next () {
|
||||
if (q.length) {
|
||||
file= q.pop();
|
||||
walk2();
|
||||
}
|
||||
else if (queue.length-= 1) {
|
||||
q= queue[queue.length-1];
|
||||
next();
|
||||
}
|
||||
}
|
||||
|
||||
function getDirectory (cb) {
|
||||
fs.readdir(file, function(err, files) {
|
||||
if (!files) return;
|
||||
//if (err) throw Error(err);
|
||||
files.sort(sort);
|
||||
files.forEach(fullPath);
|
||||
cb(files);
|
||||
});
|
||||
}
|
||||
|
||||
function fullPath (v,i,o) {
|
||||
o[i]= [file, '/', v].join('');
|
||||
}
|
||||
|
||||
function sort (a,b) {
|
||||
a= a.toLowerCase();
|
||||
b= b.toLowerCase();
|
||||
if (a > b) return -1;
|
||||
if (a < b) return 1;
|
||||
else return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// your callback here
|
||||
var ctr= 0;
|
||||
function callBack (file) { console.log( ["[", ++ctr, "] ", file].join('') ) };
|
||||
|
||||
process.argv.forEach(function(val, index, array) {
|
||||
if (index > 1) walk(val, callBack);
|
||||
});
|
||||
333
lib/walk.js
333
lib/walk.js
|
|
@ -1,230 +1,157 @@
|
|||
(function () {
|
||||
// TODO
|
||||
// * add types by listener dynamically
|
||||
// * unroll loops for better readability?
|
||||
// * should emitted errors wait for `next()`?
|
||||
(function (undefined) {
|
||||
var fs = require('fs'),
|
||||
Futures = require('futures'),
|
||||
joinPath = require('path').join,
|
||||
upath = require('path'),
|
||||
util = require('util'),
|
||||
ev = require("events"),
|
||||
emitter = new ev.EventEmitter(),
|
||||
oneNodeEvent = [
|
||||
"file",
|
||||
"directory",
|
||||
"blockDevice",
|
||||
"characterDevice",
|
||||
"symbolicLink",
|
||||
"fifo",
|
||||
"socket"
|
||||
Futures = require('futures'),
|
||||
events = require('events'),
|
||||
noop = function () {},
|
||||
// "FIFO" isn't easy to convert to camelCame and back reliably
|
||||
isFnodeTypes = [
|
||||
"isFile", "isDirectory", "isBlockDevice", "isCharacterDevice", "isSymbolicLink", "isFIFO", "isSocket"
|
||||
],
|
||||
multiNodeEvents = [
|
||||
// multiple
|
||||
"files",
|
||||
"directories",
|
||||
"blockDevices",
|
||||
"characterDevices",
|
||||
"symbolicLinks",
|
||||
"fifos",
|
||||
"sockets"
|
||||
fnodeTypes = [
|
||||
"file", "directory", "blockDevice", "characterDevice", "symbolicLink", "FIFO", "socket"
|
||||
],
|
||||
eventsTpl = {
|
||||
listeners: function () { return []; },
|
||||
next: function () { return; }
|
||||
},
|
||||
events = {},
|
||||
nexts = {};
|
||||
fnodeTypesPlural = [
|
||||
"files", "directories", "blockDevices", "characterDevices", "symbolicLinks", "FIFOs", "sockets"
|
||||
];
|
||||
|
||||
function newVersion() {
|
||||
throw new Error("see README.md at http://github.com/coolaj86/node-walk");
|
||||
}
|
||||
// Create a new walk instance
|
||||
function create(path, options) {
|
||||
var emitter = new events.EventEmitter(),
|
||||
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
|
||||
|
||||
function noop() {
|
||||
}
|
||||
|
||||
function remove(arr, obj) {
|
||||
return arr.splice(arr.indexOf(obj), 1);
|
||||
}
|
||||
// Get the current number of listeners (which may change)
|
||||
// Emit events to each listener
|
||||
// Wait for all listeners to `next()` before continueing
|
||||
// (in theory this may avoid disk thrashing)
|
||||
function emitSingleEvents(path, stats, next) {
|
||||
var num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
|
||||
|
||||
oneNodeEvent.forEach(function (key) {
|
||||
var e = events[key] = {}, next;
|
||||
function nextWhenReady() {
|
||||
num -= 1;
|
||||
if (0 === num) { next(); }
|
||||
}
|
||||
|
||||
Object.keys(eventsTpl).forEach(function (k) {
|
||||
e[k] = eventsTpl[k]();
|
||||
});
|
||||
|
||||
emitter.on("newListener", function (ev, listener) {
|
||||
var count = 0,
|
||||
num = e.listeners.length + 1;
|
||||
|
||||
e.listeners.push(listener);
|
||||
e.next = function (cb) {
|
||||
cb = noop;
|
||||
return function () {
|
||||
if (count === num) { cb(); }
|
||||
count += 1;
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
// TODO
|
||||
next = function () {
|
||||
|
||||
};
|
||||
});
|
||||
|
||||
function sortNodesByType(path, stats, o, cb) {
|
||||
if (stats.isFile()) {
|
||||
o.files.push(stats);
|
||||
emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
|
||||
} else if (stats.isDirectory()) {
|
||||
o.dirs.push(stats);
|
||||
emitter.emit("directory", path, stats, function () {
|
||||
remove(o.dirs, stats);
|
||||
}, (nexts["directory"]||noop)(cb));
|
||||
} else if (stats.isBlockDevice()) {
|
||||
o.blocks.push(stats);
|
||||
emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
|
||||
} else if (stats.isCharacterDevice()) {
|
||||
o.chars.push(stats);
|
||||
emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
|
||||
} else if (stats.isSymbolicLink()) {
|
||||
o.links.push(stats);
|
||||
emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
|
||||
} else if (stats.isFIFO()) {
|
||||
o.fifos.push(stats);
|
||||
emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
|
||||
} else if (stats.isSocket()) {
|
||||
o.sockets.push(stats);
|
||||
emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
|
||||
} else {
|
||||
// emitter.emit("error", stats);
|
||||
util.debug(stats.name + 'is not of any node type');
|
||||
emitter.emit(stats.type, path, stats, nextWhenReady);
|
||||
emitter.emit("node", path, stats, nextWhenReady);
|
||||
nextWhenReady();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
import os
|
||||
from os.path import join, getsize
|
||||
for root, dirs, files in os.walk('python/Lib/email'):
|
||||
print root, "consumes",
|
||||
print sum(getsize(join(root, name)) for name in files),
|
||||
print "bytes in", len(files), "non-directory files"
|
||||
if 'CVS' in dirs:
|
||||
dirs.remove('CVS') # don't visit CVS directories
|
||||
*/
|
||||
|
||||
/*
|
||||
fs.walk(path, function ({ err, root, dirs, files }) {}, {
|
||||
// currently ignored
|
||||
topdown: boolean,
|
||||
onerror: boolean, // ignored
|
||||
followLinks: boolean // lstat or stat
|
||||
});
|
||||
*/
|
||||
// Since the risk for disk thrashing among anything
|
||||
// other than files is relatively low, all types are
|
||||
// emitted at once, but all must complete before advancing
|
||||
function emitPluralEvents(path, nodes, next) {
|
||||
var num = 1;
|
||||
|
||||
function walk(firstPath, options, callback) {
|
||||
options = options || {};
|
||||
var fstat = options.followLinks ? fs.stat : fs.lstat,
|
||||
subscription = Futures.subscription();
|
||||
function nextWhenReady() {
|
||||
num -= 1;
|
||||
if (0 === num) { next(); }
|
||||
}
|
||||
|
||||
if (callback) { subscription.subscribe(callback); }
|
||||
fnodeTypesPlural.concat(["nodes", "errors"]).forEach(function (fnodeType) {
|
||||
if (0 === nodes[fnodeType].length) { return; }
|
||||
num += emitter.listeners(fnodeType).length;
|
||||
emitter.emit(fnodeType, path, nodes[fnodeType], nextWhenReady);
|
||||
});
|
||||
nextWhenReady();
|
||||
}
|
||||
|
||||
function readDir(path) {
|
||||
var p = Futures.promise();
|
||||
|
||||
fs.readdir(path, function (err, files) {
|
||||
if (err) {
|
||||
err.path = path;
|
||||
subscription.deliver(err, path);
|
||||
// Signal the completion of this readdir attempt
|
||||
p.fulfill();
|
||||
// Determine each file node's type
|
||||
//
|
||||
function sortFnodesByType(path, stats, fnodes, nextFile) {
|
||||
isFnodeTypes.forEach(function (isType, i) {
|
||||
if (stats[isType]()) {
|
||||
if (stats.type) { throw new Error("is_" + type + " and " + isType); }
|
||||
stats.type = fnodeTypes[i];
|
||||
fnodes[fnodeTypesPlural[i]].push(stats);
|
||||
// TODO throw to break;
|
||||
}
|
||||
});
|
||||
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
|
||||
emitSingleEvents(path, stats, nextFile);
|
||||
}
|
||||
|
||||
|
||||
// Asynchronously get the stats
|
||||
//
|
||||
function getStats(path, files, walkDirs) {
|
||||
var nodeGroups = {};
|
||||
|
||||
fnodeTypesPlural.concat("nodes", "errors").forEach(function (fnodeTypePlural) {
|
||||
nodeGroups[fnodeTypePlural] = [];
|
||||
});
|
||||
|
||||
function nextFile() {
|
||||
var file = files.pop(), dirs = [], fnames = [];
|
||||
|
||||
if (undefined === file) {
|
||||
emitPluralEvents(path, nodeGroups, function () {
|
||||
nodeGroups.directories.forEach(function (dir) {
|
||||
dirs.push(dir.name);
|
||||
});
|
||||
walkDirs(dirs);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO fix futures sequence to not require a first function like this
|
||||
var s = Futures.sequence(function(n){n();}),
|
||||
nodes = [],
|
||||
o = {
|
||||
errors: [],
|
||||
dirs: [],
|
||||
files: [],
|
||||
links: [],
|
||||
blocks: [],
|
||||
chars: [],
|
||||
fifos: [],
|
||||
sockets: []
|
||||
};
|
||||
|
||||
files.forEach(function (file) {
|
||||
// pushes onto the sequence stack without recursion
|
||||
s.then(function (next) {
|
||||
fstat(joinPath(path, file), function (err, stats) {
|
||||
stats = stats || {};
|
||||
stats.name = file;
|
||||
nodes.push(stats);
|
||||
|
||||
if (err) {
|
||||
stats.err = err;
|
||||
o.errors.push(stats);
|
||||
} else {
|
||||
sortNodesByType(path, stats, o);
|
||||
}
|
||||
|
||||
next();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
s.then(function (next) {
|
||||
var s2 = Futures.sequence(function(n){n();});
|
||||
if (nodes.length > 0) {
|
||||
subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
|
||||
if (o.errors.length > 0) {
|
||||
emitter.emit("errors", path, o.errors);
|
||||
}
|
||||
if (o.dirs.length > 0) {
|
||||
emitter.emit("directories", path, o.dirs);
|
||||
}
|
||||
if (o.files.length > 0) {
|
||||
emitter.emit("files", path, o.files);
|
||||
}
|
||||
if (o.links.length > 0) {
|
||||
emitter.emit("symbolicLinks", path, o.links);
|
||||
}
|
||||
if (o.blocks.length > 0) {
|
||||
emitter.emit("blockDevices", path, o.blocks);
|
||||
}
|
||||
if (o.chars.length > 0) {
|
||||
emitter.emit("characterDevices", path, o.chars);
|
||||
}
|
||||
if (o.fifos.length > 0) {
|
||||
emitter.emit("fifos", path, o.fifos);
|
||||
}
|
||||
if (o.sockets.length > 0) {
|
||||
emitter.emit("sockets", path, o.fifos);
|
||||
}
|
||||
p.fulfill();
|
||||
|
||||
o.dirs.forEach(function (dir) {
|
||||
s2.then(function (next2) {
|
||||
readDir(joinPath(path, dir.name))
|
||||
.when(function () { next2(); });
|
||||
});
|
||||
});
|
||||
|
||||
next();
|
||||
fstat(upath.join(path, file), function (err, stats) {
|
||||
//console.log("`stat`ed file " + file);
|
||||
stats = stats || {};
|
||||
stats.name = file;
|
||||
nodeGroups.nodes.push(stats);
|
||||
if (err) {
|
||||
stats.error = err;
|
||||
stats.type = 'error';
|
||||
nodeGroups.errors.push(stats);
|
||||
//emitter.emit('fileError', path, stats, noop);
|
||||
return nextFile();
|
||||
}
|
||||
sortFnodesByType(path, stats, nodeGroups, nextFile);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
return p.passable();
|
||||
}
|
||||
nextFile();
|
||||
}
|
||||
|
||||
readDir(firstPath) //.whenever(callback);
|
||||
function walk(path, next) {
|
||||
fs.readdir(path, function (err, nodes) {
|
||||
if (err) {
|
||||
emitter.emit('directoryError', path, { error: err, name: path }, noop);
|
||||
return next(); /*TODO*/ throw err;
|
||||
}
|
||||
getStats(path, nodes, function (dirs) {
|
||||
walkDirs(path, dirs, next);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
emitter.whenever = subscription.subscribe;
|
||||
function walkDirs(path, dirs, cb) {
|
||||
function nextDir() {
|
||||
var dir = dirs.pop();
|
||||
if (undefined === dir) {
|
||||
delete dirs;
|
||||
return cb();
|
||||
}
|
||||
walk(upath.join(path, dir), nextDir);
|
||||
}
|
||||
nextDir();
|
||||
}
|
||||
|
||||
walk(upath.normalize(path), function () {
|
||||
//throw Error("hey");
|
||||
//console.log("Utterly Done!");
|
||||
});
|
||||
return emitter;
|
||||
}
|
||||
|
||||
newVersion.walk = walk;
|
||||
newVersion.remove = remove;
|
||||
module.exports = newVersion;
|
||||
module.exports = create;
|
||||
module.exports.isFnodeTypes = isFnodeTypes;
|
||||
module.exports.fnodeTypes = fnodeTypes;
|
||||
module.exports.fnodeTypesPlural = fnodeTypesPlural;
|
||||
}());
|
||||
|
|
|
|||
138
lib/walk2.js
138
lib/walk2.js
|
|
@ -1,138 +0,0 @@
|
|||
(function () {
|
||||
var fs = require('fs'),
|
||||
upath = require('path'),
|
||||
util = require('util'),
|
||||
Futures = require('futures'),
|
||||
events = require('events'),
|
||||
emitter = new events.EventEmitter(),
|
||||
oneNodeEvents = [
|
||||
"file",
|
||||
"directory",
|
||||
"blockDevice",
|
||||
"characterDevice",
|
||||
"symbolicLink",
|
||||
"fifo",
|
||||
"socket"
|
||||
],
|
||||
multiNodeEvents = [
|
||||
// multiple
|
||||
"files",
|
||||
"directories",
|
||||
"blockDevices",
|
||||
"characterDevices",
|
||||
"symbolicLinks",
|
||||
"fifos",
|
||||
"sockets"
|
||||
],
|
||||
fstat;
|
||||
|
||||
function sortNodesByType(path, stats, o, next) {
|
||||
var type, listeners, num, count;
|
||||
|
||||
if (stats.isFile()) {
|
||||
type = "file";
|
||||
o.files.push(stats);
|
||||
} else if (stats.isDirectory()) {
|
||||
type = "directory";
|
||||
o.dirs.push(stats);
|
||||
} else if (stats.isBlockDevice()) {
|
||||
type = "blockDevice";
|
||||
o.blocks.push(stats);
|
||||
} else if (stats.isCharacterDevice()) {
|
||||
type = "characterDevice";
|
||||
o.chars.push(stats);
|
||||
} else if (stats.isSymbolicLink()) {
|
||||
type = "symbolicLink";
|
||||
o.links.push(stats);
|
||||
} else if (stats.isFIFO()) {
|
||||
type = "fifo";
|
||||
o.fifos.push(stats);
|
||||
} else if (stats.isSocket()) {
|
||||
type = "socket";
|
||||
o.sockets.push(stats);
|
||||
} else {
|
||||
throw new Error(upath.join(path,stats.name) + 'is not of any tested node type');
|
||||
}
|
||||
|
||||
listeners = emitter.listeners(type);
|
||||
// get the current number of listeners (may change)
|
||||
num = listeners.length;
|
||||
if (!num) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
// join all; once all listeners have responded, continue
|
||||
count = 0;
|
||||
emitter.emit(type, path, stats, function () {
|
||||
count += 1;
|
||||
if (num === count) {
|
||||
next();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function handleFiles(path, files) {
|
||||
var s = Futures.sequence(),
|
||||
nodes = [],
|
||||
o = {
|
||||
errors: [],
|
||||
dirs: [],
|
||||
files: [],
|
||||
links: [],
|
||||
blocks: [],
|
||||
chars: [],
|
||||
fifos: [],
|
||||
sockets: []
|
||||
};
|
||||
|
||||
files.forEach(function (file) {
|
||||
s.then(function (next) {
|
||||
fstat(upath.join(path, file), function (err, stats) {
|
||||
if (err) {
|
||||
util.debug("[Error] " + util.inspect(err));
|
||||
next();
|
||||
return;
|
||||
}
|
||||
stats.name = file;
|
||||
sortNodesByType(path, stats, o, next);
|
||||
});
|
||||
});
|
||||
});
|
||||
s.then(function (next) {
|
||||
// TODO copycat the emitters above
|
||||
next();
|
||||
});
|
||||
return s;
|
||||
}
|
||||
|
||||
function handlePath(path) {
|
||||
|
||||
}
|
||||
|
||||
function walk(fullpath, options) {
|
||||
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
|
||||
|
||||
var path, file, len, s;
|
||||
|
||||
upath.normalize(fullpath);
|
||||
|
||||
len = fullpath.length - 1;
|
||||
if (len > 1 && '/' === fullpath[len]) {
|
||||
fullpath = fullpath.substr(0, len);
|
||||
}
|
||||
|
||||
path = upath.dirname(fullpath);
|
||||
file = upath.basename(fullpath);
|
||||
|
||||
s = handleFiles(path, [file]);
|
||||
s(function (next) { next(); });
|
||||
return emitter;
|
||||
}
|
||||
|
||||
var walker = walk("/System");
|
||||
walker.on("directory", function (path, dir, next) {
|
||||
console.log(path, dir.name);
|
||||
next();
|
||||
});
|
||||
}());
|
||||
156
lib/walk3.js
156
lib/walk3.js
|
|
@ -1,156 +0,0 @@
|
|||
(function (undefined) {
|
||||
var fs = require('fs'),
|
||||
upath = require('path'),
|
||||
util = require('util'),
|
||||
Futures = require('futures'),
|
||||
events = require('events');
|
||||
|
||||
function create(path, options) {
|
||||
// TODO add types by listener dynamically
|
||||
var emitter = new events.EventEmitter(),
|
||||
oneNodeEvents = [
|
||||
"file",
|
||||
"directory",
|
||||
"blockDevice",
|
||||
"characterDevice",
|
||||
"symbolicLink",
|
||||
"FIFO",
|
||||
"socket"
|
||||
],
|
||||
oneNodeTypes = [
|
||||
"isFile",
|
||||
"isDirectory",
|
||||
"isBlockDevice",
|
||||
"isCharacterDevice",
|
||||
"isSymbolicLink",
|
||||
"isFIFO",
|
||||
"isSocket"
|
||||
],
|
||||
multiNodeEvents = [
|
||||
// multiple
|
||||
"files",
|
||||
"directories",
|
||||
"blockDevices",
|
||||
"characterDevices",
|
||||
"symbolicLinks",
|
||||
"FIFOs",
|
||||
"sockets"
|
||||
],
|
||||
fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
|
||||
|
||||
function normalize(path) {
|
||||
return path;
|
||||
var index;
|
||||
|
||||
path = upath.normalize(path);
|
||||
index = path.length - 1;
|
||||
if (index > 0 && '/' === path[index]) {
|
||||
path = path.substr(0, index);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
function sortNodesByType(path, stats, nodes, next) {
|
||||
// This could easily be unrolled
|
||||
// but in this case concise was more readable for me
|
||||
oneNodeTypes.forEach(function (isType, i) {
|
||||
if (stats[isType]()) {
|
||||
if (stats.type) { throw new Error("is_" + type + " and " + isType); }
|
||||
stats.type = oneNodeEvents[i];
|
||||
nodes[multiNodeEvents[i]].push(stats);
|
||||
// TODO throw to break;
|
||||
}
|
||||
});
|
||||
if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
|
||||
function emitSingleEvents(stats, next) {
|
||||
var num;
|
||||
|
||||
// get the current number of listeners (may change)
|
||||
num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
|
||||
|
||||
function nextWhenReady() {
|
||||
num -= 1;
|
||||
if (0 === num) { next(); }
|
||||
}
|
||||
nextWhenReady();
|
||||
|
||||
// Total number of listeners
|
||||
emitter.emit(stats.type, path, stats, function () {
|
||||
nextWhenReady();
|
||||
});
|
||||
|
||||
emitter.emit("node", path, stats, function () {
|
||||
nextWhenReady();
|
||||
});
|
||||
} // emitSingleEvents
|
||||
emitSingleEvents(stats, next);
|
||||
}
|
||||
|
||||
function getStats(path, files, walkDirs) {
|
||||
var nodes = [], o = {};
|
||||
|
||||
// TODO unroll for better readability
|
||||
multiNodeEvents.forEach(function (eventType) {
|
||||
o[eventType] = [];
|
||||
});
|
||||
|
||||
function nextFile() {
|
||||
var file = files.pop(), dirs = [], fnames = [];
|
||||
|
||||
if (undefined === file) {
|
||||
o.directories.forEach(function (dir) {
|
||||
dirs.push(dir.name);
|
||||
});
|
||||
return walkDirs(dirs);
|
||||
}
|
||||
|
||||
fstat(upath.join(path, file), function (err, stats) {
|
||||
//console.log("`stat`ed file " + file);
|
||||
stats = stats || {};
|
||||
stats.name = file;
|
||||
nodes.push(stats);
|
||||
|
||||
if (err) {
|
||||
stats.error = err
|
||||
// TODO single error emittor
|
||||
o.errors.push(stats);
|
||||
util.debug("[Error 1] " + util.inspect(err));
|
||||
return nextFile();
|
||||
}
|
||||
sortNodesByType(path, stats, o, nextFile);
|
||||
});
|
||||
}
|
||||
nextFile();
|
||||
}
|
||||
|
||||
function walk(path, next) {
|
||||
fs.readdir(path, function (err, nodes) {
|
||||
if (err) { return next(); /*TODO*/ throw err; }
|
||||
getStats(path, nodes, function (dirs) {
|
||||
walkDirs(path, dirs, function () {
|
||||
next();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function walkDirs(path, dirs, cb) {
|
||||
function nextDir() {
|
||||
var dir = dirs.pop();
|
||||
if (undefined === dir) {
|
||||
delete dirs;
|
||||
return cb();
|
||||
}
|
||||
walk(upath.join(path, dir), nextDir);
|
||||
}
|
||||
nextDir();
|
||||
}
|
||||
|
||||
walk(normalize(path), function () {
|
||||
//throw Error("hey");
|
||||
//console.log("Utterly Done!");
|
||||
});
|
||||
return emitter;
|
||||
}
|
||||
module.exports = create;
|
||||
}());
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name" : "walk",
|
||||
"description" : "A node port of python's os.walk",
|
||||
"url" : "github.com/coolaj86/walk",
|
||||
"keywords" : ["util", "os", "fs", "walk"],
|
||||
"url" : "github.com/coolaj86/node-walk",
|
||||
"keywords" : ["util", "os", "sys", "fs", "walk", "walkSync"],
|
||||
"author" : "AJ ONeal <coolaj86@gmail.com>",
|
||||
"contributors" : [],
|
||||
"dependencies" : ["futures"],
|
||||
"dependencies" : [],
|
||||
"lib" : "lib",
|
||||
"main" : "./lib/walk.js",
|
||||
"version" : "0.9.2"
|
||||
"version" : "1.0.0"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue