From 8d4dc9804ab8c9f9e5bfab21de7bdef682a1ae24 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 2 May 2011 21:11:03 -0600 Subject: [PATCH] added synchronous version --- README.md | 12 ++- lib/walk-async-only.js | 92 ++++++++++++++++++++++ lib/walk.js | 170 +++++++++++++++++++++++++++-------------- testSync.js | 40 ++++++++++ walk-test.sh | 5 +- 5 files changed, 258 insertions(+), 61 deletions(-) create mode 100644 lib/walk-async-only.js create mode 100644 testSync.js diff --git a/README.md b/README.md index 3076763..8e62ea6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This is somewhat of a port python's `os.walk`, but using Node.JS conventions. * Asynchronous * Chronological (optionally) * Built-in flow-control + * includes Synchronous version (same API as Asynchronous) As few file descriptors are opened at a time as possible. This is particularly well suited for single hard disks which are not flash or solid state. @@ -21,6 +22,10 @@ Installation Usage ==== +Both Asynchronous and Synchronous versions are provided. + +The Synchronous version still uses callbacks, so it is safe to use with other Asynchronous functions and will still work as expected. + var walk = require('walk'), fs = require('fs'), options, @@ -30,7 +35,10 @@ Usage followLinks: false, }; - walker = walk("/tmp", options); + walker = walk.walk("/tmp", options); + + // OR + // walker = walk.walkSync("/tmp", options); walker.on("names", function (root, nodeNamesArray) { nodeNames.sort(function (a, b) { @@ -148,4 +156,4 @@ Note that `find.js` omits the start directory user 0m17.661s sys 0m24.008s -In conclusion node.js asynchronous walk is much slower than regular "find". \ No newline at end of file +In conclusion node.js asynchronous walk is much slower than regular "find". diff --git a/lib/walk-async-only.js b/lib/walk-async-only.js new file mode 100644 index 0000000..319f5ec --- /dev/null +++ b/lib/walk-async-only.js @@ -0,0 +1,92 @@ +(function () { + "use strict" + + // Array.prototype.forEachAsync(next, item, i, collection) + require('futures/forEachAsync'); + + function noop() {} + + var fs = require('fs'), + EventEmitter = require('events').EventEmitter, + TypeEmitter = require('./node-type-emitter'); + + // 2010-11-25 jorge@jorgechamorro.com + function create(pathname, cb) { + var emitter = new EventEmitter(), + q = [], + queue = [q], + curpath; + + function walk() { + fs.readdir(curpath, function(err, files) { + if (err) { + emitter.emit('directoryError', curpath, { error: err }, noop); + //emitter.emit('error', curpath, { error: err }); + } + // XXX bug was here. next() was omitted + if (!files || 0 == files.length) { + return next(); + } + + var fnodeGroups = TypeEmitter.createNodeGroups(); + + // TODO could allow user to selectively stat + // and don't stat if there are no stat listeners + emitter.emit('names', curpath, files, noop); + files.forEachAsync(function (cont, file) { + emitter.emit('name', curpath, file, noop); + fs.lstat(curpath + '/' + file, function (err, stat) { + stat = stat || {}; + stat.name = file; + if (err) { + stat.error = err; + //emitter.emit('error', curpath, stat); + emitter.emit('nodeError', curpath, stat, noop); + fnodeGroups.errors.push(stat); + cont(); + } else { + TypeEmitter.sortFnodesByType(stat, fnodeGroups); + TypeEmitter.emitNodeType(emitter, curpath, stat, cont); + } + }); + }).then(function () { + if (fnodeGroups.errors.length) { + emitter.emit('errors', curpath, fnodeGroups.errors, noop); + } + TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () { + var dirs = []; + fnodeGroups.directories.forEach(function (stat) { + dirs.push(stat.name); + }); + dirs.forEach(fullPath); + queue.push(q = dirs); + next(); + }); + }); + }); + } + + function next() { + if (q.length) { + curpath = q.pop(); + return walk(); + } + if (queue.length -= 1) { + q = queue[queue.length-1]; + return next(); + } + emitter.emit('end'); + } + + function fullPath(v,i,o) { + o[i]= [curpath, '/', v].join(''); + } + + curpath = pathname; + walk(); + + return emitter; + } + + module.exports = create; +}()); diff --git a/lib/walk.js b/lib/walk.js index 319f5ec..4ff2745 100644 --- a/lib/walk.js +++ b/lib/walk.js @@ -1,3 +1,4 @@ +// Adapted from work by jorge@jorgechamorro.com on 2010-11-25 (function () { "use strict" @@ -10,60 +11,104 @@ EventEmitter = require('events').EventEmitter, TypeEmitter = require('./node-type-emitter'); - // 2010-11-25 jorge@jorgechamorro.com - function create(pathname, cb) { - var emitter = new EventEmitter(), - q = [], - queue = [q], - curpath; + function create(pathname, options, sync) { + var emitter = new EventEmitter() + , q = [] + , queue = [q] + , curpath; + + function readdirHandler(err, files) { + var fnodeGroups = TypeEmitter.createNodeGroups(); + + function filesHandler(cont, file) { + var statPath; + + emitter.emit('name', curpath, file, noop); + + function lstatHandler(err, stat) { + stat = stat || {}; + stat.name = file; + if (err) { + stat.error = err; + //emitter.emit('error', curpath, stat); + emitter.emit('nodeError', curpath, stat, noop); + fnodeGroups.errors.push(stat); + cont(); + } else { + TypeEmitter.sortFnodesByType(stat, fnodeGroups); + TypeEmitter.emitNodeType(emitter, curpath, stat, cont); + } + } + + statPath = curpath + '/' + file; + if (sync) { + try { + lstatHandler(null, fs.lstatSync(statPath)); + } catch(e) { + lstatHandler(e); + } + } else { + fs.lstat(statPath, lstatHandler); + } + } + + function postFilesHandler() { + if (fnodeGroups.errors.length) { + emitter.emit('errors', curpath, fnodeGroups.errors, noop); + } + TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () { + var dirs = []; + fnodeGroups.directories.forEach(function (stat) { + dirs.push(stat.name); + }); + dirs.forEach(fullPath); + queue.push(q = dirs); + next(); + }); + } + + if (err) { + emitter.emit('directoryError', curpath, { error: err }, noop); + //emitter.emit('error', curpath, { error: err }); + } + + if (!files || 0 == files.length) { + return next(); + } + + // TODO could allow user to selectively stat + // and don't stat if there are no stat listeners + emitter.emit('names', curpath, files, noop); + + if (sync) { + files.forEach(function (items) { + filesHandler(noop, items); + }); + postFilesHandler(); + } else { + files.forEachAsync(filesHandler).then(postFilesHandler); + } + } + + function walkSync() { + var err, files; + + try { + files = fs.readdirSync(curpath); + } catch(e) { + err = e; + } + + readdirHandler(err, files); + } function walk() { - fs.readdir(curpath, function(err, files) { - if (err) { - emitter.emit('directoryError', curpath, { error: err }, noop); - //emitter.emit('error', curpath, { error: err }); - } - // XXX bug was here. next() was omitted - if (!files || 0 == files.length) { - return next(); - } + if (sync) { + walkSync(); + return; + } - var fnodeGroups = TypeEmitter.createNodeGroups(); - - // TODO could allow user to selectively stat - // and don't stat if there are no stat listeners - emitter.emit('names', curpath, files, noop); - files.forEachAsync(function (cont, file) { - emitter.emit('name', curpath, file, noop); - fs.lstat(curpath + '/' + file, function (err, stat) { - stat = stat || {}; - stat.name = file; - if (err) { - stat.error = err; - //emitter.emit('error', curpath, stat); - emitter.emit('nodeError', curpath, stat, noop); - fnodeGroups.errors.push(stat); - cont(); - } else { - TypeEmitter.sortFnodesByType(stat, fnodeGroups); - TypeEmitter.emitNodeType(emitter, curpath, stat, cont); - } - }); - }).then(function () { - if (fnodeGroups.errors.length) { - emitter.emit('errors', curpath, fnodeGroups.errors, noop); - } - TypeEmitter.emitNodeTypeGroups(emitter, curpath, fnodeGroups, function () { - var dirs = []; - fnodeGroups.directories.forEach(function (stat) { - dirs.push(stat.name); - }); - dirs.forEach(fullPath); - queue.push(q = dirs); - next(); - }); - }); - }); + fs.readdir(curpath, readdirHandler); } function next() { @@ -72,21 +117,32 @@ return walk(); } if (queue.length -= 1) { - q = queue[queue.length-1]; + q = queue[queue.length - 1]; return next(); } emitter.emit('end'); } - function fullPath(v,i,o) { - o[i]= [curpath, '/', v].join(''); + function fullPath(v, i, o) { + o[i] = [curpath, '/', v].join(''); } curpath = pathname; - walk(); - + + if (sync) { + process.nextTick(walk); + } else { + walk(); + } + return emitter; } - module.exports = create; + exports.walk = function (path, opts) { + return create(path, opts, false); + }; + + exports.walkSync = function (path, opts) { + return create(path, opts, true); + }; }()); diff --git a/testSync.js b/testSync.js new file mode 100644 index 0000000..30da449 --- /dev/null +++ b/testSync.js @@ -0,0 +1,40 @@ +(function () { + "use strict"; + + var walk = require('./lib/walk') + , _ = require('underscore') + , fs = require('fs') + , sync = false + , walker; + + console.log(walk); + + walker = walk.walk("."); + + walker.on("directory", function (root, dirStatsArray, next) { + // dirStatsArray is an array of `stat` objects with the additional attributes + // * type + // * error + // * name + //console.log(_.pluck(dirStatsArray, 'name')); + console.log(root + '/' + dirStatsArray.name); + + next(); + }); + + walker.on("file", function (root, fileStats, next) { + console.log(root + '/' + fileStats.name); + + next(); + }); + + walker.on("errors", function (root, nodeStatsArray, next) { + //console.log(nodeStatsArray); + + next(); + }); + + walker.on("end", function () { + console.log("all done"); + }); +}()); diff --git a/walk-test.sh b/walk-test.sh index 15504fc..4be671f 100755 --- a/walk-test.sh +++ b/walk-test.sh @@ -1,5 +1,6 @@ mkdir -p walk-test/dir1 +touch walk-test/file-0 touch walk-test/file-1 -touch walk-test/file-2 -touch walk-test/dir1/file-1 touch walk-test/dir1/file-2 +touch walk-test/dir1/file-3 +echo "4 files and 2 directories (including '.')"