From e68088526ed08864da9f84e8fb77c4c80cb8d704 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 12 Dec 2010 01:58:43 -0700 Subject: [PATCH] walk is now awesomesauce... but still a memory hog --- README.md | 169 ++++++++------ examples/find-test.js | 35 +++ examples/find.js | 11 +- lib/walk-0.9.js | 230 ++++++++++++++++++ lib/walk-alternate-implementation.js | 60 +++++ lib/walk.js | 333 +++++++++++---------------- lib/walk2.js | 138 ----------- lib/walk3.js | 156 ------------- package.json | 8 +- 9 files changed, 566 insertions(+), 574 deletions(-) create mode 100644 examples/find-test.js create mode 100644 lib/walk-0.9.js create mode 100644 lib/walk-alternate-implementation.js delete mode 100644 lib/walk2.js delete mode 100644 lib/walk3.js diff --git a/README.md b/README.md index 6db84fe..93a6446 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,18 @@ node-walk ==== -A not-quite-port of python's `os.walk`. +A port python's `os.walk`, but using Node.JS conventions. + + * EventEmitter + * Asynchronous + * Chronological (optionally) + * Built-in flow-control + +As few file descriptors are opened at a time as possible. +This is particularly well suited for single hard disks which are not flash or solid state. + +Memory usage is high (120mb for 60,000 dirs), but reduction is being investigated. +Patches welcome. Installation ---- @@ -11,91 +22,107 @@ Installation Usage ==== -All of the examples in the folder included in this repository work and have no typos. - -without emitter ----- - var walk = require('walk').walk, - options = { + options, + walker; + + options = { followLinks: false, - }; + }; - function fileHandler(err, path, errors, dirs, files, links, blocks, chars, fifos, sockets) { - // handle each path - } + walker = walk("path/to/dir", options); - walk("path/to/dir", options, fileHandler); - // This also works - // walk("path/to/dir", options).whenever(fileHandler); + walker.on("directories", function (root, dirStatsArray, next) { + // dirStatsArray is an array of `stat` objects with the additional attributes + // * type + // * error + // * name + + next(); + }); -Single Arguments + walker.on("file", function (root, fileStats, next) { + fs.readFile(file, function () { + // doStuff + next(); + }); + }); - * `err` - Error when reading path (Probably due to permissions). - * `path` - the current path being read + walker.on("errors", function (root, nodeStatsArray, next) { + next(); + }); + +API +==== + +Emitted Values + + * `root` - the containing the files to be inspected + * *stats[Array]* - a single `stats` object or an array with some added attributes + * type - 'file', 'directory', etc + * error + * name - the name of the file, dir, etc + * next - no more files will be read until this is called + +Single Events - fired immediately + + * `directoryError` - Error when `fstat` succeeded, but reading path failed (Probably due to permissions). + * `node` - a `stats` object for a node of any type + * `file` - includes links when `followLinks` is `true` + * `directory` + * `blockDevice` + * `characterDevice` + * `symbolicLink` - always empty when `followLinks` is `true` + * `FIFO` + * `socket` Array Arguments - * `errors` - `fs.stat` encountered on files in the directory - * `dirs` - directories (modification of this array - sorting, removing, etc - affects traversal) - * `files` - regular files (includes links when `followLinks` is `true`) - * `links` - symbolic links (always empty when `followLinks` is `true`) - * `blocks` - block devices - * `chars` - character devices - * `fifos` - FIFOs - * `sockets` - sockets + * `errors` - errors encountered by `fs.stat` when reading ndes in a directory + * `nodes` - an array of `stats` of any type + * `files` + * `directories` - modification of this array - sorting, removing, etc - affects traversal + * `blockDevices` + * `characterDevices` + * `symbolicLinks` + * `FIFOs` + * `sockets` -using emitter ----- +**Warning** when following links, an infinite loop is possible -`errors`, `directories`, `files`, `symbolicLinks` - - var walk = require('walk').walk, - emitter = walk('./walk-test'); - - emitter.on("directories", function (path, dirs) { - // the second directory will not be traversed - dirs.splice(1,1); - dirs.forEach(function (dir) { - console.log(dir); - }); - }); - - emitter.on("files", function (path, files) { - files.forEach(function (dir) { - console.log(dir); - }); - }); - -Example +Comparisons ==== - mkdir -p walk-test/dir1 - touch walk-test/file-1 - touch walk-test/file-2 - touch walk-test/dir1/file-1 - touch walk-test/dir1/file-2 +Tested on my `/System` containing 59,490 (+ self) directories (and lots of files). +The size of the text output was 6mb. -node-walk-test ----- +`find`: + time bash -c "find /System -type d | wc" + 59491 97935 6262916 - var walk = require('walk'); + real 2m27.114s + user 0m1.193s + sys 0m14.859s - walk('./walk-test', undefined, function (err, path, errors, dirs, files, links) { - if (err) { - console.log('ERROR: '); - console.log(err); - return; - } +`find.js`: - dirs.forEach(function (item, i, arr) { - if (item.name.match(/trash/i)) { - console.log('found a trash'); - arr.splice(i,1); - } - }); +Note that `find.js` omits the start directory + + time bash -c "node examples/find.js /System -type d | wc" + 59490 97934 6262908 + + # Test 1 + real 2m52.273s + user 0m20.374s + sys 0m27.800s + + # Test 2 + real 2m23.725s + user 0m18.019s + sys 0m23.202s + + # Test 3 + real 2m50.077s + user 0m17.661s + sys 0m24.008s - console.log("PATH: " + path); - console.log("SORTED: "); - console.log(errors, dirs, files, links); - }); diff --git a/examples/find-test.js b/examples/find-test.js new file mode 100644 index 0000000..3eaeb8b --- /dev/null +++ b/examples/find-test.js @@ -0,0 +1,35 @@ +(function () { + var walk = require("../lib/walk3.js"), + emit = walk(process.argv[2] || "/tmp"), + util = require('util'), + path = require('path'); + + // nor the root, nor the node should ever be empty + walk.fnodeTypesPlural.forEach(function (fnodeType) { + emit.on(fnodeType, function (root, nodes, next) { + if (!nodes || !nodes.length || !root) { + console.log(fnodeType, "empty set", root, nodes.length); //JSON.stringify(nodes)); + } + next(); + }); + }); + walk.fnodeTypes.forEach(function (fnodeType) { + emit.on(fnodeType, function (root, node, next) { + if (!node || !node.name || !root) { + console.log(fnodeType, "empty item", root, node.name); //JSON.stringify(node)); + } + next(); + }); + }); + emit.on('directory', function (root, dir, next) { + console.log(path.join(root, dir.name)); + next(); + }); + /* + emit.on('file', function (root, file, next) { + console.log(path.join(root, file.name)); + next(); + }); + */ +}()); + diff --git a/examples/find.js b/examples/find.js index 82d1d79..39dab76 100644 --- a/examples/find.js +++ b/examples/find.js @@ -3,13 +3,20 @@ emit = walk(process.argv[2] || "/tmp"); //icount = 0; + emit.on('directories', function (path, dirs, next) { + dirs.forEach(function (dir) { + console.log(path + '/' + dir.name); + }); + next(); + }); + /* emit.on('directory', function (path, file, next) { //icount += 1; console.log(path + '/' + file.name); // + " " + icount); - process.nextTick(next); + next(); + //process.nextTick(next); //setTimeout(next, 100); }); - /* emit.on('file', function (path, file, next) { console.log("FILE:", file.name, "\n"); next(); diff --git a/lib/walk-0.9.js b/lib/walk-0.9.js new file mode 100644 index 0000000..ba63b57 --- /dev/null +++ b/lib/walk-0.9.js @@ -0,0 +1,230 @@ +(function () { + var fs = require('fs'), + Futures = require('futures'), + joinPath = require('path').join, + util = require('util'), + ev = require("events"), + emitter = new ev.EventEmitter(), + oneNodeEvent = [ + "file", + "directory", + "blockDevice", + "characterDevice", + "symbolicLink", + "fifo", + "socket" + ], + multiNodeEvents = [ + // multiple + "files", + "directories", + "blockDevices", + "characterDevices", + "symbolicLinks", + "fifos", + "sockets" + ], + eventsTpl = { + listeners: function () { return []; }, + next: function () { return; } + }, + events = {}, + nexts = {}; + + function newVersion() { + throw new Error("see README.md at http://github.com/coolaj86/node-walk"); + } + + function noop() { + } + + function remove(arr, obj) { + return arr.splice(arr.indexOf(obj), 1); + } + + oneNodeEvent.forEach(function (key) { + var e = events[key] = {}, next; + + Object.keys(eventsTpl).forEach(function (k) { + e[k] = eventsTpl[k](); + }); + + emitter.on("newListener", function (ev, listener) { + var count = 0, + num = e.listeners.length + 1; + + e.listeners.push(listener); + e.next = function (cb) { + cb = noop; + return function () { + if (count === num) { cb(); } + count += 1; + }; + }; + }); + + // TODO + next = function () { + + }; + }); + + function sortNodesByType(path, stats, o, cb) { + if (stats.isFile()) { + o.files.push(stats); + emitter.emit("file", path, stats, (nexts["file"]||noop)(cb)); + } else if (stats.isDirectory()) { + o.dirs.push(stats); + emitter.emit("directory", path, stats, function () { + remove(o.dirs, stats); + }, (nexts["directory"]||noop)(cb)); + } else if (stats.isBlockDevice()) { + o.blocks.push(stats); + emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb)); + } else if (stats.isCharacterDevice()) { + o.chars.push(stats); + emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb)); + } else if (stats.isSymbolicLink()) { + o.links.push(stats); + emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb)); + } else if (stats.isFIFO()) { + o.fifos.push(stats); + emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb)); + } else if (stats.isSocket()) { + o.sockets.push(stats); + emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb)); + } else { + // emitter.emit("error", stats); + util.debug(stats.name + 'is not of any node type'); + } + } + + /* + import os + from os.path import join, getsize + for root, dirs, files in os.walk('python/Lib/email'): + print root, "consumes", + print sum(getsize(join(root, name)) for name in files), + print "bytes in", len(files), "non-directory files" + if 'CVS' in dirs: + dirs.remove('CVS') # don't visit CVS directories + */ + + /* + fs.walk(path, function ({ err, root, dirs, files }) {}, { + // currently ignored + topdown: boolean, + onerror: boolean, // ignored + followLinks: boolean // lstat or stat + }); + */ + + function walk(firstPath, options, callback) { + options = options || {}; + var fstat = options.followLinks ? fs.stat : fs.lstat, + subscription = Futures.subscription(); + + if (callback) { subscription.subscribe(callback); } + + function readDir(path) { + var p = Futures.promise(); + + fs.readdir(path, function (err, files) { + if (err) { + err.path = path; + subscription.deliver(err, path); + // Signal the completion of this readdir attempt + p.fulfill(); + return; + } + + // TODO fix futures sequence to not require a first function like this + var s = Futures.sequence(function(n){n();}), + nodes = [], + o = { + errors: [], + dirs: [], + files: [], + links: [], + blocks: [], + chars: [], + fifos: [], + sockets: [] + }; + + files.forEach(function (file) { + // pushes onto the sequence stack without recursion + s.then(function (next) { + fstat(joinPath(path, file), function (err, stats) { + stats = stats || {}; + stats.name = file; + nodes.push(stats); + + if (err) { + stats.err = err; + o.errors.push(stats); + } else { + sortNodesByType(path, stats, o); + } + + next(); + }); + }); + }); + + s.then(function (next) { + var s2 = Futures.sequence(function(n){n();}); + if (nodes.length > 0) { + subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets); + if (o.errors.length > 0) { + emitter.emit("errors", path, o.errors); + } + if (o.dirs.length > 0) { + emitter.emit("directories", path, o.dirs); + } + if (o.files.length > 0) { + emitter.emit("files", path, o.files); + } + if (o.links.length > 0) { + emitter.emit("symbolicLinks", path, o.links); + } + if (o.blocks.length > 0) { + emitter.emit("blockDevices", path, o.blocks); + } + if (o.chars.length > 0) { + emitter.emit("characterDevices", path, o.chars); + } + if (o.fifos.length > 0) { + emitter.emit("fifos", path, o.fifos); + } + if (o.sockets.length > 0) { + emitter.emit("sockets", path, o.fifos); + } + p.fulfill(); + + o.dirs.forEach(function (dir) { + s2.then(function (next2) { + readDir(joinPath(path, dir.name)) + .when(function () { next2(); }); + }); + }); + + next(); + } + }); + + }); + + return p.passable(); + } + + readDir(firstPath) //.whenever(callback); + + emitter.whenever = subscription.subscribe; + return emitter; + } + + newVersion.walk = walk; + newVersion.remove = remove; + module.exports = newVersion; +}()); diff --git a/lib/walk-alternate-implementation.js b/lib/walk-alternate-implementation.js new file mode 100644 index 0000000..ab0fd83 --- /dev/null +++ b/lib/walk-alternate-implementation.js @@ -0,0 +1,60 @@ +// 2011-11-25 jorge@jorgechamorro.com + +function walk (file, cb) { + var fs = require('fs'); + var q= []; + var queue= [q]; + walk2(); + + function walk2 () { + cb(file); + fs.lstat(file, function (err, stat) { + if (err || !stat.isDirectory()) return next(); + getDirectory(function (files) { + queue.push(q= files); + next(); + }); + }); + } + + function next () { + if (q.length) { + file= q.pop(); + walk2(); + } + else if (queue.length-= 1) { + q= queue[queue.length-1]; + next(); + } + } + + function getDirectory (cb) { + fs.readdir(file, function(err, files) { + if (!files) return; + //if (err) throw Error(err); + files.sort(sort); + files.forEach(fullPath); + cb(files); + }); + } + + function fullPath (v,i,o) { + o[i]= [file, '/', v].join(''); + } + + function sort (a,b) { + a= a.toLowerCase(); + b= b.toLowerCase(); + if (a > b) return -1; + if (a < b) return 1; + else return 0; + } +} + +// your callback here +var ctr= 0; +function callBack (file) { console.log( ["[", ++ctr, "] ", file].join('') ) }; + +process.argv.forEach(function(val, index, array) { + if (index > 1) walk(val, callBack); +}); diff --git a/lib/walk.js b/lib/walk.js index ba63b57..e6e3743 100644 --- a/lib/walk.js +++ b/lib/walk.js @@ -1,230 +1,157 @@ -(function () { +// TODO +// * add types by listener dynamically +// * unroll loops for better readability? +// * should emitted errors wait for `next()`? +(function (undefined) { var fs = require('fs'), - Futures = require('futures'), - joinPath = require('path').join, + upath = require('path'), util = require('util'), - ev = require("events"), - emitter = new ev.EventEmitter(), - oneNodeEvent = [ - "file", - "directory", - "blockDevice", - "characterDevice", - "symbolicLink", - "fifo", - "socket" + Futures = require('futures'), + events = require('events'), + noop = function () {}, + // "FIFO" isn't easy to convert to camelCame and back reliably + isFnodeTypes = [ + "isFile", "isDirectory", "isBlockDevice", "isCharacterDevice", "isSymbolicLink", "isFIFO", "isSocket" ], - multiNodeEvents = [ - // multiple - "files", - "directories", - "blockDevices", - "characterDevices", - "symbolicLinks", - "fifos", - "sockets" + fnodeTypes = [ + "file", "directory", "blockDevice", "characterDevice", "symbolicLink", "FIFO", "socket" ], - eventsTpl = { - listeners: function () { return []; }, - next: function () { return; } - }, - events = {}, - nexts = {}; + fnodeTypesPlural = [ + "files", "directories", "blockDevices", "characterDevices", "symbolicLinks", "FIFOs", "sockets" + ]; - function newVersion() { - throw new Error("see README.md at http://github.com/coolaj86/node-walk"); - } + // Create a new walk instance + function create(path, options) { + var emitter = new events.EventEmitter(), + fstat = (options||{}).followLinks ? fs.stat : fs.lstat; - function noop() { - } - function remove(arr, obj) { - return arr.splice(arr.indexOf(obj), 1); - } + // Get the current number of listeners (which may change) + // Emit events to each listener + // Wait for all listeners to `next()` before continueing + // (in theory this may avoid disk thrashing) + function emitSingleEvents(path, stats, next) { + var num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length; - oneNodeEvent.forEach(function (key) { - var e = events[key] = {}, next; + function nextWhenReady() { + num -= 1; + if (0 === num) { next(); } + } - Object.keys(eventsTpl).forEach(function (k) { - e[k] = eventsTpl[k](); - }); - - emitter.on("newListener", function (ev, listener) { - var count = 0, - num = e.listeners.length + 1; - - e.listeners.push(listener); - e.next = function (cb) { - cb = noop; - return function () { - if (count === num) { cb(); } - count += 1; - }; - }; - }); - - // TODO - next = function () { - - }; - }); - - function sortNodesByType(path, stats, o, cb) { - if (stats.isFile()) { - o.files.push(stats); - emitter.emit("file", path, stats, (nexts["file"]||noop)(cb)); - } else if (stats.isDirectory()) { - o.dirs.push(stats); - emitter.emit("directory", path, stats, function () { - remove(o.dirs, stats); - }, (nexts["directory"]||noop)(cb)); - } else if (stats.isBlockDevice()) { - o.blocks.push(stats); - emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb)); - } else if (stats.isCharacterDevice()) { - o.chars.push(stats); - emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb)); - } else if (stats.isSymbolicLink()) { - o.links.push(stats); - emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb)); - } else if (stats.isFIFO()) { - o.fifos.push(stats); - emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb)); - } else if (stats.isSocket()) { - o.sockets.push(stats); - emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb)); - } else { - // emitter.emit("error", stats); - util.debug(stats.name + 'is not of any node type'); + emitter.emit(stats.type, path, stats, nextWhenReady); + emitter.emit("node", path, stats, nextWhenReady); + nextWhenReady(); } - } - /* - import os - from os.path import join, getsize - for root, dirs, files in os.walk('python/Lib/email'): - print root, "consumes", - print sum(getsize(join(root, name)) for name in files), - print "bytes in", len(files), "non-directory files" - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories - */ - /* - fs.walk(path, function ({ err, root, dirs, files }) {}, { - // currently ignored - topdown: boolean, - onerror: boolean, // ignored - followLinks: boolean // lstat or stat - }); - */ + // Since the risk for disk thrashing among anything + // other than files is relatively low, all types are + // emitted at once, but all must complete before advancing + function emitPluralEvents(path, nodes, next) { + var num = 1; - function walk(firstPath, options, callback) { - options = options || {}; - var fstat = options.followLinks ? fs.stat : fs.lstat, - subscription = Futures.subscription(); + function nextWhenReady() { + num -= 1; + if (0 === num) { next(); } + } - if (callback) { subscription.subscribe(callback); } + fnodeTypesPlural.concat(["nodes", "errors"]).forEach(function (fnodeType) { + if (0 === nodes[fnodeType].length) { return; } + num += emitter.listeners(fnodeType).length; + emitter.emit(fnodeType, path, nodes[fnodeType], nextWhenReady); + }); + nextWhenReady(); + } - function readDir(path) { - var p = Futures.promise(); - fs.readdir(path, function (err, files) { - if (err) { - err.path = path; - subscription.deliver(err, path); - // Signal the completion of this readdir attempt - p.fulfill(); + // Determine each file node's type + // + function sortFnodesByType(path, stats, fnodes, nextFile) { + isFnodeTypes.forEach(function (isType, i) { + if (stats[isType]()) { + if (stats.type) { throw new Error("is_" + type + " and " + isType); } + stats.type = fnodeTypes[i]; + fnodes[fnodeTypesPlural[i]].push(stats); + // TODO throw to break; + } + }); + if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); } + emitSingleEvents(path, stats, nextFile); + } + + + // Asynchronously get the stats + // + function getStats(path, files, walkDirs) { + var nodeGroups = {}; + + fnodeTypesPlural.concat("nodes", "errors").forEach(function (fnodeTypePlural) { + nodeGroups[fnodeTypePlural] = []; + }); + + function nextFile() { + var file = files.pop(), dirs = [], fnames = []; + + if (undefined === file) { + emitPluralEvents(path, nodeGroups, function () { + nodeGroups.directories.forEach(function (dir) { + dirs.push(dir.name); + }); + walkDirs(dirs); + }); return; } - // TODO fix futures sequence to not require a first function like this - var s = Futures.sequence(function(n){n();}), - nodes = [], - o = { - errors: [], - dirs: [], - files: [], - links: [], - blocks: [], - chars: [], - fifos: [], - sockets: [] - }; - - files.forEach(function (file) { - // pushes onto the sequence stack without recursion - s.then(function (next) { - fstat(joinPath(path, file), function (err, stats) { - stats = stats || {}; - stats.name = file; - nodes.push(stats); - - if (err) { - stats.err = err; - o.errors.push(stats); - } else { - sortNodesByType(path, stats, o); - } - - next(); - }); - }); - }); - - s.then(function (next) { - var s2 = Futures.sequence(function(n){n();}); - if (nodes.length > 0) { - subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets); - if (o.errors.length > 0) { - emitter.emit("errors", path, o.errors); - } - if (o.dirs.length > 0) { - emitter.emit("directories", path, o.dirs); - } - if (o.files.length > 0) { - emitter.emit("files", path, o.files); - } - if (o.links.length > 0) { - emitter.emit("symbolicLinks", path, o.links); - } - if (o.blocks.length > 0) { - emitter.emit("blockDevices", path, o.blocks); - } - if (o.chars.length > 0) { - emitter.emit("characterDevices", path, o.chars); - } - if (o.fifos.length > 0) { - emitter.emit("fifos", path, o.fifos); - } - if (o.sockets.length > 0) { - emitter.emit("sockets", path, o.fifos); - } - p.fulfill(); - - o.dirs.forEach(function (dir) { - s2.then(function (next2) { - readDir(joinPath(path, dir.name)) - .when(function () { next2(); }); - }); - }); - - next(); + fstat(upath.join(path, file), function (err, stats) { + //console.log("`stat`ed file " + file); + stats = stats || {}; + stats.name = file; + nodeGroups.nodes.push(stats); + if (err) { + stats.error = err; + stats.type = 'error'; + nodeGroups.errors.push(stats); + //emitter.emit('fileError', path, stats, noop); + return nextFile(); } + sortFnodesByType(path, stats, nodeGroups, nextFile); }); - - }); - - return p.passable(); + } + nextFile(); } - readDir(firstPath) //.whenever(callback); + function walk(path, next) { + fs.readdir(path, function (err, nodes) { + if (err) { + emitter.emit('directoryError', path, { error: err, name: path }, noop); + return next(); /*TODO*/ throw err; + } + getStats(path, nodes, function (dirs) { + walkDirs(path, dirs, next); + }); + }); + } - emitter.whenever = subscription.subscribe; + function walkDirs(path, dirs, cb) { + function nextDir() { + var dir = dirs.pop(); + if (undefined === dir) { + delete dirs; + return cb(); + } + walk(upath.join(path, dir), nextDir); + } + nextDir(); + } + + walk(upath.normalize(path), function () { + //throw Error("hey"); + //console.log("Utterly Done!"); + }); return emitter; } - - newVersion.walk = walk; - newVersion.remove = remove; - module.exports = newVersion; + module.exports = create; + module.exports.isFnodeTypes = isFnodeTypes; + module.exports.fnodeTypes = fnodeTypes; + module.exports.fnodeTypesPlural = fnodeTypesPlural; }()); diff --git a/lib/walk2.js b/lib/walk2.js deleted file mode 100644 index 5373b39..0000000 --- a/lib/walk2.js +++ /dev/null @@ -1,138 +0,0 @@ -(function () { - var fs = require('fs'), - upath = require('path'), - util = require('util'), - Futures = require('futures'), - events = require('events'), - emitter = new events.EventEmitter(), - oneNodeEvents = [ - "file", - "directory", - "blockDevice", - "characterDevice", - "symbolicLink", - "fifo", - "socket" - ], - multiNodeEvents = [ - // multiple - "files", - "directories", - "blockDevices", - "characterDevices", - "symbolicLinks", - "fifos", - "sockets" - ], - fstat; - - function sortNodesByType(path, stats, o, next) { - var type, listeners, num, count; - - if (stats.isFile()) { - type = "file"; - o.files.push(stats); - } else if (stats.isDirectory()) { - type = "directory"; - o.dirs.push(stats); - } else if (stats.isBlockDevice()) { - type = "blockDevice"; - o.blocks.push(stats); - } else if (stats.isCharacterDevice()) { - type = "characterDevice"; - o.chars.push(stats); - } else if (stats.isSymbolicLink()) { - type = "symbolicLink"; - o.links.push(stats); - } else if (stats.isFIFO()) { - type = "fifo"; - o.fifos.push(stats); - } else if (stats.isSocket()) { - type = "socket"; - o.sockets.push(stats); - } else { - throw new Error(upath.join(path,stats.name) + 'is not of any tested node type'); - } - - listeners = emitter.listeners(type); - // get the current number of listeners (may change) - num = listeners.length; - if (!num) { - next(); - return; - } - - // join all; once all listeners have responded, continue - count = 0; - emitter.emit(type, path, stats, function () { - count += 1; - if (num === count) { - next(); - } - }); - } - - function handleFiles(path, files) { - var s = Futures.sequence(), - nodes = [], - o = { - errors: [], - dirs: [], - files: [], - links: [], - blocks: [], - chars: [], - fifos: [], - sockets: [] - }; - - files.forEach(function (file) { - s.then(function (next) { - fstat(upath.join(path, file), function (err, stats) { - if (err) { - util.debug("[Error] " + util.inspect(err)); - next(); - return; - } - stats.name = file; - sortNodesByType(path, stats, o, next); - }); - }); - }); - s.then(function (next) { - // TODO copycat the emitters above - next(); - }); - return s; - } - - function handlePath(path) { - - } - - function walk(fullpath, options) { - fstat = (options||{}).followLinks ? fs.stat : fs.lstat; - - var path, file, len, s; - - upath.normalize(fullpath); - - len = fullpath.length - 1; - if (len > 1 && '/' === fullpath[len]) { - fullpath = fullpath.substr(0, len); - } - - path = upath.dirname(fullpath); - file = upath.basename(fullpath); - - s = handleFiles(path, [file]); - s(function (next) { next(); }); - return emitter; - } - - var walker = walk("/System"); - walker.on("directory", function (path, dir, next) { - console.log(path, dir.name); - next(); - }); -}()); diff --git a/lib/walk3.js b/lib/walk3.js deleted file mode 100644 index e98a672..0000000 --- a/lib/walk3.js +++ /dev/null @@ -1,156 +0,0 @@ -(function (undefined) { - var fs = require('fs'), - upath = require('path'), - util = require('util'), - Futures = require('futures'), - events = require('events'); - - function create(path, options) { - // TODO add types by listener dynamically - var emitter = new events.EventEmitter(), - oneNodeEvents = [ - "file", - "directory", - "blockDevice", - "characterDevice", - "symbolicLink", - "FIFO", - "socket" - ], - oneNodeTypes = [ - "isFile", - "isDirectory", - "isBlockDevice", - "isCharacterDevice", - "isSymbolicLink", - "isFIFO", - "isSocket" - ], - multiNodeEvents = [ - // multiple - "files", - "directories", - "blockDevices", - "characterDevices", - "symbolicLinks", - "FIFOs", - "sockets" - ], - fstat = (options||{}).followLinks ? fs.stat : fs.lstat; - - function normalize(path) { - return path; - var index; - - path = upath.normalize(path); - index = path.length - 1; - if (index > 0 && '/' === path[index]) { - path = path.substr(0, index); - } - return path; - } - - function sortNodesByType(path, stats, nodes, next) { - // This could easily be unrolled - // but in this case concise was more readable for me - oneNodeTypes.forEach(function (isType, i) { - if (stats[isType]()) { - if (stats.type) { throw new Error("is_" + type + " and " + isType); } - stats.type = oneNodeEvents[i]; - nodes[multiNodeEvents[i]].push(stats); - // TODO throw to break; - } - }); - if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); } - function emitSingleEvents(stats, next) { - var num; - - // get the current number of listeners (may change) - num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length; - - function nextWhenReady() { - num -= 1; - if (0 === num) { next(); } - } - nextWhenReady(); - - // Total number of listeners - emitter.emit(stats.type, path, stats, function () { - nextWhenReady(); - }); - - emitter.emit("node", path, stats, function () { - nextWhenReady(); - }); - } // emitSingleEvents - emitSingleEvents(stats, next); - } - - function getStats(path, files, walkDirs) { - var nodes = [], o = {}; - - // TODO unroll for better readability - multiNodeEvents.forEach(function (eventType) { - o[eventType] = []; - }); - - function nextFile() { - var file = files.pop(), dirs = [], fnames = []; - - if (undefined === file) { - o.directories.forEach(function (dir) { - dirs.push(dir.name); - }); - return walkDirs(dirs); - } - - fstat(upath.join(path, file), function (err, stats) { - //console.log("`stat`ed file " + file); - stats = stats || {}; - stats.name = file; - nodes.push(stats); - - if (err) { - stats.error = err - // TODO single error emittor - o.errors.push(stats); - util.debug("[Error 1] " + util.inspect(err)); - return nextFile(); - } - sortNodesByType(path, stats, o, nextFile); - }); - } - nextFile(); - } - - function walk(path, next) { - fs.readdir(path, function (err, nodes) { - if (err) { return next(); /*TODO*/ throw err; } - getStats(path, nodes, function (dirs) { - walkDirs(path, dirs, function () { - next(); - }); - }); - }); - } - - function walkDirs(path, dirs, cb) { - function nextDir() { - var dir = dirs.pop(); - if (undefined === dir) { - delete dirs; - return cb(); - } - walk(upath.join(path, dir), nextDir); - } - nextDir(); - } - - walk(normalize(path), function () { - //throw Error("hey"); - //console.log("Utterly Done!"); - }); - return emitter; - } - module.exports = create; -}()); diff --git a/package.json b/package.json index e5bfbf3..6ef246c 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "name" : "walk", "description" : "A node port of python's os.walk", - "url" : "github.com/coolaj86/walk", - "keywords" : ["util", "os", "fs", "walk"], + "url" : "github.com/coolaj86/node-walk", + "keywords" : ["util", "os", "sys", "fs", "walk", "walkSync"], "author" : "AJ ONeal ", "contributors" : [], - "dependencies" : ["futures"], + "dependencies" : [], "lib" : "lib", "main" : "./lib/walk.js", - "version" : "0.9.2" + "version" : "1.0.0" }