walk is now awesomesauce... but still a memory hog

README.md

@@ -1,7 +1,18 @@
 node-walk
 ====
 
-A not-quite-port of python's `os.walk`.
+A port python's `os.walk`, but using Node.JS conventions.
+
+  * EventEmitter
+  * Asynchronous
+  * Chronological (optionally)
+  * Built-in flow-control
+
+As few file descriptors are opened at a time as possible.
+This is particularly well suited for single hard disks which are not flash or solid state.
+
+Memory usage is high (120mb for 60,000 dirs), but reduction is being investigated.
+Patches welcome.
 
 Installation
 ----
@@ -11,91 +22,107 @@ Installation
 Usage
 ====
 
-All of the examples in the folder included in this repository work and have no typos.
-
-without emitter
-----
-
     var walk = require('walk').walk,
-      options = {
+      options,
+      walker;
+
+    options = {
         followLinks: false,
-      };
+    };
 
-    function fileHandler(err, path, errors, dirs, files, links, blocks, chars, fifos, sockets) {
+    walker = walk("path/to/dir", options);
-      // handle each path  
-    }
 
-    walk("path/to/dir", options, fileHandler);
+    walker.on("directories", function (root, dirStatsArray, next) {
-    // This also works
+      // dirStatsArray is an array of `stat` objects with the additional attributes
-    // walk("path/to/dir", options).whenever(fileHandler);
+      // * type
+      // * error
+      // * name
+      
+      next();
+    });
 
-Single Arguments
+    walker.on("file", function (root, fileStats, next) {
+      fs.readFile(file, function () {
+        // doStuff
+        next();
+      });
+    });
 
-  * `err` - Error when reading path (Probably due to permissions).
+    walker.on("errors", function (root, nodeStatsArray, next) {
-  * `path` - the current path being read
+      next();
+    });
+
+API
+====
+
+Emitted Values
+
+  * `root` - the containing the files to be inspected
+  * *stats[Array]* - a single `stats` object or an array with some added attributes
+    * type - 'file', 'directory', etc
+    * error
+    * name - the name of the file, dir, etc 
+  * next - no more files will be read until this is called
+
+Single Events - fired immediately
+
+  * `directoryError` - Error when `fstat` succeeded, but reading path failed (Probably due to permissions).
+  * `node` - a `stats` object for a node of any type
+  * `file` - includes links when `followLinks` is `true`
+  * `directory`
+  * `blockDevice`
+  * `characterDevice`
+  * `symbolicLink` - always empty when `followLinks` is `true`
+  * `FIFO`
+  * `socket`
 
 Array Arguments
 
-  * `errors` - `fs.stat` encountered on files in the directory
+  * `errors` - errors encountered by `fs.stat` when reading ndes in a directory
-  * `dirs` - directories (modification of this array - sorting, removing, etc - affects traversal)
+  * `nodes` - an array of `stats` of any type
-  * `files` - regular files (includes links when `followLinks` is `true`)
+  * `files`
-  * `links` - symbolic links (always empty when `followLinks` is `true`)
+  * `directories` - modification of this array - sorting, removing, etc - affects traversal
-  * `blocks` - block devices
+  * `blockDevices`
-  * `chars` - character devices
+  * `characterDevices`
-  * `fifos` - FIFOs
+  * `symbolicLinks`
-  * `sockets` - sockets
+  * `FIFOs`
+  * `sockets`
 
-using emitter
+**Warning** when following links, an infinite loop is possible
-----
 
-`errors`, `directories`, `files`, `symbolicLinks`
+Comparisons
-
-    var walk = require('walk').walk,
-      emitter = walk('./walk-test');
-
-    emitter.on("directories", function (path, dirs) {
-      // the second directory will not be traversed
-      dirs.splice(1,1);
-      dirs.forEach(function (dir) {
-        console.log(dir);
-      });
-    });
-
-    emitter.on("files", function (path, files) {
-      files.forEach(function (dir) {
-        console.log(dir);
-      });
-    });
-
-Example
 ====
 
-    mkdir -p walk-test/dir1
+Tested on my `/System` containing 59,490 (+ self) directories (and lots of files).
-    touch walk-test/file-1
+The size of the text output was 6mb.
-    touch walk-test/file-2
-    touch walk-test/dir1/file-1
-    touch walk-test/dir1/file-2
 
-node-walk-test
+`find`:
-----
+    time bash -c "find /System -type d | wc"
+    59491   97935 6262916
 
-    var walk = require('walk');
+    real  2m27.114s
+    user  0m1.193s
+    sys 0m14.859s
 
-    walk('./walk-test', undefined,  function (err, path, errors, dirs, files, links) {
+`find.js`:
-      if (err) {
-        console.log('ERROR: ');
-        console.log(err);
-        return;
-      }
 
-      dirs.forEach(function (item, i, arr) {
+Note that `find.js` omits the start directory
-        if (item.name.match(/trash/i)) {
+
-          console.log('found a trash');
+    time bash -c "node examples/find.js /System -type d | wc"
-          arr.splice(i,1);
+    59490   97934 6262908
-        }
+   
-      });
+    # Test 1 
+    real  2m52.273s
+    user  0m20.374s
+    sys 0m27.800s
+    
+    # Test 2
+    real  2m23.725s
+    user  0m18.019s
+    sys 0m23.202s
+
+    # Test 3
+    real  2m50.077s
+    user  0m17.661s
+    sys 0m24.008s
 
-      console.log("PATH: " + path);
-      console.log("SORTED: ");
-      console.log(errors, dirs, files, links);
-    });

examples/find-test.js

@@ -0,0 +1,35 @@
+(function () {
+  var walk = require("../lib/walk3.js"),
+    emit = walk(process.argv[2] || "/tmp"),
+    util = require('util'),
+    path = require('path');
+
+  // nor the root, nor the node should ever be empty
+  walk.fnodeTypesPlural.forEach(function (fnodeType) {
+    emit.on(fnodeType, function (root, nodes, next) {
+      if (!nodes || !nodes.length || !root) {
+        console.log(fnodeType, "empty set", root, nodes.length); //JSON.stringify(nodes));
+      }
+      next();
+    });
+  });
+  walk.fnodeTypes.forEach(function (fnodeType) {
+    emit.on(fnodeType, function (root, node, next) {
+      if (!node || !node.name || !root) {
+        console.log(fnodeType, "empty item", root, node.name); //JSON.stringify(node));
+      }
+      next();
+    });
+  });
+  emit.on('directory', function (root, dir, next) {
+    console.log(path.join(root, dir.name));
+    next();
+  });
+  /*
+  emit.on('file', function (root, file, next) {
+    console.log(path.join(root, file.name));
+    next();
+  });
+  */
+}());
+

examples/find.js

@@ -3,13 +3,20 @@
     emit = walk(process.argv[2] || "/tmp");
     //icount = 0;
 
+  emit.on('directories', function (path, dirs, next) {
+    dirs.forEach(function (dir) {
+      console.log(path + '/' + dir.name);
+    });
+    next();
+  });
+  /*
   emit.on('directory', function (path, file, next) {
     //icount += 1;
     console.log(path + '/' + file.name); // + " " + icount);
-    process.nextTick(next);
+    next();
+    //process.nextTick(next);
     //setTimeout(next, 100);
   });
-  /*
   emit.on('file', function (path, file, next) {
     console.log("FILE:", file.name, "\n");
     next();

lib/walk-0.9.js

@@ -0,0 +1,230 @@
+(function () {
+  var fs = require('fs'),
+    Futures = require('futures'),
+    joinPath = require('path').join,
+    util = require('util'),
+    ev = require("events"),
+    emitter = new ev.EventEmitter(),
+    oneNodeEvent = [
+      "file",
+      "directory",
+      "blockDevice",
+      "characterDevice",
+      "symbolicLink",
+      "fifo",
+      "socket"
+    ],
+    multiNodeEvents = [
+      // multiple
+      "files",
+      "directories",
+      "blockDevices",
+      "characterDevices",
+      "symbolicLinks",
+      "fifos",
+      "sockets"
+    ],
+    eventsTpl = {
+      listeners: function () { return []; },
+      next: function () { return; }
+    },
+    events = {},
+    nexts = {};
+
+  function newVersion() {
+    throw new Error("see README.md at  http://github.com/coolaj86/node-walk");
+  }
+
+  function noop() {
+  }
+
+  function remove(arr, obj) {
+    return arr.splice(arr.indexOf(obj), 1);
+  }
+
+  oneNodeEvent.forEach(function (key) {
+    var e = events[key] = {}, next;
+
+    Object.keys(eventsTpl).forEach(function (k) {
+      e[k] = eventsTpl[k]();
+    });
+
+    emitter.on("newListener", function (ev, listener) {
+      var count = 0,
+        num = e.listeners.length + 1;
+
+      e.listeners.push(listener);
+      e.next = function (cb) {
+        cb = noop;
+        return function () {
+          if (count === num) { cb(); }
+          count += 1;
+        };
+      };
+    });
+
+    // TODO
+    next = function () {
+      
+    };
+  });
+
+  function sortNodesByType(path, stats, o, cb) {
+    if (stats.isFile()) {
+      o.files.push(stats);
+      emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
+    } else if (stats.isDirectory()) {
+      o.dirs.push(stats);
+      emitter.emit("directory", path, stats, function () {
+        remove(o.dirs, stats);
+      }, (nexts["directory"]||noop)(cb));
+    } else if (stats.isBlockDevice()) {
+      o.blocks.push(stats);
+      emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
+    } else if (stats.isCharacterDevice()) {
+      o.chars.push(stats);
+      emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
+    } else if (stats.isSymbolicLink()) {
+      o.links.push(stats);
+      emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
+    } else if (stats.isFIFO()) {
+      o.fifos.push(stats);
+      emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
+    } else if (stats.isSocket()) {
+      o.sockets.push(stats);
+      emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
+    } else {
+      // emitter.emit("error", stats);
+      util.debug(stats.name + 'is not of any node type');
+    }
+  }
+
+  /*
+  import os
+  from os.path import join, getsize
+  for root, dirs, files in os.walk('python/Lib/email'):
+      print root, "consumes",
+      print sum(getsize(join(root, name)) for name in files),
+      print "bytes in", len(files), "non-directory files"
+      if 'CVS' in dirs:
+          dirs.remove('CVS')  # don't visit CVS directories
+  */
+
+  /*
+    fs.walk(path, function ({ err, root, dirs, files }) {}, {
+      // currently ignored
+      topdown: boolean,
+      onerror: boolean, // ignored
+      followLinks: boolean // lstat or stat
+    });
+  */
+
+  function walk(firstPath, options, callback) {
+    options = options || {};
+    var fstat = options.followLinks ? fs.stat : fs.lstat,
+      subscription = Futures.subscription();
+
+    if (callback) { subscription.subscribe(callback); }
+
+    function readDir(path) {
+      var p = Futures.promise();
+
+      fs.readdir(path, function (err, files) {
+        if (err) {
+          err.path = path;
+          subscription.deliver(err, path);
+          // Signal the completion of this readdir attempt
+          p.fulfill();
+          return;
+        }
+
+        // TODO fix futures sequence to not require a first function like this
+        var s = Futures.sequence(function(n){n();}), 
+          nodes = [], 
+          o = {
+            errors: [], 
+            dirs: [], 
+            files: [],
+            links: [], 
+            blocks: [], 
+            chars: [], 
+            fifos: [], 
+            sockets: []
+          };
+
+        files.forEach(function (file) {
+          // pushes onto the sequence stack without recursion
+          s.then(function (next) {
+            fstat(joinPath(path, file), function (err, stats) {
+              stats = stats || {};
+              stats.name = file;
+              nodes.push(stats);
+
+              if (err) {
+                stats.err = err;
+                o.errors.push(stats);
+              } else {
+                sortNodesByType(path, stats, o);
+              }
+
+              next();
+            });
+          });
+        });
+
+        s.then(function (next) {
+          var s2 = Futures.sequence(function(n){n();});
+          if (nodes.length > 0) {
+            subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
+            if (o.errors.length > 0) {
+              emitter.emit("errors", path, o.errors);
+            }
+            if (o.dirs.length > 0) {
+              emitter.emit("directories", path, o.dirs);
+            }
+            if (o.files.length > 0) {
+              emitter.emit("files", path, o.files);
+            }
+            if (o.links.length > 0) {
+              emitter.emit("symbolicLinks", path, o.links);
+            }
+            if (o.blocks.length > 0) {
+              emitter.emit("blockDevices", path, o.blocks);
+            }
+            if (o.chars.length > 0) {
+              emitter.emit("characterDevices", path, o.chars);
+            }
+            if (o.fifos.length > 0) {
+              emitter.emit("fifos", path, o.fifos);
+            }
+            if (o.sockets.length > 0) {
+              emitter.emit("sockets", path, o.fifos);
+            }
+            p.fulfill();
+
+            o.dirs.forEach(function (dir) {
+              s2.then(function (next2) {
+                readDir(joinPath(path, dir.name))
+                  .when(function () { next2(); });
+              });
+            });
+
+            next();
+          }
+        });
+
+      });
+
+      return p.passable();
+    }
+
+    readDir(firstPath) //.whenever(callback);
+
+    emitter.whenever = subscription.subscribe;
+    return emitter;
+  }
+
+  newVersion.walk = walk;
+  newVersion.remove = remove;
+  module.exports = newVersion;
+}());

lib/walk-alternate-implementation.js

@@ -0,0 +1,60 @@
+// 2011-11-25 jorge@jorgechamorro.com
+
+function walk (file, cb) {
+  var fs = require('fs');
+  var q= [];
+  var queue= [q];
+  walk2();
+  
+  function walk2 () { 
+    cb(file);
+    fs.lstat(file, function (err, stat) {
+      if (err || !stat.isDirectory()) return next();
+      getDirectory(function (files) {
+        queue.push(q= files);
+        next();
+      });
+    });
+  }
+  
+  function next () {
+    if (q.length) {
+      file= q.pop();
+      walk2();
+    }
+    else if (queue.length-= 1) {
+      q= queue[queue.length-1];
+      next();
+    }
+  }
+  
+  function getDirectory (cb) {
+    fs.readdir(file, function(err, files) {
+      if (!files) return;
+      //if (err) throw Error(err);
+      files.sort(sort);
+      files.forEach(fullPath);
+      cb(files);
+    });
+  }
+  
+  function fullPath (v,i,o) {
+    o[i]= [file, '/', v].join('');
+  }
+  
+  function sort (a,b) {
+    a= a.toLowerCase();
+    b= b.toLowerCase();
+    if (a > b) return -1;
+    if (a < b) return  1;
+    else       return  0;
+  }
+}
+
+// your callback here
+var ctr= 0;
+function callBack (file) { console.log( ["[", ++ctr, "] ", file].join('') ) };
+
+process.argv.forEach(function(val, index, array) {
+  if (index > 1) walk(val, callBack);
+});

lib/walk.js

@@ -1,230 +1,157 @@
-(function () {
+// TODO
+//  * add types by listener dynamically
+//  * unroll loops for better readability?
+//  * should emitted errors wait for `next()`?
+(function (undefined) {
   var fs = require('fs'),
-    Futures = require('futures'),
+    upath = require('path'),
-    joinPath = require('path').join,
     util = require('util'),
-    ev = require("events"),
+    Futures = require('futures'),
-    emitter = new ev.EventEmitter(),
+    events = require('events'),
-    oneNodeEvent = [
+    noop = function () {},
-      "file",
+    // "FIFO" isn't easy to convert to camelCame and back reliably
-      "directory",
+    isFnodeTypes = [
-      "blockDevice",
+      "isFile", "isDirectory",  "isBlockDevice",  "isCharacterDevice",  "isSymbolicLink", "isFIFO", "isSocket"
-      "characterDevice",
-      "symbolicLink",
-      "fifo",
-      "socket"
     ],
-    multiNodeEvents = [
+    fnodeTypes = [
-      // multiple
+      "file",   "directory",    "blockDevice",    "characterDevice",    "symbolicLink",   "FIFO",   "socket"
-      "files",
-      "directories",
-      "blockDevices",
-      "characterDevices",
-      "symbolicLinks",
-      "fifos",
-      "sockets"
     ],
-    eventsTpl = {
+    fnodeTypesPlural = [
-      listeners: function () { return []; },
+      "files",  "directories",  "blockDevices",   "characterDevices",   "symbolicLinks",  "FIFOs",  "sockets"
-      next: function () { return; }
+    ];
-    },
-    events = {},
-    nexts = {};
 
-  function newVersion() {
+  // Create a new walk instance
-    throw new Error("see README.md at  http://github.com/coolaj86/node-walk");
+  function create(path, options) {
-  }
+    var emitter = new events.EventEmitter(),
+      fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
 
-  function noop() {
-  }
 
-  function remove(arr, obj) {
+    // Get the current number of listeners (which may change)
-    return arr.splice(arr.indexOf(obj), 1);
+    // Emit events to each listener
-  }
+    // Wait for all listeners to `next()` before continueing
+    // (in theory this may avoid disk thrashing)
+    function emitSingleEvents(path, stats, next) {
+      var num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
 
-  oneNodeEvent.forEach(function (key) {
+      function nextWhenReady() {
-    var e = events[key] = {}, next;
+        num -= 1;
+        if (0 === num) { next(); }
+      }
 
-    Object.keys(eventsTpl).forEach(function (k) {
+      emitter.emit(stats.type, path, stats, nextWhenReady);
-      e[k] = eventsTpl[k]();
+      emitter.emit("node", path, stats, nextWhenReady);
-    });
+      nextWhenReady();
-
-    emitter.on("newListener", function (ev, listener) {
-      var count = 0,
-        num = e.listeners.length + 1;
-
-      e.listeners.push(listener);
-      e.next = function (cb) {
-        cb = noop;
-        return function () {
-          if (count === num) { cb(); }
-          count += 1;
-        };
-      };
-    });
-
-    // TODO
-    next = function () {
-      
-    };
-  });
-
-  function sortNodesByType(path, stats, o, cb) {
-    if (stats.isFile()) {
-      o.files.push(stats);
-      emitter.emit("file", path, stats, (nexts["file"]||noop)(cb));
-    } else if (stats.isDirectory()) {
-      o.dirs.push(stats);
-      emitter.emit("directory", path, stats, function () {
-        remove(o.dirs, stats);
-      }, (nexts["directory"]||noop)(cb));
-    } else if (stats.isBlockDevice()) {
-      o.blocks.push(stats);
-      emitter.emit("blockDevice", path, stats, (nexts["blockDevice"]||noop)(cb));
-    } else if (stats.isCharacterDevice()) {
-      o.chars.push(stats);
-      emitter.emit("characterDevice", path, stats, (nexts["characterDevice"]||noop)(cb));
-    } else if (stats.isSymbolicLink()) {
-      o.links.push(stats);
-      emitter.emit("symbolicLink", path, stats, (nexts["symbolicLink"]||noop)(cb));
-    } else if (stats.isFIFO()) {
-      o.fifos.push(stats);
-      emitter.emit("fifo", path, stats, (nexts["fifo"]||noop)(cb));
-    } else if (stats.isSocket()) {
-      o.sockets.push(stats);
-      emitter.emit("socket", path, stats, (nexts["socket"]||noop)(cb));
-    } else {
-      // emitter.emit("error", stats);
-      util.debug(stats.name + 'is not of any node type');
     }
-  }
 
-  /*
-  import os
-  from os.path import join, getsize
-  for root, dirs, files in os.walk('python/Lib/email'):
-      print root, "consumes",
-      print sum(getsize(join(root, name)) for name in files),
-      print "bytes in", len(files), "non-directory files"
-      if 'CVS' in dirs:
-          dirs.remove('CVS')  # don't visit CVS directories
-  */
 
-  /*
+    // Since the risk for disk thrashing among anything
-    fs.walk(path, function ({ err, root, dirs, files }) {}, {
+    // other than files is relatively low, all types are
-      // currently ignored
+    // emitted at once, but all must complete before advancing
-      topdown: boolean,
+    function emitPluralEvents(path, nodes, next) {
-      onerror: boolean, // ignored
+      var num = 1;
-      followLinks: boolean // lstat or stat
-    });
-  */
 
-  function walk(firstPath, options, callback) {
+      function nextWhenReady() {
-    options = options || {};
+        num -= 1;
-    var fstat = options.followLinks ? fs.stat : fs.lstat,
+        if (0 === num) { next(); }
-      subscription = Futures.subscription();
+      }
 
-    if (callback) { subscription.subscribe(callback); }
+      fnodeTypesPlural.concat(["nodes", "errors"]).forEach(function (fnodeType) {
+        if (0 === nodes[fnodeType].length) { return; }
+        num += emitter.listeners(fnodeType).length;
+        emitter.emit(fnodeType, path, nodes[fnodeType], nextWhenReady);
+      });
+      nextWhenReady();
+    }
 
-    function readDir(path) {
-      var p = Futures.promise();
 
-      fs.readdir(path, function (err, files) {
+    // Determine each file node's type
-        if (err) {
+    // 
-          err.path = path;
+    function sortFnodesByType(path, stats, fnodes, nextFile) {
-          subscription.deliver(err, path);
+      isFnodeTypes.forEach(function (isType, i) {
-          // Signal the completion of this readdir attempt
+        if (stats[isType]()) {
-          p.fulfill();
+          if (stats.type) { throw new Error("is_" + type + " and " + isType); }
+          stats.type = fnodeTypes[i];
+          fnodes[fnodeTypesPlural[i]].push(stats);
+          // TODO throw to break;
+        }
+      });
+      if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
+      emitSingleEvents(path, stats, nextFile);
+    }
+
+
+    // Asynchronously get the stats 
+    //
+    function getStats(path, files, walkDirs) {
+      var nodeGroups = {};
+
+      fnodeTypesPlural.concat("nodes", "errors").forEach(function (fnodeTypePlural) {
+        nodeGroups[fnodeTypePlural] = [];
+      });
+
+      function nextFile() {
+        var file = files.pop(), dirs = [], fnames = [];
+
+        if (undefined === file) {
+          emitPluralEvents(path, nodeGroups, function () {
+            nodeGroups.directories.forEach(function (dir) {
+              dirs.push(dir.name);
+            });
+            walkDirs(dirs);
+          });
           return;
         }
 
-        // TODO fix futures sequence to not require a first function like this
+        fstat(upath.join(path, file), function (err, stats) {
-        var s = Futures.sequence(function(n){n();}), 
+          //console.log("`stat`ed file " + file);
-          nodes = [], 
+          stats = stats || {};
-          o = {
+          stats.name = file;
-            errors: [], 
+          nodeGroups.nodes.push(stats);
-            dirs: [], 
+          if (err) {
-            files: [],
+            stats.error = err;
-            links: [], 
+            stats.type = 'error';
-            blocks: [], 
+            nodeGroups.errors.push(stats);
-            chars: [], 
+            //emitter.emit('fileError', path, stats, noop);
-            fifos: [], 
+            return nextFile();
-            sockets: []
-          };
-
-        files.forEach(function (file) {
-          // pushes onto the sequence stack without recursion
-          s.then(function (next) {
-            fstat(joinPath(path, file), function (err, stats) {
-              stats = stats || {};
-              stats.name = file;
-              nodes.push(stats);
-
-              if (err) {
-                stats.err = err;
-                o.errors.push(stats);
-              } else {
-                sortNodesByType(path, stats, o);
-              }
-
-              next();
-            });
-          });
-        });
-
-        s.then(function (next) {
-          var s2 = Futures.sequence(function(n){n();});
-          if (nodes.length > 0) {
-            subscription.deliver(undefined, path, o.errors, o.dirs, o.files, o.links, o.blocks, o.chars, o.fifos, o.sockets);
-            if (o.errors.length > 0) {
-              emitter.emit("errors", path, o.errors);
-            }
-            if (o.dirs.length > 0) {
-              emitter.emit("directories", path, o.dirs);
-            }
-            if (o.files.length > 0) {
-              emitter.emit("files", path, o.files);
-            }
-            if (o.links.length > 0) {
-              emitter.emit("symbolicLinks", path, o.links);
-            }
-            if (o.blocks.length > 0) {
-              emitter.emit("blockDevices", path, o.blocks);
-            }
-            if (o.chars.length > 0) {
-              emitter.emit("characterDevices", path, o.chars);
-            }
-            if (o.fifos.length > 0) {
-              emitter.emit("fifos", path, o.fifos);
-            }
-            if (o.sockets.length > 0) {
-              emitter.emit("sockets", path, o.fifos);
-            }
-            p.fulfill();
-
-            o.dirs.forEach(function (dir) {
-              s2.then(function (next2) {
-                readDir(joinPath(path, dir.name))
-                  .when(function () { next2(); });
-              });
-            });
-
-            next();
           }
+          sortFnodesByType(path, stats, nodeGroups, nextFile);
         });
-
+      }
-      });
+      nextFile();
-
-      return p.passable();
     }
 
-    readDir(firstPath) //.whenever(callback);
+    function walk(path, next) {
+      fs.readdir(path, function (err, nodes) {
+        if (err) { 
+          emitter.emit('directoryError', path, { error: err, name: path }, noop);
+          return next(); /*TODO*/ throw err;
+        }
+        getStats(path, nodes, function (dirs) {
+          walkDirs(path, dirs, next);
+        });
+      });
+    }
 
-    emitter.whenever = subscription.subscribe;
+    function walkDirs(path, dirs, cb) {
+      function nextDir() {
+        var dir = dirs.pop();
+        if (undefined === dir) {
+          delete dirs;
+          return cb();
+        }
+        walk(upath.join(path, dir), nextDir);
+      }
+      nextDir();
+    }
+
+    walk(upath.normalize(path), function () {
+      //throw Error("hey");
+      //console.log("Utterly Done!");
+    });
     return emitter;
   }
-
+  module.exports = create;
-  newVersion.walk = walk;
+  module.exports.isFnodeTypes = isFnodeTypes;
-  newVersion.remove = remove;
+  module.exports.fnodeTypes = fnodeTypes;
-  module.exports = newVersion;
+  module.exports.fnodeTypesPlural = fnodeTypesPlural;
 }());

lib/walk2.js

@@ -1,138 +0,0 @@
-(function () {
-  var fs = require('fs'),
-    upath = require('path'),
-    util = require('util'),
-    Futures = require('futures'),
-    events = require('events'),
-    emitter = new events.EventEmitter(),
-    oneNodeEvents = [
-      "file",
-      "directory",
-      "blockDevice",
-      "characterDevice",
-      "symbolicLink",
-      "fifo",
-      "socket"
-    ],
-    multiNodeEvents = [
-      // multiple
-      "files",
-      "directories",
-      "blockDevices",
-      "characterDevices",
-      "symbolicLinks",
-      "fifos",
-      "sockets"
-    ],
-    fstat;
-
-  function sortNodesByType(path, stats, o, next) {
-    var type, listeners, num, count;
-
-    if (stats.isFile()) {
-      type = "file";
-      o.files.push(stats);
-    } else if (stats.isDirectory()) {
-      type = "directory";
-      o.dirs.push(stats);
-    } else if (stats.isBlockDevice()) {
-      type = "blockDevice";
-      o.blocks.push(stats);
-    } else if (stats.isCharacterDevice()) {
-      type = "characterDevice";
-      o.chars.push(stats);
-    } else if (stats.isSymbolicLink()) {
-      type = "symbolicLink";
-      o.links.push(stats);
-    } else if (stats.isFIFO()) {
-      type = "fifo";
-      o.fifos.push(stats);
-    } else if (stats.isSocket()) {
-      type = "socket";
-      o.sockets.push(stats);
-    } else {
-      throw new Error(upath.join(path,stats.name) + 'is not of any tested node type');
-    }
-   
-    listeners = emitter.listeners(type);
-    // get the current number of listeners (may change)
-    num = listeners.length;
-    if (!num) {
-      next();
-      return;
-    }
-
-    // join all; once all listeners have responded, continue
-    count = 0;
-    emitter.emit(type, path, stats, function () {
-      count += 1;
-      if (num === count) {
-        next();
-      }
-    });
-  }
-
-  function handleFiles(path, files) {
-    var s = Futures.sequence(),
-      nodes = [],
-      o = {
-        errors: [],
-        dirs: [],
-        files: [],
-        links: [],
-        blocks: [],
-        chars: [],
-        fifos: [],
-        sockets: []
-      };
-
-    files.forEach(function (file) {
-      s.then(function (next) {
-        fstat(upath.join(path, file), function (err, stats) {
-          if (err) {
-            util.debug("[Error] " + util.inspect(err));
-            next();
-            return;
-          }
-          stats.name = file;
-          sortNodesByType(path, stats, o, next);
-        });
-      });
-    });
-    s.then(function (next) {
-      // TODO copycat the emitters above
-      next();
-    });
-    return s;
-  }
-
-  function handlePath(path) {
-    
-  }
-
-  function walk(fullpath, options) {
-    fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
-
-    var path, file, len, s;
-
-    upath.normalize(fullpath);
-
-    len = fullpath.length - 1;
-    if (len > 1 && '/' === fullpath[len]) {
-      fullpath = fullpath.substr(0, len);
-    }
-
-    path = upath.dirname(fullpath);
-    file = upath.basename(fullpath);
-
-    s = handleFiles(path, [file]);
-    s(function (next) { next(); });
-    return emitter;
-  }
-
-  var walker = walk("/System");
-  walker.on("directory", function (path, dir, next) {
-    console.log(path, dir.name);
-    next();
-  });
-}());

lib/walk3.js

@@ -1,156 +0,0 @@
-(function (undefined) {
-  var fs = require('fs'),
-    upath = require('path'),
-    util = require('util'),
-    Futures = require('futures'),
-    events = require('events');
-
-  function create(path, options) {
-    // TODO add types by listener dynamically
-    var emitter = new events.EventEmitter(),
-      oneNodeEvents = [
-        "file",
-        "directory",
-        "blockDevice",
-        "characterDevice",
-        "symbolicLink",
-        "FIFO",
-        "socket"
-      ],
-      oneNodeTypes = [
-        "isFile",
-        "isDirectory",
-        "isBlockDevice",
-        "isCharacterDevice",
-        "isSymbolicLink",
-        "isFIFO",
-        "isSocket"
-      ],
-      multiNodeEvents = [
-        // multiple
-        "files",
-        "directories",
-        "blockDevices",
-        "characterDevices",
-        "symbolicLinks",
-        "FIFOs",
-        "sockets"
-      ],
-      fstat = (options||{}).followLinks ? fs.stat : fs.lstat;
-
-    function normalize(path) {
-      return path;
-      var index;
-
-      path = upath.normalize(path);
-      index = path.length - 1;
-      if (index > 0 && '/' === path[index]) {
-        path = path.substr(0, index);
-      }
-      return path;
-    }
-
-    function sortNodesByType(path, stats, nodes, next) {
-      // This could easily be unrolled
-      // but in this case concise was more readable for me
-      oneNodeTypes.forEach(function (isType, i) {
-        if (stats[isType]()) {
-          if (stats.type) { throw new Error("is_" + type + " and " + isType); }
-          stats.type = oneNodeEvents[i];
-          nodes[multiNodeEvents[i]].push(stats);
-          // TODO throw to break;
-        }
-      });
-      if (!stats.type) { throw new Error(upath.join(path, stats.name) + ' isAnUndefinedType'); }
-    function emitSingleEvents(stats, next) {
-      var num;
-     
-      // get the current number of listeners (may change)
-      num = 1 + emitter.listeners(stats.type).length + emitter.listeners("node").length;
-
-      function nextWhenReady() {
-        num -= 1;
-        if (0 === num) { next(); }
-      }
-      nextWhenReady();
-
-      // Total number of listeners
-      emitter.emit(stats.type, path, stats, function () {
-        nextWhenReady();
-      });
-
-      emitter.emit("node", path, stats, function () {
-        nextWhenReady();
-      });
-    } // emitSingleEvents
-      emitSingleEvents(stats, next);
-    }
-
-    function getStats(path, files, walkDirs) {
-      var nodes = [], o = {};
-
-      // TODO unroll for better readability
-      multiNodeEvents.forEach(function (eventType) {
-        o[eventType] = [];
-      });
-
-      function nextFile() {
-        var file = files.pop(), dirs = [], fnames = [];
-
-        if (undefined === file) {
-          o.directories.forEach(function (dir) {
-            dirs.push(dir.name);
-          });
-          return walkDirs(dirs);
-        }
-
-        fstat(upath.join(path, file), function (err, stats) {
-          //console.log("`stat`ed file " + file);
-          stats = stats || {};
-          stats.name = file;
-          nodes.push(stats);
-
-          if (err) {
-            stats.error = err
-            // TODO single error emittor
-            o.errors.push(stats);
-            util.debug("[Error 1] " + util.inspect(err));
-            return nextFile();
-          }
-          sortNodesByType(path, stats, o, nextFile);
-        });
-      }
-      nextFile();
-    }
-
-    function walk(path, next) {
-      fs.readdir(path, function (err, nodes) {
-        if (err) { return next(); /*TODO*/ throw err; }
-        getStats(path, nodes, function (dirs) {
-          walkDirs(path, dirs, function () {
-            next();
-          });
-        });
-      });
-    }
-
-    function walkDirs(path, dirs, cb) {
-      function nextDir() {
-        var dir = dirs.pop();
-        if (undefined === dir) {
-          delete dirs;
-          return cb();
-        }
-        walk(upath.join(path, dir), nextDir);
-      }
-      nextDir();
-    }
-
-    walk(normalize(path), function () {
-      //throw Error("hey");
-      //console.log("Utterly Done!");
-    });
-    return emitter;
-  }
-  module.exports = create;
-}());

package.json

@@ -1,12 +1,12 @@
 {
   "name"          : "walk",
   "description"   : "A node port of python's os.walk",
-  "url"           : "github.com/coolaj86/walk",
+  "url"           : "github.com/coolaj86/node-walk",
-  "keywords"      : ["util", "os", "fs", "walk"],
+  "keywords"      : ["util", "os", "sys", "fs", "walk", "walkSync"],
   "author"        : "AJ ONeal <coolaj86@gmail.com>",
   "contributors"  : [],
-  "dependencies"  : ["futures"],
+  "dependencies"  : [],
   "lib"           : "lib",
   "main"          : "./lib/walk.js",
-  "version"       : "0.9.2"
+  "version"       : "1.0.0"
 }