.gitignore

@@ -1,2 +1,3 @@
 node_modules
 .*.sw*
+local-db.js

bin/digd.js

@@ -387,7 +387,7 @@ cli.main(function (args, cli) {
     }
 
     try {
-      engine = engine || require('../lib/store.json.js').create(engineOpts);
+      engine = engine || require('../lib/store').create(engineOpts);
     } catch(e) {
       respondWithResults(e);
       return;
@@ -413,7 +413,7 @@ cli.main(function (args, cli) {
   }
   if (cli.http) {
     try {
-      engine = engine || require('../lib/store.json.js').create(engineOpts);
+      engine = engine || require('../lib/store').create(engineOpts);
     } catch(e) {
       console.error(e);
       return;

lib/digd.js

@@ -184,7 +184,7 @@ function getNs(engine, zs, results, cb) {
     // d.vanityNs should only be vanity nameservers (pointing to this same server)
     if (z.vanityNs || results.authority.some(function (ns) {
       console.log('[debug] ns', ns);
-      return -1 !== engine.primaryNameservers.indexOf(ns.data.toLowerCase());
+      return -1 !== engine.primaryNameservers().indexOf(ns.data.toLowerCase());
     })) {
       results.authority.length = 0;
       results.authority.push(engine.zones._toSoa(z));
@@ -359,7 +359,7 @@ module.exports.query = function (engine, query, cb) {
       // NOTE: I think that the issue here is EXTERNAL vs INTERNAL vanity NS
       // We _should_ reply for EXTERNAL vanity NS... but not when it's listed on the SOA internally?
       // It's surrounding the problem of what if I do sub domain delegation to the same server.
-      if (-1 === engine.primaryNameservers.indexOf(r.data.toLowerCase())) {
+      if (-1 === engine.primaryNameservers().indexOf(r.data.toLowerCase())) {
         console.log("It's a vanity NS");
         return false;
       }

lib/httpd.js

@@ -230,6 +230,11 @@ module.exports.create = function (cli, engine/*, dnsd*/) {
         zone.class = zone.className;
         zone.type = zone.typeName;
         zone.soa = true;
+
+        // TODO: consider sending a predicate object through the engine
+        // to the actual store in case it is highly inefficient to transfer
+        // a large number of records from the store that will just be
+        // thrown away.
         engine.records.all(function (err, records) {
           records = records.filter(function (r) {
             return r.zone === zonename;
@@ -239,6 +244,8 @@ module.exports.create = function (cli, engine/*, dnsd*/) {
         });
       });
     });
+
+    // I wonder what an API that gets ALL records from all zones is for
     app.get('/api/records', function (req, res) {
       engine.records.all(function (err, records) {
         res.send({ records: records.map(mapRecord) });

lib/store/index.js

@@ -1,101 +1,80 @@
 'use strict';
 
+var crypto = require('crypto');
+var dns = require('dns');
+var os = require('os');
+var path = require('path');
+
+var pathResolvers = {
+  '.': function fromCwd(relPath) {
+    return path.join(process.cwd(), relPath);
+  },
+  '~': function fromHomedir(relPath) {
+    if (!os.homedir) {
+      throw new Error(
+        'Resolving home directory relative paths is not supported in this version of node.'
+      );
+    }
+    return path.join(os.homedir(), relPath);
+  },
+  noop: function (p) { return p; }
+}
+
 module.exports.create = function (opts) {
   // opts = { filepath };
+  // `opts.filepath` is a module id or path to a module that contains a store plugin or file
+  var pathFn = pathResolvers[opts.filepath[0]] || pathResolvers.noop;
+  var storeId = pathFn(opts.filepath);
+  var pathToStore = require.resolve(storeId);
+
   var engine = { db: null };
 
   function notDeleted(r) {
     return !r.revokedAt && !r.deletedAt;
   }
 
-  var db = require(opts.filepath);
+  // instantiate the DB module
-  var stat = require('fs').statSync(opts.filepath);
+  var db = (pathToStore.slice(-5) === '.json') ?
-  var crypto = require('crypto');
+    // JSON files should be loaded using our built in store.json.js
-  //
+    require('./store.json.js')(pathToStore) :
-  // Manual Migration
+    // everything else should be loaded as a module and passed our opts object
-  //
+    require(storeId)(opts);
-  db.primaryNameservers.forEach(function (ns, i, arr) {
-    if ('string' === typeof ns) {
-      ns = { name: ns };
-      arr[i] = ns;
-    }
-    if (!ns.id) {
-      ns.id = crypto.randomBytes(16).toString('hex');
-    }
-  });
-  db.zones = db.zones || [];
-  if (db.domains) {
-    db.zones = db.zones.concat(db.domains);
-  }
-  db.zones.forEach(function (zone) {
-    if (!zone.name) {
-      zone.name = zone.id;
-      zone.id = null;
-    }
-    if (!zone.id) {
-      zone.id = crypto.randomBytes(16).toString('hex');
-    }
-    if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
-    if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
-  });
-  db.records.forEach(function (record) {
-    if (!record.id) {
-      record.id = crypto.randomBytes(16).toString('hex');
-    }
-  });
-  require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
-  //
-  // End Migration
-  //
 
-  db.save = function (cb) {
+  // TODO: examine usage of engine.primaryNameservers to see if we are supporting it right
-    if (db.save._saving) {
+  engine.primaryNameservers = db.primaryNameservers.list;
-      console.log('make pending');
-      db.save._pending.push(cb);
-      return;
-    }
 
-    db.save._saving = true;
-    require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
-      console.log('done writing');
-      var pending = db.save._pending.splice(0);
-      db.save._saving = false;
-      cb(err);
-      if (!pending.length) {
-        return;
-      }
-      db.save(function (err) {
-        console.log('double save');
-        pending.forEach(function (cb) { cb(err); });
-      });
-    });
-  };
-  db.save._pending = [];
-
-  engine.primaryNameservers = db.primaryNameservers;
   engine.peers = {
     all: function (cb) {
-      var dns = require('dns');
+      var pNS = db.primaryNameservers.list();
-      var count = db.primaryNameservers.length;
+
-      function gotRecord() {
+      function getRecord(ns, done) {
-        count -= 1;
-        if (!count) {
-          cb(null, db.primaryNameservers);
-        }
-      }
-      function getRecord(ns) {
         dns.resolve4(ns.name, function (err, addresses) {
           console.log('ns addresses:');
           console.log(addresses);
-          if (err) { console.error(err); gotRecord(); return; }
+          if (err) { console.error(err); done(); return; }
           ns.type = 'A';
           ns.address = addresses[0];
-          gotRecord();
+          done();
         });
       }
-      db.primaryNameservers.forEach(getRecord);
+
+      // resolve addreses for all of the primary nameservers in parallel
+      pNS.forEach(function (ns) {
+        var status = { pending: true };
+        function done() {
+          status.pending = false;
+          // TODO: determine if the locally stored records should get updated
+          var incomplete = tasks.filter(function (s) { return s.pending; });
+          if (incomplete.length < 1) {
+            cb(null, pNS);
+          }
+        }
+        getRecord(ns, done);
+        return status;
+      });
     }
   };
+
   engine.zones = {
     _immutableKeys: [ 'id', 'name', 'primary', 'serial', 'revokedAt', 'changedAt', 'insertedAt', 'updatedAt', 'deletedAt' ]
   , _mutableKeys: [ 'admin', 'expiration', 'minimum', 'refresh', 'retry', 'ttl', 'vanity' ]
@@ -105,8 +84,9 @@ module.exports.create = function (opts) {
       // epoch in seconds will do
       return parseInt(Math.round(date/1000).toString().slice(-10), 10);
     }
+    // NOTE/TODO: despite the _, _toSoa is used outside this file (in lib/digd.js and lib/httpd.js)
   , _toSoa: function (domain) {
-      var nameservers = domain.vanityNs || engine.primaryNameservers.map(function (n) { return n.name; });
+      var nameservers = domain.vanityNs || engine.primaryNameservers().map(function (n) { return n.name; });
 
       var index = Math.floor(Math.random() * nameservers.length) % nameservers.length;
       var nameserver = nameservers[index];
@@ -122,6 +102,7 @@ module.exports.create = function (opts) {
       , name_server: nameserver
 
         // admin -- email address or domain for admin
+        // default is effectively admin@{domain name}
       , admin: domain.admin || ('admin.' + domain.name)
       , email_addr: domain.admin || ('admin.' + domain.name)
 
@@ -148,7 +129,7 @@ module.exports.create = function (opts) {
     }
   , all: function (cb) {
       process.nextTick(function () {
-        cb(null, db.zones.slice(0).filter(notDeleted));
+        cb(null, db.zones().filter(notDeleted));
       });
     }
   , get: function (queries, cb) {
@@ -157,7 +138,7 @@ module.exports.create = function (opts) {
           return { name: n };
         });
       }
-      var myDomains = db.zones.filter(function (d) {
+      var myDomains = db.zones().filter(function (d) {
         return queries.some(function (q) {
           return (d.name.toLowerCase() === q.name) && notDeleted(d);
         });
@@ -167,20 +148,18 @@ module.exports.create = function (opts) {
       });
     }
   , touch: function (zone, cb) {
-      var existing;
+      db.zones.get(zone, function (err, existing) {
-      db.zones.some(function (z) {
+        if (err || !existing) {
-        if (z.id && zone.id === z.id) { existing = z; return true; }
+          cb(err, null);
-        if (z.name && zone.name === z.name) { existing = z; return true; }
-      });
-      if (!existing) {
-        cb(null, null);
           return;
         }
         existing.updatedAt = new Date().valueOf(); // toISOString();
         console.log('touch saving...');
-      db.save(function (err) {
+        db.zone.update(existing, function (err) {
           cb(err, !err && existing || null);
         });
+        return;
+      });
     }
   , save: function (zone, cb) {
       if (zone.id) {
@@ -191,65 +170,69 @@ module.exports.create = function (opts) {
       }
     }
   , update: function (zone, cb) {
-      var existing;
+      db.zones.get({ id: zone.id }, function (err, found) {
         var dirty;
 
-      db.zones.some(function (z) {
+        if (err) {
-        if (z.id === zone.id) {
+          console.log('error finding zone');
-          existing = z;
+          cb(new Error("Error finding zone for '" + zone.id + "'"), null);
-          return true;
+          return;
         }
-      });
 
-      if (!existing) {
+        if (!found) {
           console.log('no existing zone');
           cb(new Error("zone for '" + zone.id + "' does not exist"), null);
           return;
         }
 
         console.log('found existing zone');
-      console.log(existing);
+        console.log(found);
         console.log(zone);
         Object.keys(zone).forEach(function (key) {
           if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; }
-        if (existing[key] !== zone[key]) {
+          if (found[key] !== zone[key]) {
             dirty = true;
-          console.log('existing key', key, existing[key], zone[key]);
+            console.log('existing key', key, found[key], zone[key]);
-          existing[key] = zone[key];
+            found[key] = zone[key];
           }
         });
 
-      zone.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000);
+        found.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000);
         if (dirty) {
-        zone.changedAt = zone.updatedAt;
+          found.changedAt = found.updatedAt;
         }
 
         console.log('saving...');
-      db.save(function (err) {
+        db.zones.update(found, function (err) {
-        cb(err, !err && existing || null);
+          cb(err, !err && found || null);
+        });
       });
     }
   , create: function (zone, cb) {
-      var newZone = { id: crypto.randomBytes(16).toString('hex') };
+    var zoneName = (zone.name||'').toLowerCase();
-      var existing;
+    db.zones.get({ name: zoneName }, function (err, found) {
-      var nss = [];
+      if (err) {
-
+        console.error(err);
-      zone.name = (zone.name||'').toLowerCase();
+        cb(new Error("error attempting to create new zone '" + zoneName + "'"));
-      db.zones.some(function (z) {
-        if (z.name === zone.name) {
-          existing = z;
-          return true;
-        }
-      });
-
-      if (existing) {
-        cb(new Error("tried to create new zone, but '" + existing.name + "' already exists"));
         return;
       }
-      newZone.name = zone.name;
+      if (found) {
+        cb(new Error("tried to create new zone, but '" + found.name + "' already exists"));
+        return;
+      }
+
+      var newZone = {
+        id: crypto.randomBytes(16).toString('hex'),
+        name: zoneName
+      };
+      var nss = [];
+
       newZone.createdAt = Date.now();
       newZone.updatedAt = newZone.createdAt;
 
+      /*
+      Set only the mutable keys in the new zone from the proposed zone object
+      */
       Object.keys(zone).forEach(function (key) {
         //if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; }
         if (-1 === engine.zones._mutableKeys.indexOf(key)) { return; }
@@ -262,7 +245,12 @@ module.exports.create = function (opts) {
       } else {
         newZone.vanity = false;
       }
-      db.primaryNameservers.forEach(function (ns, i) {
+
+      // TODO: distinguish between primary and secondary zones
+      // TODO: determine if we need to do anything special for delegation
+
+      // create records for the primary nameservers (or vanity name servers)
+      db.primaryNameservers.list().forEach(function (ns, i) {
         var nsx = 'ns' + (i + 1);
         var nsZone;
         var ttl = 43200; // 12h // TODO pick a well-reasoned number
@@ -302,7 +290,13 @@ module.exports.create = function (opts) {
         });
       });
 
-      db.zones.push(newZone);
+      db.zones.create(newZone, function (err) {
+        // WIP: going to need to figure out how to manage this as a transaction
+        // Significant benefit to having records owned by the zone is we won't have
+        // records for zones that don't otherwise exist - at least at the engine level.
+
+        // every line below this one is not yet modified...
+      });
       nss.forEach(function (ns) {
         db.records.push(ns);
       });

lib/store/store.json.js

@@ -0,0 +1,167 @@
+'use strict';
+
+function jsonDeepClone(target) {
+  return JSON.parse(
+    JSON.stringify(target)
+  );
+}
+/*
+init() should return an object with: {
+  save: function -> undefined - changes to in memory representation should be persisted
+  This could be considered the equivalent of committing a transaction to the database.
+  primaryNameservers: {
+    list: function -> list nameservers
+  },
+  zones: {
+    list: function -> list zones,
+    find: function -> read zone by ???,
+    create:
+    update:
+    delete:
+  },
+  records: {
+    list: function -> list records,
+    find: function -> read record by ???,
+    create:
+    update:
+    delete:
+  }
+}
+
+All lists will be a deep copy of the data actually stored.
+ */
+
+module.exports = function init (opts) {
+  // opts = { filepath };
+
+  var db = require(opts.filepath);
+  var stat = require('fs').statSync(opts.filepath);
+  var crypto = require('crypto');
+  //
+  // Manual Migration
+  //
+
+  // Convert the primary nameservers from strings to objects with names and IDs.
+  db.primaryNameservers.forEach(function (ns, i, arr) {
+    if ('string' === typeof ns) {
+      ns = { name: ns };
+      arr[i] = ns;
+    }
+    if (!ns.id) {
+      ns.id = crypto.randomBytes(16).toString('hex');
+    }
+  });
+
+  // Convert domains to zones and ensure that they have proper IDs and timestamps
+  db.zones = db.zones || [];
+  if (db.domains) {
+    db.zones = db.zones.concat(db.domains);
+  }
+  db.zones.forEach(function (zone) {
+    if (!zone.name) {
+      zone.name = zone.id;
+      zone.id = null;
+    }
+    if (!zone.id) {
+      zone.id = crypto.randomBytes(16).toString('hex');
+    }
+    if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
+    if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
+  });
+
+  // Records belong to zones, but they (currently) refer to them by a zone property.
+  // NOTE/TODO: This may pose problems where the whole list of records is not easily
+  // filtered / kept in memory / indexed and/or retrieved by zone. Traditionally,
+  // records are stored "within a zone" in a zone file. We may wish to have the
+  // DB API behave more traditionally, even though some stores (like a SQL database
+  // table) might actually store the zone as a property of a record as we currently do.
+  db.records.forEach(function (record) {
+    if (!record.id) {
+      record.id = crypto.randomBytes(16).toString('hex');
+    }
+  });
+
+  // Write the migrated data
+  require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
+  //
+  // End Migration
+  //
+
+  var save = function save (cb) {
+    if (save._saving) {
+      console.log('make pending');
+      save._pending.push(cb);
+      return;
+    }
+
+    save._saving = true;
+    // TODO: replace with something not destructive to original non-json data
+    require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
+      console.log('done writing');
+      var pending = save._pending.splice(0);
+      save._saving = false;
+      cb(err);
+      if (!pending.length) {
+        return;
+      }
+      save(function (err) {
+        console.log('double save');
+        pending.forEach(function (cb) { cb(err); });
+      });
+    });
+  };
+  save._pending = [];
+
+  var dbApi = {
+    save: function () {
+      // hide _pending and _saving from callers
+      var args = [].slice.call(arguments);
+      return save.apply(null, args);
+    },
+    // primaryNameservers really isn't editable - it's literally the list of FQDN's
+    // that this database is replicated to in a multi-master fashion.
+    //
+    // However, lib/store/index.js does plenty to update these records in support
+    // of the SOA records that are built from them (as does this file in the "migration"
+    // section). I'm toying with the idea of not storing them seperately or creating the
+    // SOA records somewhat immediately.
+    primaryNameservers: {
+      list: function listNameservers() {
+        return jsonDeepClone(db.primaryNameservers);
+      }
+    },
+    zones: {
+      list: function listZones() {
+        return jsonDeepClone(db.zones);
+      },
+      find: function getZone(predicate, cb) {
+        var found;
+        db.zones.some(function (z) {
+          if (z.id && predicate.id === z.id) { found = z; return true; }
+          if (z.name && predicate.name === z.name) { found = z; return true; }
+        });
+        if (!found) {
+          cb(null, null);
+          return;
+        }
+        cb(null, jsonDeepClone(found));
+        return;
+      },
+      create: function() {},
+      update: function() {},
+      delete: function() {}
+    },
+    records: {
+      list: function listRecords() {
+        return jsonDeepClone(db.records);
+      },
+      find: function getRecord(predicate, cb) {
+      },
+      create: function() {},
+      update: function() {},
+      delete: function() {}
+    }
+  };
+
+  return dbApi;
+};