WIP: keep iterating on the store API

lib/store/store.json.js

@@ -1,10 +1,99 @@
 'use strict';
 
+var crypto = require('crypto');
+
 function jsonDeepClone(target) {
   return JSON.parse(
     JSON.stringify(target)
   );
 }
+
+function mergeObjects() {
+  // arguments should be an array of objects. We
+  // reverse it because the last argument to set
+  // a value wins.
+  var args = [].slice.call(arguments).reverse();
+  var len = args.length;
+  if (len === 1) {
+    return args[0];
+  }
+
+  // gather the set of keys from all arguments
+  var keyLists = args.map(function (arg) {
+    return Object.keys(arg);
+  });
+
+  var keys = Object.keys(keyLists.reduce(function (all, list) {
+    list.forEach(function (k) {
+      all[k] = true;
+    });
+    return all;
+  }, {}));
+
+  // for each key
+  return keys.reduce(function (target, k) {
+    // find the first argument (because of the reverse() above) with the key set
+    var values = [];
+    var isObject = false;
+    for (var i = 0; i < len; i++) {
+      var v = args[i];
+      var vType = typeof v;
+
+      if (vType === 'object') {
+        if (!v) {
+          // typeof null is object. null is the only falsey object. null represents
+          // a delete or the end of our argument list;
+          break;
+        }
+        // we need to collect values until we get a non-object, so we can merge them
+        values.push(v);
+        isObject = true;
+      } else if (!isObject) {
+        if (vType === 'undefined') {
+          // if the arg actually has the key set this is effectively a "delete"
+          if (keyList[i].indexOf(k) != -1) {
+            break;
+          }
+          // otherwise we need to check the next argument's value, so we don't break the loop
+        } else {
+          values.push(v);
+          break;
+        }
+      } else {
+        // a previous value was an object, this one isn't
+        // That means we are done collecting values.
+        break;
+      }
+    }
+
+    if (values.length > 0) {
+      target[k] = mergeObjects.apply(null, values);
+    }
+
+    return target;
+  }, {});
+}
+
+function prepareZone(zone, options) {
+  var opts = options || {};
+  var timestamp = opts.timestamp || Date.now();
+  if (!zone.name) {
+    zone.name = zone.id;
+    zone.id = null;
+  }
+  if (!zone.id) {
+    zone.id = crypto.randomBytes(16).toString('hex');
+  }
+  if (!zone.createdAt) { zone.createdAt = timestamp; }
+  if (!zone.updatedAt || opts.isUpdate) { zone.updatedAt = timestamp; }
+
+  // create a names set for the zone, keyed by record name mapped to
+  // an object for the various records with that name, by type (A, MX, TXT, etc.)
+  zone.records = zone.records || {};
+
+  return zone;
+}
+
 /*
 init() should return an object with: {
   save: function -> undefined - changes to in memory representation should be persisted
@@ -14,14 +103,12 @@ init() should return an object with: {
   },
   zones: {
     list: function -> list zones,
-    find: function -> read zone by ???,
     create:
     update:
     delete:
   },
   records: {
     list: function -> list records,
-    find: function -> read record by ???,
     create:
     update:
     delete:
@@ -35,51 +122,71 @@ module.exports = function init (opts) {
   // opts = { filepath };
 
   var db = require(opts.filepath);
-  var stat = require('fs').statSync(opts.filepath);
+  var mtime = require('fs').statSync(opts.filepath).mtime.valueOf();
-  var crypto = require('crypto');
+
   //
-  // Manual Migration
+  // Migration from other formats
   //
 
-  // Convert the primary nameservers from strings to objects with names and IDs.
+  // Convert the primary nameservers from an array of strings to objects with names and IDs.
-  db.primaryNameservers.forEach(function (ns, i, arr) {
+  // also switch to the 'peers' name, since we are really interested in the other FQDNs that
-    if ('string' === typeof ns) {
+  // use the same data store and are kept in sync.
-      ns = { name: ns };
+  var peerList = (!db.peers || Array.isArray(db.peers))? db.peers : Object.keys(db.peers).map(function (p) {
-      arr[i] = ns;
+    return db.peers[p];
-    }
-    if (!ns.id) {
-      ns.id = crypto.randomBytes(16).toString('hex');
-    }
   });
+  db.peers = [].concat(db.primaryNameservers, peerList).filter(function (p) {
+    // filer out empty strings, undefined, etc.
+    return !!p;
+  }).map(function (ns) {
+    var peer = ('string' === typeof ns)? ns : { name: ns };
+    if (!ns.id) {
+      peer.id = crypto.randomBytes(16).toString('hex');
+    }
+    return peer;
+  }).reduce(function (peers, p) {
+    peers[p.name] = p;
+    return peers;
+  }, {});
+  delete db.primaryNameservers;
 
   // Convert domains to zones and ensure that they have proper IDs and timestamps
-  db.zones = db.zones || [];
+  // Organize zones as a set of zone names
-  if (db.domains) {
+  var zoneList = (!db.zones || Array.isArray(db.zones))? db.zones : Object.keys(db.zones).map(function (z) {
-    db.zones = db.zones.concat(db.domains);
+    return db.zones[z];
-  }
-  db.zones.forEach(function (zone) {
-    if (!zone.name) {
-      zone.name = zone.id;
-      zone.id = null;
-    }
-    if (!zone.id) {
-      zone.id = crypto.randomBytes(16).toString('hex');
-    }
-    if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); }
-    if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); }
   });
 
-  // Records belong to zones, but they (currently) refer to them by a zone property.
+  db.zones = [].concat(db.domains, zoneList).filter(function (z) {
-  // NOTE/TODO: This may pose problems where the whole list of records is not easily
+    // filer out empty strings, undefined, etc.
+    return !!z;
+  }).map(function (zone) {
+    return prepareZone(zone, { timestamp: mtime });
+  }).reduce(function (zones, z) {
+    zones[z.name] = z;
+    return zones;
+  }, {});
+  delete db.domains;
+
+  // NOTE: Records belong to zones, but they previously referred to them only by a
+  // zone property. This may pose problems where the whole list of records is not easily
   // filtered / kept in memory / indexed and/or retrieved by zone. Traditionally,
-  // records are stored "within a zone" in a zone file. We may wish to have the
+  // records are stored "within a zone" in a zone file. We want to have the store API
-  // DB API behave more traditionally, even though some stores (like a SQL database
+  // behave more traditionally, even though some stores (like a SQL database
   // table) might actually store the zone as a property of a record as we currently do.
-  db.records.forEach(function (record) {
+  // (This fits with the somewhat unexpected and confusing logic of wildcard records.)
+  (db.records || []).forEach(function (record) {
+    // make sure the record has an ID
     if (!record.id) {
       record.id = crypto.randomBytes(16).toString('hex');
     }
+
+    // put it in it's zone - synthesize one if needed
+    db.zones[record.zone] = db.zones[record.zone] || prepareZone({ name: record.zone });
+    var zone = db.zones[record.zone];
+    zone.records[record.name] = zone.records[record.name] || [];
+    var recordsForName = zone.records[record.name];
+    recordsForName.push(record);
   });
+  delete db.records;
 
   // Write the migrated data
   require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2));
@@ -95,7 +202,6 @@ module.exports = function init (opts) {
     }
 
     save._saving = true;
-    // TODO: replace with something not destructive to original non-json data
     require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) {
       console.log('done writing');
       var pending = save._pending.splice(0);
@@ -112,54 +218,217 @@ module.exports = function init (opts) {
   };
   save._pending = [];
 
+  function matchPredicate(predicate) {
+    return function (toCheck) {
+      // which items match the predicate?
+      if (!toCheck) {
+        return false;
+      }
+
+      // check all the keys in the predicate - only supporting exact match
+      // of at least one listed option for all keys right now
+      if (Object.keys(predicate || {}).some(function (k) {
+        return [].concat(predicate[k]).indexOf(toCheck[k]) === -1;
+      })) {
+        return false;
+      }
+
+      // we have a match
+      return true;
+    };
+  }
+
+  function matchZone(predicate) {
+    var zonenames = !!predicate.name ? [].concat(predicate.name) : Object.keys(db.zones);
+    var check = matchPredicate(predicate);
+    // TODO: swap the filter() for a functional style "loop" recursive function
+    // that lets us return early if we have a limit, etc.
+    var found = zonenames.filter(function (zonename) {
+      /*
+      if (predicate.id && predicate.id !== z.id) { return false; }
+      if (predicate.name && predicate.name !== z.name) { return false; }
+      */
+      return check(db.zones[zonename]);
+    }).map(function (zonename) {
+      return db.zones[zonename];
+    });
+
+    return found;
+  }
+
   var dbApi = {
     save: function () {
       // hide _pending and _saving from callers
       var args = [].slice.call(arguments);
       return save.apply(null, args);
     },
-    // primaryNameservers really isn't editable - it's literally the list of FQDN's
+    // peers really isn't editable - it's literally the list of FQDN's
     // that this database is replicated to in a multi-master fashion.
     //
     // However, lib/store/index.js does plenty to update these records in support
     // of the SOA records that are built from them (as does this file in the "migration"
     // section). I'm toying with the idea of not storing them seperately or creating the
     // SOA records somewhat immediately.
-    primaryNameservers: {
+    peers: function listPeers(cb) {
-      list: function listNameservers() {
+      // Most data stores are going to have an asynchronous storage API. If we need
-        return jsonDeepClone(db.primaryNameservers);
+      // synchronous access to the data it is going to have to be cached. If it is
-      }
+      // cached, there is still the issue the cache getting out of sync (a legitimate
+      // issue anyway). If we explicitly make all of these operations async then we
+      // have greater flexibility for store implmentations to address these issues.
+      return setImmediate(cb, null, jsonDeepClone(db.peers));
     },
     zones: {
-      list: function listZones() {
+      /*
-        return jsonDeepClone(db.zones);
+      I'm fairly certan that zone names must be unique and therefore are legitimately
-      },
+      IDs within the zones namespace. This is similarly true of record names within a zone.
-      find: function getZone(predicate, cb) {
+      I'm not certain that having a distinct ID adds value and it may add confusion / complexity.
-        var found;
+       */
-        db.zones.some(function (z) {
+      // NOTE: `opts` exists so we can add options - like properties to read - easily in the future
-          if (z.id && predicate.id === z.id) { found = z; return true; }
+      // without modifying the function signature
-          if (z.name && predicate.name === z.name) { found = z; return true; }
+      list: function listZones(predicate, opts, cb) {
+        // TODO: consider whether we should just return the zone names
+        var found = jsonDeepClone(matchZone(predicate)).map(function (z) {
+          // This is fairly inefficient!! Consider alternative storage
+          // that does not require deleting the records like this.
+          delete z.records;
+          return z;
         });
-        if (!found) {
+        return setImmediate(cb, null, found);
-          cb(null, null);
-          return;
-        }
-        cb(null, jsonDeepClone(found));
-        return;
       },
-      create: function() {},
+      // // NOTE: I'm not sure we need a distinct 'find()' operation in the API
-      update: function() {},
+      // // unless we are going to limit the output of the
-      delete: function() {}
+      // // 'list()' operation in some incompatible way.
+      // // NOTE: `opts` exists so we can add options - like properties to read - easily in the future
+      // // without modifying the function signature
+      // find: function getZone(predicate, opts, cb) {
+      //   if (!predicate.name || predicate.id) {
+      //     return setImmediate(cb, new Error('Finding a zone requires a `name` or `id`'));
+      //   }
+      //   // TODO: implement a limit / short circuit and possibly offset
+      //   // to allow for paging of zone data.
+      //   var found = matchZone(predicate);
+      //   if (!found[0]) {
+      //     // TODO: make error message more specific?
+      //     return setImmediate(cb, new Error('Zone not found'));
+      //   }
+
+      //   var z = jsonDeepClone(found[0]);
+      //   delete z.records;
+      //   return setImmediate(cb, null, z);
+      // },
+      create: function createZone(zone, cb) {
+        // We'll need a lock mechanism of some sort that works
+        // for simultaneous requests and multiple processes.
+        matchZone({ name: zone.name }, function (err, matched) {
+          if (err) {
+            return setImmediate(cb, err);
+          }
+
+          var found = matched[0];
+          if (found) {
+            return setImmediate(cb, new Error('Zone ' + zone.name + ' already exists'));
+          }
+          
+          db.zones[zone.name] = prepareZone(zone);
+          return setImmediate(function () {
+            cb(null, jsonDeepClone(db.zones[zone.name]));
+            // release lock
+          });
+        });
+      },
+      update: function updateZone(zone, cb) {
+        // We'll need a lock mechanism of some sort that works
+        // for simultaneous requests and multiple processes.
+        matchZone({ name: zone.name }, function (err, matched) {
+          if (err) {
+            return setImmediate(cb, err);
+          }
+          var found = matched[0];
+          if (!found) {
+            return setImmediate(cb, new Error('Zone not found'));
+          }
+          // make sure we are not writing records through this interface
+          delete zone.records;
+
+          var combined = mergeObjects(found, zone);
+          db.zones[zone.name] = prepareZone(combined, { isUpdate: true });
+          return setImmediate(function () {
+            cb(null, jsonDeepClone(db.zones[zone.name]));
+            // release lock
+          });
+        });
+      },
+      delete: function(zone, cb) {
+        // We'll need a lock mechanism of some sort that works
+        // for simultaneous requests and multiple processes.
+        matchZone({ name: zone.name }, function (err, matched) {
+          if (err) {
+            return setImmediate(cb, err);
+          }
+          var found = matched[0];
+          if (!found) {
+            return setImmediate(cb, new Error('Zone not found'));
+          }
+
+          delete db.zones[zone.name];
+          return setImmediate(function () {
+            cb();
+            // release lock
+          });
+        });
+      }
     },
     records: {
-      list: function listRecords() {
+      list: function listRecords(rPredicate, cb) {
-        return jsonDeepClone(db.records);
+        var recordNames = [].concat(rPredicate.name);
+        var check = matchPredicate(rPredicate);
+
+        var found = matchZone({ name: rPredicate.zone }).reduce(function (records, zone) {
+          // get the records from the zone that match the record predicate
+          var zFound = recordNames.filter(function (name) {
+            return !!zone.records[name];
+          }).map(function (name) {
+            return zone.records[name].filter(check);
+          });
+          return records.concat(zFound);
+        }, []);
+
+        return setImmediate(cb, null, jsonDeepClone(found));
       },
-      find: function getRecord(predicate, cb) {
+      // find: function getRecord(rPredicate, cb) {
+      //   var recordNames = [].concat(rPredicate.name);
+      //   var check = matchPredicate(rPredicate);
+
+      //   // TODO: swap the `filter()` and `map()` for a functional style "loop"
+      //   // recursive function that lets us return early if we have a limit, etc.
+      //   var found = matchZone({ name: rPredicate.zone }).reduce(function (records, zone) {
+      //     // get the records from the zone that match the record predicate
+      //     var zFound = recordNames.filter(function (name) {
+      //       return !!zone.records[name];
+      //     }).map(function (name) {
+      //       return zone.records[name].filter(check);
+      //     });
+      //     return records.concat(zFound);
+      //   }, []);
+
+      //   return setImmediate(cb, null, jsonDeepClone(found[0]));
+      // },
+      create: function(record, cb) {
+        var zone = matchZone({ name: record.zone })[0];
+        if (!zone) {
+          return setImmediate(cb, new Error('Unble to find zone ' + record.zone + ' to create record'));
+        }
+
+        var records = zone.records[record.name] = zone.records[record.name] || [];
+        var check = matchPredicate(record);
+        if (records.filter(check)[0]) {
+          return setImmediate(cb, new Error('Exact record already exists in zone ' + record.zone ));
+        }
+
+        return setImmediate(cb, null, jsonDeepClone(found));
       },
-      create: function() {},
+      update: function(record, cb) {},
-      update: function() {},
+      delete: function(record, cb) {}
-      delete: function() {}
     }
   };