From 8eec24c555bc296ce5b3a5fe3a6c0a2684760092 Mon Sep 17 00:00:00 2001 From: Aaron Madsen Date: Wed, 21 Nov 2018 10:53:54 -0700 Subject: [PATCH] WIP: split the engine and the store via an API --- .gitignore | 1 + bin/digd.js | 4 +- lib/digd.js | 4 +- lib/httpd.js | 7 + lib/{store.json.js => store/index.js} | 268 +++++++++++++------------- lib/store/store.json.js | 167 ++++++++++++++++ 6 files changed, 310 insertions(+), 141 deletions(-) rename lib/{store.json.js => store/index.js} (68%) create mode 100644 lib/store/store.json.js diff --git a/.gitignore b/.gitignore index 97c3e07..1151208 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules .*.sw* +local-db.js diff --git a/bin/digd.js b/bin/digd.js index 8f90164..3663abb 100755 --- a/bin/digd.js +++ b/bin/digd.js @@ -387,7 +387,7 @@ cli.main(function (args, cli) { } try { - engine = engine || require('../lib/store.json.js').create(engineOpts); + engine = engine || require('../lib/store').create(engineOpts); } catch(e) { respondWithResults(e); return; @@ -413,7 +413,7 @@ cli.main(function (args, cli) { } if (cli.http) { try { - engine = engine || require('../lib/store.json.js').create(engineOpts); + engine = engine || require('../lib/store').create(engineOpts); } catch(e) { console.error(e); return; diff --git a/lib/digd.js b/lib/digd.js index 63c797b..7761c56 100644 --- a/lib/digd.js +++ b/lib/digd.js @@ -184,7 +184,7 @@ function getNs(engine, zs, results, cb) { // d.vanityNs should only be vanity nameservers (pointing to this same server) if (z.vanityNs || results.authority.some(function (ns) { console.log('[debug] ns', ns); - return -1 !== engine.primaryNameservers.indexOf(ns.data.toLowerCase()); + return -1 !== engine.primaryNameservers().indexOf(ns.data.toLowerCase()); })) { results.authority.length = 0; results.authority.push(engine.zones._toSoa(z)); @@ -359,7 +359,7 @@ module.exports.query = function (engine, query, cb) { // NOTE: I think that the issue here is EXTERNAL vs INTERNAL vanity NS // We _should_ reply for EXTERNAL vanity NS... but not when it's listed on the SOA internally? // It's surrounding the problem of what if I do sub domain delegation to the same server. - if (-1 === engine.primaryNameservers.indexOf(r.data.toLowerCase())) { + if (-1 === engine.primaryNameservers().indexOf(r.data.toLowerCase())) { console.log("It's a vanity NS"); return false; } diff --git a/lib/httpd.js b/lib/httpd.js index 7594286..c2c1476 100644 --- a/lib/httpd.js +++ b/lib/httpd.js @@ -230,6 +230,11 @@ module.exports.create = function (cli, engine/*, dnsd*/) { zone.class = zone.className; zone.type = zone.typeName; zone.soa = true; + + // TODO: consider sending a predicate object through the engine + // to the actual store in case it is highly inefficient to transfer + // a large number of records from the store that will just be + // thrown away. engine.records.all(function (err, records) { records = records.filter(function (r) { return r.zone === zonename; @@ -239,6 +244,8 @@ module.exports.create = function (cli, engine/*, dnsd*/) { }); }); }); + + // I wonder what an API that gets ALL records from all zones is for app.get('/api/records', function (req, res) { engine.records.all(function (err, records) { res.send({ records: records.map(mapRecord) }); diff --git a/lib/store.json.js b/lib/store/index.js similarity index 68% rename from lib/store.json.js rename to lib/store/index.js index 672ebcb..6507812 100644 --- a/lib/store.json.js +++ b/lib/store/index.js @@ -1,101 +1,80 @@ 'use strict'; +var crypto = require('crypto'); +var dns = require('dns'); +var os = require('os'); +var path = require('path'); + +var pathResolvers = { + '.': function fromCwd(relPath) { + return path.join(process.cwd(), relPath); + }, + '~': function fromHomedir(relPath) { + if (!os.homedir) { + throw new Error( + 'Resolving home directory relative paths is not supported in this version of node.' + ); + } + return path.join(os.homedir(), relPath); + }, + noop: function (p) { return p; } +} + module.exports.create = function (opts) { // opts = { filepath }; + // `opts.filepath` is a module id or path to a module that contains a store plugin or file + var pathFn = pathResolvers[opts.filepath[0]] || pathResolvers.noop; + var storeId = pathFn(opts.filepath); + var pathToStore = require.resolve(storeId); + var engine = { db: null }; function notDeleted(r) { return !r.revokedAt && !r.deletedAt; } - var db = require(opts.filepath); - var stat = require('fs').statSync(opts.filepath); - var crypto = require('crypto'); - // - // Manual Migration - // - db.primaryNameservers.forEach(function (ns, i, arr) { - if ('string' === typeof ns) { - ns = { name: ns }; - arr[i] = ns; - } - if (!ns.id) { - ns.id = crypto.randomBytes(16).toString('hex'); - } - }); - db.zones = db.zones || []; - if (db.domains) { - db.zones = db.zones.concat(db.domains); - } - db.zones.forEach(function (zone) { - if (!zone.name) { - zone.name = zone.id; - zone.id = null; - } - if (!zone.id) { - zone.id = crypto.randomBytes(16).toString('hex'); - } - if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); } - if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); } - }); - db.records.forEach(function (record) { - if (!record.id) { - record.id = crypto.randomBytes(16).toString('hex'); - } - }); - require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2)); - // - // End Migration - // + // instantiate the DB module + var db = (pathToStore.slice(-5) === '.json') ? + // JSON files should be loaded using our built in store.json.js + require('./store.json.js')(pathToStore) : + // everything else should be loaded as a module and passed our opts object + require(storeId)(opts); - db.save = function (cb) { - if (db.save._saving) { - console.log('make pending'); - db.save._pending.push(cb); - return; - } + // TODO: examine usage of engine.primaryNameservers to see if we are supporting it right + engine.primaryNameservers = db.primaryNameservers.list; - db.save._saving = true; - require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) { - console.log('done writing'); - var pending = db.save._pending.splice(0); - db.save._saving = false; - cb(err); - if (!pending.length) { - return; - } - db.save(function (err) { - console.log('double save'); - pending.forEach(function (cb) { cb(err); }); - }); - }); - }; - db.save._pending = []; - - engine.primaryNameservers = db.primaryNameservers; engine.peers = { all: function (cb) { - var dns = require('dns'); - var count = db.primaryNameservers.length; - function gotRecord() { - count -= 1; - if (!count) { - cb(null, db.primaryNameservers); - } - } - function getRecord(ns) { + var pNS = db.primaryNameservers.list(); + + function getRecord(ns, done) { dns.resolve4(ns.name, function (err, addresses) { console.log('ns addresses:'); console.log(addresses); - if (err) { console.error(err); gotRecord(); return; } + if (err) { console.error(err); done(); return; } ns.type = 'A'; ns.address = addresses[0]; - gotRecord(); + done(); }); } - db.primaryNameservers.forEach(getRecord); + + // resolve addreses for all of the primary nameservers in parallel + pNS.forEach(function (ns) { + var status = { pending: true }; + function done() { + status.pending = false; + // TODO: determine if the locally stored records should get updated + var incomplete = tasks.filter(function (s) { return s.pending; }); + if (incomplete.length < 1) { + cb(null, pNS); + } + } + getRecord(ns, done); + return status; + }); } }; + engine.zones = { _immutableKeys: [ 'id', 'name', 'primary', 'serial', 'revokedAt', 'changedAt', 'insertedAt', 'updatedAt', 'deletedAt' ] , _mutableKeys: [ 'admin', 'expiration', 'minimum', 'refresh', 'retry', 'ttl', 'vanity' ] @@ -105,8 +84,9 @@ module.exports.create = function (opts) { // epoch in seconds will do return parseInt(Math.round(date/1000).toString().slice(-10), 10); } + // NOTE/TODO: despite the _, _toSoa is used outside this file (in lib/digd.js and lib/httpd.js) , _toSoa: function (domain) { - var nameservers = domain.vanityNs || engine.primaryNameservers.map(function (n) { return n.name; }); + var nameservers = domain.vanityNs || engine.primaryNameservers().map(function (n) { return n.name; }); var index = Math.floor(Math.random() * nameservers.length) % nameservers.length; var nameserver = nameservers[index]; @@ -122,6 +102,7 @@ module.exports.create = function (opts) { , name_server: nameserver // admin -- email address or domain for admin + // default is effectively admin@{domain name} , admin: domain.admin || ('admin.' + domain.name) , email_addr: domain.admin || ('admin.' + domain.name) @@ -148,7 +129,7 @@ module.exports.create = function (opts) { } , all: function (cb) { process.nextTick(function () { - cb(null, db.zones.slice(0).filter(notDeleted)); + cb(null, db.zones().filter(notDeleted)); }); } , get: function (queries, cb) { @@ -157,7 +138,7 @@ module.exports.create = function (opts) { return { name: n }; }); } - var myDomains = db.zones.filter(function (d) { + var myDomains = db.zones().filter(function (d) { return queries.some(function (q) { return (d.name.toLowerCase() === q.name) && notDeleted(d); }); @@ -167,19 +148,17 @@ module.exports.create = function (opts) { }); } , touch: function (zone, cb) { - var existing; - db.zones.some(function (z) { - if (z.id && zone.id === z.id) { existing = z; return true; } - if (z.name && zone.name === z.name) { existing = z; return true; } - }); - if (!existing) { - cb(null, null); + db.zones.get(zone, function (err, existing) { + if (err || !existing) { + cb(err, null); + return; + } + existing.updatedAt = new Date().valueOf(); // toISOString(); + console.log('touch saving...'); + db.zone.update(existing, function (err) { + cb(err, !err && existing || null); + }); return; - } - existing.updatedAt = new Date().valueOf(); // toISOString(); - console.log('touch saving...'); - db.save(function (err) { - cb(err, !err && existing || null); }); } , save: function (zone, cb) { @@ -191,65 +170,69 @@ module.exports.create = function (opts) { } } , update: function (zone, cb) { - var existing; - var dirty; + db.zones.get({ id: zone.id }, function (err, found) { + var dirty; - db.zones.some(function (z) { - if (z.id === zone.id) { - existing = z; - return true; + if (err) { + console.log('error finding zone'); + cb(new Error("Error finding zone for '" + zone.id + "'"), null); + return; } - }); - if (!existing) { - console.log('no existing zone'); - cb(new Error("zone for '" + zone.id + "' does not exist"), null); - return; - } - - console.log('found existing zone'); - console.log(existing); - console.log(zone); - Object.keys(zone).forEach(function (key) { - if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; } - if (existing[key] !== zone[key]) { - dirty = true; - console.log('existing key', key, existing[key], zone[key]); - existing[key] = zone[key]; + if (!found) { + console.log('no existing zone'); + cb(new Error("zone for '" + zone.id + "' does not exist"), null); + return; } - }); - zone.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000); - if (dirty) { - zone.changedAt = zone.updatedAt; - } + console.log('found existing zone'); + console.log(found); + console.log(zone); + Object.keys(zone).forEach(function (key) { + if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; } + if (found[key] !== zone[key]) { + dirty = true; + console.log('existing key', key, found[key], zone[key]); + found[key] = zone[key]; + } + }); - console.log('saving...'); - db.save(function (err) { - cb(err, !err && existing || null); + found.updatedAt = new Date().valueOf(); // toISOString(); // Math.round(Date.now() / 1000); + if (dirty) { + found.changedAt = found.updatedAt; + } + + console.log('saving...'); + db.zones.update(found, function (err) { + cb(err, !err && found || null); + }); }); } , create: function (zone, cb) { - var newZone = { id: crypto.randomBytes(16).toString('hex') }; - var existing; - var nss = []; - - zone.name = (zone.name||'').toLowerCase(); - db.zones.some(function (z) { - if (z.name === zone.name) { - existing = z; - return true; - } - }); - - if (existing) { - cb(new Error("tried to create new zone, but '" + existing.name + "' already exists")); + var zoneName = (zone.name||'').toLowerCase(); + db.zones.get({ name: zoneName }, function (err, found) { + if (err) { + console.error(err); + cb(new Error("error attempting to create new zone '" + zoneName + "'")); return; } - newZone.name = zone.name; + if (found) { + cb(new Error("tried to create new zone, but '" + found.name + "' already exists")); + return; + } + + var newZone = { + id: crypto.randomBytes(16).toString('hex'), + name: zoneName + }; + var nss = []; + newZone.createdAt = Date.now(); newZone.updatedAt = newZone.createdAt; + /* + Set only the mutable keys in the new zone from the proposed zone object + */ Object.keys(zone).forEach(function (key) { //if (-1 !== engine.zones._immutableKeys.indexOf(key)) { return; } if (-1 === engine.zones._mutableKeys.indexOf(key)) { return; } @@ -262,7 +245,12 @@ module.exports.create = function (opts) { } else { newZone.vanity = false; } - db.primaryNameservers.forEach(function (ns, i) { + + // TODO: distinguish between primary and secondary zones + // TODO: determine if we need to do anything special for delegation + + // create records for the primary nameservers (or vanity name servers) + db.primaryNameservers.list().forEach(function (ns, i) { var nsx = 'ns' + (i + 1); var nsZone; var ttl = 43200; // 12h // TODO pick a well-reasoned number @@ -302,7 +290,13 @@ module.exports.create = function (opts) { }); }); - db.zones.push(newZone); + db.zones.create(newZone, function (err) { + // WIP: going to need to figure out how to manage this as a transaction + // Significant benefit to having records owned by the zone is we won't have + // records for zones that don't otherwise exist - at least at the engine level. + + // every line below this one is not yet modified... + }); nss.forEach(function (ns) { db.records.push(ns); }); diff --git a/lib/store/store.json.js b/lib/store/store.json.js new file mode 100644 index 0000000..392ce18 --- /dev/null +++ b/lib/store/store.json.js @@ -0,0 +1,167 @@ +'use strict'; + +function jsonDeepClone(target) { + return JSON.parse( + JSON.stringify(target) + ); +} +/* +init() should return an object with: { + save: function -> undefined - changes to in memory representation should be persisted + This could be considered the equivalent of committing a transaction to the database. + primaryNameservers: { + list: function -> list nameservers + }, + zones: { + list: function -> list zones, + find: function -> read zone by ???, + create: + update: + delete: + }, + records: { + list: function -> list records, + find: function -> read record by ???, + create: + update: + delete: + } +} + +All lists will be a deep copy of the data actually stored. + */ + +module.exports = function init (opts) { + // opts = { filepath }; + + var db = require(opts.filepath); + var stat = require('fs').statSync(opts.filepath); + var crypto = require('crypto'); + // + // Manual Migration + // + + // Convert the primary nameservers from strings to objects with names and IDs. + db.primaryNameservers.forEach(function (ns, i, arr) { + if ('string' === typeof ns) { + ns = { name: ns }; + arr[i] = ns; + } + if (!ns.id) { + ns.id = crypto.randomBytes(16).toString('hex'); + } + }); + + // Convert domains to zones and ensure that they have proper IDs and timestamps + db.zones = db.zones || []; + if (db.domains) { + db.zones = db.zones.concat(db.domains); + } + db.zones.forEach(function (zone) { + if (!zone.name) { + zone.name = zone.id; + zone.id = null; + } + if (!zone.id) { + zone.id = crypto.randomBytes(16).toString('hex'); + } + if (!zone.createdAt) { zone.createdAt = stat.mtime.valueOf(); } + if (!zone.updatedAt) { zone.updatedAt = stat.mtime.valueOf(); } + }); + + // Records belong to zones, but they (currently) refer to them by a zone property. + // NOTE/TODO: This may pose problems where the whole list of records is not easily + // filtered / kept in memory / indexed and/or retrieved by zone. Traditionally, + // records are stored "within a zone" in a zone file. We may wish to have the + // DB API behave more traditionally, even though some stores (like a SQL database + // table) might actually store the zone as a property of a record as we currently do. + db.records.forEach(function (record) { + if (!record.id) { + record.id = crypto.randomBytes(16).toString('hex'); + } + }); + + // Write the migrated data + require('fs').writeFileSync(opts.filepath, JSON.stringify(db, null, 2)); + // + // End Migration + // + + var save = function save (cb) { + if (save._saving) { + console.log('make pending'); + save._pending.push(cb); + return; + } + + save._saving = true; + // TODO: replace with something not destructive to original non-json data + require('fs').writeFile(opts.filepath, JSON.stringify(db, null, 2), function (err) { + console.log('done writing'); + var pending = save._pending.splice(0); + save._saving = false; + cb(err); + if (!pending.length) { + return; + } + save(function (err) { + console.log('double save'); + pending.forEach(function (cb) { cb(err); }); + }); + }); + }; + save._pending = []; + + var dbApi = { + save: function () { + // hide _pending and _saving from callers + var args = [].slice.call(arguments); + return save.apply(null, args); + }, + // primaryNameservers really isn't editable - it's literally the list of FQDN's + // that this database is replicated to in a multi-master fashion. + // + // However, lib/store/index.js does plenty to update these records in support + // of the SOA records that are built from them (as does this file in the "migration" + // section). I'm toying with the idea of not storing them seperately or creating the + // SOA records somewhat immediately. + primaryNameservers: { + list: function listNameservers() { + return jsonDeepClone(db.primaryNameservers); + } + }, + zones: { + list: function listZones() { + return jsonDeepClone(db.zones); + }, + find: function getZone(predicate, cb) { + var found; + db.zones.some(function (z) { + if (z.id && predicate.id === z.id) { found = z; return true; } + if (z.name && predicate.name === z.name) { found = z; return true; } + }); + if (!found) { + cb(null, null); + return; + } + cb(null, jsonDeepClone(found)); + return; + }, + create: function() {}, + update: function() {}, + delete: function() {} + }, + records: { + list: function listRecords() { + return jsonDeepClone(db.records); + }, + find: function getRecord(predicate, cb) { + }, + create: function() {}, + update: function() {}, + delete: function() {} + } + }; + + return dbApi; +};