reworking vhost routes

lib/master.js

@@ -3,6 +3,7 @@
 var cluster = require('cluster');
 var PromiseA = require('bluebird');
 var memstore;
+var sqlstore;
 // TODO
 // var rootMasterKey;
 
@@ -26,24 +27,24 @@ function init(conf/*, state*/) {
   if (!conf.ipcKey) {
     conf.ipcKey = require('crypto').randomBytes(16).toString('base64');
   }
+  if (!conf.sqlite3Sock) {
+    conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
+  }
+  if (!conf.memstoreSock) {
+    conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
+  }
 
-  var memstoreOpts = {
-    sock: conf.memstoreSock || '/tmp/memstore.sock'
-
-    // If left 'null' or 'undefined' this defaults to a similar memstore
-    // with no special logic for 'cookie' or 'expires'
-  , store: cluster.isMaster && null //new require('express-session/session/memory')()
-
-    // a good default to use for instances where you might want
-    // to cluster or to run standalone, but with the same API
-  , serve: cluster.isMaster
-  , connect: cluster.isWorker
-  //, standalone: (1 === numCores) // overrides serve and connect
-    // TODO implement
-  , key: conf.ipcKey
-  };
   try {
-    require('fs').unlinkSync(memstoreOpts.sock);
+    require('fs').unlinkSync(conf.memstoreSock);
+  } catch(e) {
+    if ('ENOENT' !== e.code) {
+      console.error(e.stack);
+      console.error(JSON.stringify(e));
+    }
+    // ignore
+  }
+  try {
+    require('fs').unlinkSync(conf.sqlite3Sock);
   } catch(e) {
     if ('ENOENT' !== e.code) {
       console.error(e.stack);
@@ -53,8 +54,35 @@ function init(conf/*, state*/) {
   }
 
   var cstore = require('cluster-store');
-  var memstorePromise = cstore.create(memstoreOpts).then(function (_memstore) {
+  var sqlite3 = require('sqlite3-cluster/server');
-    memstore = _memstore;
+  var promise = PromiseA.all([
+    cstore.create({
+      sock: conf.memstoreSock
+    , serve: cluster.isMaster && conf.memstoreSock
+    , store: cluster.isMaster && null //new require('express-session/session/memory')()
+      // TODO implement
+    , key: conf.ipcKey
+    }).then(function (_memstore) {
+      memstore = _memstore;
+      return memstore;
+    })
+  , sqlite3.createServer({
+      verbose: null
+    , sock: conf.sqlite3Sock 
+    , ipcKey: conf.ipcKey
+    }).then(function (_sqlstore) {
+      sqlstore = _sqlstore;
+      return sqlstore;
+    })
+  ]).then(function (/*args*/) {
+    return conf;
+    /*
+    {
+      conf: conf
+    , memstore: memstore // args[0]
+    , sqlstore: sqlstore // args[1]
+    };
+    */
   });
 
   // TODO check the IP every 5 minutes and update it every hour
@@ -62,7 +90,7 @@ function init(conf/*, state*/) {
   // we don't want this to load right away (extra procesing time)
   setTimeout(updateIps, 1);
 
-  return memstorePromise;
+  return promise;
 }
 
 function touch(conf, state) {
@@ -88,6 +116,7 @@ function touch(conf, state) {
   //var config = require('./device.json');
 
   // require('ssl-root-cas').inject();
+  // TODO try SNI loopback.example.com as result of api.ipify.com with loopback token
 
   /*
   function phoneHome() {

lib/schemes-config.js

@@ -0,0 +1,174 @@
+'use strict';
+
+function deserialize(results) {
+  var config = { apis: {}, apps: {}, domains: {} };
+  results.apis.forEach(function (api) {
+    config.apis[api.id] = api;
+    api.domains = [];
+    api.domainIds = [];
+    api.domainsMap = {};
+  });
+  results.apps.forEach(function (app) {
+    config.apps[app.id] = app;
+    app.domains = [];
+    app.domainIds = [];
+    app.domainsMap = {};
+  });
+
+  results.domains.forEach(function (domain) {
+    config.domains[domain.id] = domain;
+    // as it currently stands each of these will only have one
+    /*
+    domain.apis = [];
+    domain.apiIds = [];
+    domain.apisMap = {};
+    domain.apps = [];
+    domain.appIds = [];
+    domain.appsMap = {};
+    */
+    domain.api = null;
+    domain.apiId = null;
+    domain.app = null;
+    domain.appId = null;
+    domain.appsMap = null;
+  });
+
+  results.apisDomains.forEach(function (ad) {
+    var api = config.apis[ad.apiId];
+    var domain = config.domains[ad.domainId];
+    if (api && !api.domainsMap[domain.id]) {
+      api.domainIds.push(domain.id);
+      api.domainsMap[domain.id] = domain;
+      api.domains.push(domain);
+    }
+    if (domain) {
+      if (domain.api) {
+        console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'");
+      }
+      domain.apiId = api.id;
+      domain.api = api;
+    }
+  });
+
+  results.appsDomains.forEach(function (ad) {
+    var app = config.apps[ad.appId];
+    var domain = config.domains[ad.domainId];
+    if (app && !app.domainsMap[domain.id]) {
+      app.domainIds.push(domain.id);
+      app.domainsMap[domain.id] = domain;
+      app.domains.push(domain);
+    }
+    if (domain) {
+      if (domain.app) {
+        console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'");
+      }
+      domain.appId = app.id;
+      domain.app = app;
+    }
+  });
+
+  return config;
+}
+
+module.exports.deserialize = deserialize;
+module.exports.create = function (db) {
+  console.log('[DB -1]');
+  var wrap = require('dbwrap');
+
+  var dir = [
+    //
+    // Collections
+    //
+    { tablename: 'apis'
+    , idname: 'id'      // io.lds.auth, com.daplie.radio
+    , unique: ['id'] 
+      // name // LDS Account, Radio
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name']
+    }
+  , { tablename: 'apps'
+    , idname: 'id'      // io.lds.auth, com.daplie.radio
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name']
+    }
+  , { tablename: 'domains'
+    , idname: 'id'      // api.coolaj86.com#radio
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name', 'token', 'accountId']
+    }
+
+    //
+    // Joins
+    //
+  , { tablename: 'apis_domains'
+    , idname: 'id'      // hash(api_id + domain_id)
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'apiId', 'domainId']
+      // TODO auto-form relations
+    , hasMany: ['apis', 'domains']
+    }
+  , { tablename: 'apps_domains'
+    , idname: 'id'      // hash(domain_id + app_id)
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'appId', 'domainId']
+      // TODO auto-form relations
+    , hasMany: ['apps', 'domains']
+    }
+
+/*
+  , { tablename: 'accounts_apis'
+    , idname: 'id'      // hash(account_id + api_id)
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'apiId']
+      // TODO auto-form relations
+    , hasMany: ['accounts', 'apis']
+    }
+  , { tablename: 'accounts_domains'
+    , idname: 'id'      // hash(account_id + domain_id)
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'domainId']
+      // TODO auto-form relations
+    , hasMany: ['accounts', 'domains']
+    }
+  , { tablename: 'accounts_apps'
+    , idname: 'id'      // hash(account_id + static_id)
+    , unique: ['id']
+    , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'staticId']
+      // TODO auto-form relations
+    , hasMany: ['accounts', 'apps']
+    }
+*/
+  ];
+
+  return wrap.wrap(db, dir).then(function (models) {
+    models.Config = {
+      get: function () {
+        var PromiseA = require('bluebird');
+
+        return PromiseA.all([
+          models.Apis.find(null, { limit: 10000 })
+        , models.Apps.find(null, { limit: 10000 })
+        , models.Domains.find(null, { limit: 10000 })
+        , models.ApisDomains.find(null, { limit: 10000 })
+        , models.AppsDomains.find(null, { limit: 10000 })
+        ]).then(function (args) {
+          var results = {
+            apis: args[0]
+          , apps: args[1]
+          , domains: args[2]
+          , apisDomains: args[3]
+          , appsDomains: args[4]
+          };
+
+          // create fixture with which to test
+          // console.log(JSON.stringify(results));
+
+          var config = deserialize(results);
+          
+          return config;
+        });
+      }
+    };
+
+    return models;
+  });
+};

lib/utils.js

@@ -3,7 +3,7 @@
 module.exports.getDomainInfo = function (apppath) {
   var parts = apppath.split(/[#%]+/);
   var hostname = parts.shift();
-  var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, '');
+  var pathname = parts.join('/').replace(/\/+/g, '/').replace(/\/$/g, '').replace(/^\//g, '');
 
   return {
     hostname: hostname

lib/worker.js

@@ -1,10 +1,40 @@
 'use strict';
 
-module.exports.create = function (webserver, info) {
+module.exports.create = function (webserver, info, state) {
+  if (!state) {
+    state = {};
+  }
+
+  var PromiseA = state.Promise || require('bluebird');
   var path = require('path');
   var vhostsdir = path.join(__dirname, 'vhosts');
   var app = require('express')();
   var apiHandler;
+  var memstore;
+  var sqlstores = {};
+  var models = {};
+  var systemFactory = require('sqlite3-cluster/client').createClientFactory({
+      dirname: path.join(__dirname, '..', 'var') // TODO info.conf
+    , prefix: 'com.daplie.'
+    //, dbname: 'config'
+    , suffix: ''
+    , ext: '.sqlite3'
+    , sock: info.conf.sqlite3Sock
+    , ipcKey: info.conf.ipcKey
+  });
+  var clientFactory = require('sqlite3-cluster/client').createClientFactory({
+      algorithm: 'aes'
+    , bits: 128
+    , mode: 'cbc'
+    , dirname: path.join(__dirname, '..', 'var') // TODO info.conf
+    , prefix: 'com.daplie.'
+    //, dbname: 'cluster'
+    , suffix: ''
+    , ext: '.sqlcipher'
+    , sock: info.conf.sqlite3Sock
+    , ipcKey: info.conf.ipcKey
+  });
+  var cstore = require('cluster-store');
 
   /*
   function unlockDevice(conf, state) {
@@ -58,6 +88,11 @@ module.exports.create = function (webserver, info) {
     res.end(metaRedirect);
   }
 
+  // TODO handle insecure to actual redirect
+  // blog.coolaj86.com -> coolaj86.com/blog
+  // hmm... that won't really matter with hsts
+  // I guess I just needs letsencrypt
+
   function scrubTheDub(req, res, next) {
     var host = req.hostname;
 
@@ -115,5 +150,55 @@ module.exports.create = function (webserver, info) {
   app.use('/', scrubTheDub);
   app.use('/', handleApi);
 
-  return app;
+  return PromiseA.all([
+    cstore.create({
+      sock: info.conf.memstoreSock
+    , connect: info.conf.memstoreSock
+      // TODO implement
+    , key: info.conf.ipcKey
+    }).then(function (_memstore) {
+      memstore = _memstore;
+      return memstore;
+    })
+    // TODO mark a device as lost, stolen, missing in DNS records
+    // (and in turn allow other devices to lock it, turn on location reporting, etc)
+  , systemFactory.create({
+        init: true
+      , dbname: 'config'
+    })
+  , clientFactory.create({
+        init: true
+      , key: '00000000000000000000000000000000'
+      // TODO only complain if the values are different
+      //, algo: 'aes'
+      , dbname: 'auth'
+    })
+  , clientFactory.create({
+        init: false
+      , dbname: 'system'
+    })
+  ]).then(function (args) {
+    memstore = args[0];
+    sqlstores.config = args[1];
+    sqlstores.auth = args[2];
+    sqlstores.system = args[3];
+    sqlstores.create = clientFactory.create;
+
+    return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) {
+      models.Config = tables;
+      models.Config.Config.get().then(function (circ) {
+      
+        /*
+          // todo getDomainInfo
+          var utils = require('./utils');
+          results.domains.forEach(function (domain) {
+            utils.getDomainInfo(domain.id);
+          });
+        */
+        console.log(circ);
+
+        return app;
+      });
+    });
+  });
 };

master.js

@@ -12,7 +12,7 @@ console.log('\n\n\n[MASTER] Welcome to WALNUT!');
 var cluster = require('cluster');
 var path = require('path');
 var minWorkers = 2;
-var numCores = Math.max(minWorkers, require('os').cpus().length);
+var numCores = 1; // Math.max(minWorkers, require('os').cpus().length);
 var workers = [];
 var caddypath = '/usr/local/bin/caddy';
 var useCaddy = require('fs').existsSync(caddypath);
@@ -75,6 +75,8 @@ cluster.on('online', function (worker) {
     require('./lib/master').touch(conf, state).then(function () {
       info.type = 'com.daplie.walnut.webserver.onrequest';
       info.conf.ipcKey = conf.ipcKey;
+      info.conf.memstoreSock = conf.memstoreSock;
+      info.conf.sqlite3Sock = conf.sqlite3Sock;
       worker.send(info);
     });
   }
@@ -93,7 +95,8 @@ cluster.on('exit', function (worker, code, signal) {
     return w;
   });
 
-  fork();
+  console.log('WARNING: worker spawning turned off for debugging ');
+  //fork();
 });
 
 fork();

package.json

@@ -88,7 +88,7 @@
     "ms": "^0.7.0",
     "negotiator": "^0.5.1",
     "node-pre-gyp": "^0.6.4",
-    "node-uuid": "1.x",
+    "node-uuid": "^1.4.4",
     "nodemailer": "^1.4.0",
     "nodemailer-mailgun-transport": "1.x",
     "oauth": "0.9.x",

tests/schemes-config.js

@@ -0,0 +1,160 @@
+'use strict';
+
+// var results = {"apis":[{"id":"oauth3-api","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"apps":[{"id":"oauth3-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"hellabit-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsio-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsconnect-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"domains":[{"id":"oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com#connect","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null}],"apisDomains":[{"id":"oauth3-api_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"apiId":"oauth3-api","domainId":"oauth3.org","json":null}],"appsDomains":[{"id":"oauth3-app_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"oauth3-app","domainId":"oauth3.org","json":null},{"id":"hellabit-app_hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"hellabit-app","domainId":"hellabit.com","json":null},{"id":"ldsio-app_lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsio-app","domainId":"lds.io","json":null},{"id":"ldsconnect-app_ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsconnect-app","domainId":"ldsconnect.org","json":null}]};
+var results = {
+  "apis":[
+    {"id":"oauth3-api"}
+  ]
+, "apps":[
+    {"id":"oauth3-app"}
+  , {"id":"hellabit-app"}
+  , {"id":"ldsio-app"}
+  , {"id":"ldsconnect-app"}
+  ]
+, "domains":[
+    {"id":"oauth3.org"}
+  , {"id":"lds.io"}
+  , {"id":"ldsconnect.org"}
+  , {"id":"hellabit.com#####"}
+  , {"id":"hellabit.com"}
+  , {"id":"hellabit.com###"}
+  , {"id":"hellabit.com#connect###"}
+  , {"id":"hellabit.com#connect"}
+  , {"id":"hellabit.com#connect#too"}
+  ]
+, "apisDomains":[
+    {"id":"oauth3-api_oauth3.org","apiId":"oauth3-api","domainId":"oauth3.org"}
+  ]
+,"appsDomains":[
+    {"id":"oauth3-app_oauth3.org","appId":"oauth3-app","domainId":"oauth3.org"}
+  , {"id":"hellabit-app_hellabit.com","appId":"hellabit-app","domainId":"hellabit.com"}
+  , {"id":"hellabit-app_hellabit.com###","appId":"hellabit-app","domainId":"hellabit.com#connect###"}
+  , {"id":"ldsio-app_lds.io","appId":"ldsio-app","domainId":"lds.io"}
+  , {"id":"ldsconnect-app_ldsconnect.org","appId":"ldsconnect-app","domainId":"ldsconnect.org"}
+  ]
+};
+
+var deserialize = require('../lib/schemes-config').deserialize;
+var getDomainInfo = require('../lib/utils').getDomainInfo;
+var config = deserialize(results);
+var req = { host: 'hellabit.com', url: '/connect' };
+var vhosts = [];
+var vhostsMap = {};
+
+function sortApps(a, b) {
+  // hlen isn't important in this current use of the sorter,
+  // but is important for an alternate version
+  var hlen = b.hostname.length - a.hostname.length;
+  var plen = b.pathname.length - a.pathname.length;
+
+  // A directory could be named example.com, example.com# example.com##
+  // to indicate order of preference (for API addons, for example)
+  var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
+
+  if (!hlen) {
+    if (!plen) {
+      return dlen;
+    }
+    return plen;
+  }
+  return hlen;
+}
+
+Object.keys(config.domains).forEach(function (domainname) {
+  var domain = config.domains[domainname];
+  var info = getDomainInfo(domainname);
+
+  domain.hostname = info.hostname;
+  domain.pathname = '/' + (info.pathname || '');
+  domain.dirname = info.dirname;
+
+  vhosts.push(domain);
+});
+
+vhosts.sort(sortApps);
+
+vhosts.forEach(function (domain) {
+  console.log(domain.hostname, domain.pathname, domain.dirname);
+
+  if (!vhostsMap[domain.hostname]) {
+    vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] };
+  }
+
+  if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) {
+    vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] };
+    vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]);
+  }
+
+  vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain);
+});
+
+if (!vhostsMap[req.host]) {
+  console.log("there's no app for this hostname");
+  return;
+}
+
+//console.log("load an app", vhosts[req.host]);
+
+//console.log(vhosts[req.host]);
+
+
+function getApp(route) {
+  var PromiseA = require('bluebird');
+
+  return new PromiseA(function (resolve, reject) {
+    console.log(route);
+    // route.hostname
+  });
+}
+
+function api(req, res, next) {
+  var apps; 
+
+  vhostsMap[req.host].pathnames.some(function (route) {
+    // /connect /
+    if (req.url.match(route.pathname) && route.pathname.match(req.url)) {
+      apps = route.apps;
+      return true;
+    }
+  });
+
+  //console.log(apps);
+
+  function nextify(err) {
+    var route;
+
+    if (err) {
+      next(err);
+      return;
+    }
+    
+    // shortest to longest
+    //route = apps.pop();
+    // longest to shortest
+    route = apps.shift();
+    if (!route) {
+      next();
+      return;
+    }
+
+    if (route.route) {
+      route.route(req, res, nextify);
+      return;
+    }
+
+    getApp(route).then(function (route) {
+      route.route = route;
+      try {
+        route.route(req, res, nextify);
+      } catch(e) {
+        console.error('[App Load Error]');
+        console.error(e.stack);
+        nextify(new Error("couldn't load app"));
+      }
+    });
+  }
+
+  nextify();
+}
+
+api(req);

walnut.js

@@ -2,6 +2,23 @@
 
 var cluster = require('cluster');
 
+var crypto;
+var stacks = {};
+Math.random = function () {
+  var err = new Error("Math.random() was used");
+
+  if (!stacks[err.stack.toString()]) {
+    stacks[err.stack.toString()] = true;
+    console.warn(err.stack);
+  }
+
+  if (!crypto) {
+    crypto = require('crypto');
+  }
+
+  return parseFloat(('0.' + (parseInt(crypto.randomBytes(8).toString('hex'), 16))).replace(/(^0)|(0$)/g, ''));
+};
+
 if (cluster.isMaster) {
   require('./master');
 } else {

worker.js

@@ -5,8 +5,8 @@ var id = cluster.worker.id.toString();
 
 function waitForInit(message) {
   if ('com.daplie.walnut.init' !== message.type) {
-    console.log('[Worker] 0 got unexpected message:');
+    console.warn('[Worker] 0 got unexpected message:');
-    console.log(message);
+    console.warn(message);
     return;
   }
 
@@ -15,7 +15,7 @@ function waitForInit(message) {
 
   require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) {
     if (err) {
-      console.log('[ERROR] worker.js');
+      console.error('[ERROR] worker.js');
       console.error(err.stack);
       throw err;
     }
@@ -26,8 +26,8 @@ function waitForInit(message) {
     return new PromiseA(function (resolve) {
       function initWebServer(srvmsg) {
         if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) {
-          console.log('[Worker] 1 got unexpected message:');
+          console.warn('[Worker] 1 got unexpected message:');
-          console.log(srvmsg);
+          console.warn(srvmsg);
           return;
         }
 
@@ -63,9 +63,10 @@ process.on('beforeExit', function (msg) {
 process.on('unhandledRejection', function (err) {
   // this should always throw
   // (it means somewhere we're not using bluebird by accident)
-  console.error('[unhandledRejection]');
+  console.error('[caught] [unhandledRejection]');
+  console.error(Object.keys(err));
+  console.error(err);
   console.error(err.stack);
-  throw err;
 });
 process.on('rejectionHandled', function (msg) {
   console.error('[rejectionHandled]');