refactoring to use fs config

bin/walnut.js

@@ -62,4 +62,7 @@ function eagerLoad() {
   });
 }
 
-setTimeout(eagerLoad, 100);
+// this isn't relevant to do in the master process, duh
+if (false) {
+  setTimeout(eagerLoad, 100);
+}

boot/local-server.js

@@ -0,0 +1,59 @@
+'use strict';
+
+// Note the odd use of callbacks (instead of promises) here
+// It's to avoid loading bluebird yet (see sni-server.js for explanation)
+module.exports.create = function (lex, certPaths, port, conf, serverCallback) {
+  function initServer(err, server) {
+    var app;
+    var promiseApp;
+
+    if (err) {
+      serverCallback(err);
+      return;
+    }
+
+    server.on('error', serverCallback);
+    server.listen(port, function () {
+      // is it even theoritically possible for
+      // a request to come in before this callback has fired?
+      // I'm assuming this event must fire before any request event
+      promiseApp = serverCallback(null, server);
+    });
+    /*
+    server.listen(port, '::::', function () {
+      // is it even theoritically possible for
+      // a request to come in before this callback has fired?
+      // I'm assuming this event must fire before any request event
+      promiseApp = serverCallback(null, server);
+    });
+    */
+
+    // Get up and listening as absolutely quickly as possible
+    function onRequest(req, res) {
+      // this is a hot piece of code, so we cache the result
+      if (app) {
+        app(req, res);
+        return;
+      }
+
+      promiseApp.then(function (_app) {
+        console.log('[Server]', req.method, req.host || req.headers['x-forwarded-host'] || req.headers.host, req.url);
+        app = _app;
+        app(req, res);
+      });
+    }
+
+    if (lex) {
+      var LEX = require('letsencrypt-express');
+      server.on('request', LEX.createAcmeResponder(lex, onRequest));
+    } else {
+      server.on('request', onRequest);
+    }
+  }
+
+  if (certPaths) {
+    require('../lib/sni-server').create(lex, certPaths, initServer);
+  } else {
+    initServer(null, require('http').createServer());
+  }
+};

boot/master.js

@@ -25,28 +25,44 @@ var workers = [];
 var state = { firstRun: true };
 // TODO Should these be configurable? If so, where?
 // TODO communicate config with environment vars?
+var walnut = tryConf(
+  path.join('..', '..', 'config.walnut')
+, { externalPort: 443
+  , externalInsecurePort: 80
+  , certspath: path.join(__dirname, '..', '..', 'certs', 'live')
+  }
+);
 var caddy = tryConf(
-  path.join('..', '..', 'config.caddy.json')
+  path.join('..', '..', 'config.caddy')
-, { conf: null        // __dirname + '/Caddyfile'
+, { conf: path.join(__dirname, '..', '..', 'Caddyfile')
   , bin: null         // '/usr/local/bin/caddy'
   , sitespath: null   // path.join(__dirname, 'sites-enabled')
   , locked: false     // true
   }
 );
-var useCaddy = require('fs').existsSync(caddy.bin);
+var letsencrypt = tryConf(
+  path.join('..', '..', 'config.letsencrypt')
+, { configDir: path.join(__dirname, '..', '..', 'letsencrypt')
+  , email: null
+  , agreeTos: false
+  }
+);
+var useCaddy = caddy.bin && require('fs').existsSync(caddy.bin);
 var info = {
   type: 'walnut.init'
 , conf: {
     protocol: useCaddy ? 'http' : 'https'
-  , externalPort: 443
+  , externalPort: walnut.externalPort
-  , externalPortInsecure: 80 // TODO externalInsecurePort
+  , externalPortInsecure: walnut.externalInsecurePort         // TODO externalInsecurePort
-  , localPort: process.argv[2] || (useCaddy ? 4080 : 443) // system / local network
+  , localPort: walnut.localPort || (useCaddy ? 4080 : 443)    // system / local network
-  , insecurePort: process.argv[3] || (useCaddy ? 80 : 80) // meh
+  , insecurePort: walnut.insecurePort || (useCaddy ? 80 : 80) // meh
   , certPaths: useCaddy ? null : [
-      path.join(__dirname, '..', '..', 'certs', 'live')
+      walnut.certspath
-    , path.join(__dirname, '..', '..', 'letsencrypt', 'live')
+    , path.join(letsencrypt.configDir, 'live')
     ]
   , trustProxy: useCaddy ? true : false
+  , lexConf: letsencrypt
+  , varpath: path.join(__dirname, '..', '..', 'var')
   }
 };
 
@@ -67,6 +83,7 @@ cluster.on('online', function (worker) {
       // relies on { localPort, locked }
       caddy.spawn(caddy);
     }
+    // TODO dyndns in master?
   }
 
   function touchMaster(msg) {
@@ -76,47 +93,11 @@ cluster.on('online', function (worker) {
       return;
     }
 
-    // calls init if init has not been called
     state.caddy = caddy;
     state.workers = workers;
-    require('../lib/master').touch(info.conf, state).then(function (results) {
+    // calls init if init has not been called
-      //var memstore = results.memstore;
+    require('../lib/master').touch(info.conf, state).then(function (newConf) {
-      var sqlstore = results.sqlstore;
+      worker.send({ type: 'walnut.webserver.onrequest', conf: newConf });
-      info.type = 'walnut.webserver.onrequest';
-      // TODO let this load after server is listening
-      info.conf['org.oauth3.consumer'] = results['org.oauth3.consumer'];
-      info.conf['org.oauth3.provider'] = results['org.oauth3.provider'];
-      info.conf.keys = results.keys;
-      //info.conf.memstoreSock = config.memstoreSock;
-      //info.conf.sqlite3Sock = config.sqlite3Sock;
-      // TODO get this from db config instead
-      //info.conf.privkey = config.privkey;
-      //info.conf.pubkey = config.pubkey;
-      info.conf.redirects = [
-        { "ip": false, "id": "*", "value": false } // default no-www
-
-      , { "ip": false, "id": "daplie.domains", "value": null }
-      , { "ip": false, "id": "*.daplie.domains", "value": false }
-      , { "ip": false, "id": "no.daplie.domains", "value": false }
-      , { "ip": false, "id": "*.no.daplie.domains", "value": false }
-      , { "ip": false, "id": "ns2.daplie.domains", "value": false }
-
-      , { "ip": true, "id": "maybe.daplie.domains", "value": null }
-      , { "ip": true, "id": "*.maybe.daplie.domains", "value": null }
-
-      , { "ip": true, "id": "www.daplie.domains", "value": null }
-      , { "ip": true, "id": "yes.daplie.domains", "value": true }
-      , { "ip": true, "id": "*.yes.daplie.domains", "value": true }
-      , { "ip": true, "id": "ns1.daplie.domains", "value": false }
-      ];
-      // TODO use sqlite3 or autogenerate ?
-      info.conf.privkey = require('fs').readFileSync(__dirname + '/../../' + '/nsx.redirect-www.org.key.pem', 'ascii');
-      info.conf.pubkey = require('fs').readFileSync(__dirname + '/../../' + '/nsx.redirect-www.org.key.pem.pub', 'ascii');
-      // keys
-      // letsencrypt
-      // com.example.provider
-      // com.example.consumer
-      worker.send(info);
     });
   }
 

boot/worker.js

@@ -4,25 +4,93 @@ module.exports.create = function (opts) {
   var id = '0';
   var promiseApp;
 
-  function createAndBindInsecure(lex, message, cb) {
+  function createAndBindInsecure(lex, conf, getOrCreateHttpApp) {
     // TODO conditional if 80 is being served by caddy
-    require('../lib/insecure-server').create(lex, message.conf.externalPort, message.conf.insecurePort, message, function (err, webserver) {
-      console.info("#" + id + " Listening on http://" + webserver.address().address + ":" + webserver.address().port, '\n');
 
-      // we are returning the promise result to the caller
+    var appPromise = null;
-      return cb(null, webserver, null, message);
+    var app = null;
+    var http = require('http');
+    var insecureServer = http.createServer();
+
+    function onRequest(req, res) {
+      if (app) {
+        app(req, res);
+        return;
+      }
+
+      if (!appPromise) {
+        res.setHeader('Content-Type', 'application/json; charset=utf-8');
+        res.end('{ "error": { "code": "E_SANITY_FAIL", "message": "should have an express app, but didn\'t" } }');
+        return;
+      }
+
+      appPromise.then(function (_app) {
+        appPromise = null;
+        app = _app;
+        app(req, res);
+      });
+    }
+
+    insecureServer.listen(conf.insecurePort, function () {
+      console.info("#" + id + " Listening on http://"
+        + insecureServer.address().address + ":" + insecureServer.address().port, '\n');
+      appPromise = getOrCreateHttpApp(null, insecureServer);
+
+      if (!appPromise) {
+        throw new Error('appPromise returned nothing');
+      }
+    });
+
+    insecureServer.on('request', onRequest);
+  }
+
+  function walkLe(domainname) {
+    var PromiseA = require('bluebird');
+    var fs = PromiseA.promisifyAll(require('fs'));
+    var path = require('path');
+    var parts = domainname.split('.'); //.replace(/^www\./, '').split('.');
+    var configname = parts.join('.') + '.json';
+    var configpath = path.join(__dirname, '..', '..', 'config', configname);
+
+    if (parts.length < 2) {
+      return PromiseA.resolve(null);
+    }
+
+    // TODO configpath a la varpath
+    return fs.readFileAsync(configpath, 'utf8').then(function (text) {
+      var data = JSON.parse(text);
+      data.name = configname;
+      return data;
+    }, function (/*err*/) {
+      parts.shift();
+      return walkLe(parts.join('.'));
     });
   }
 
-  function createLe(conf) {
+  function createLe(lexConf, conf) {
     var LEX = require('letsencrypt-express');
     var lex = LEX.create({
-      configDir: conf.letsencrypt.configDir // i.e. __dirname + '/letsencrypt.config'
+      configDir: lexConf.configDir // i.e. __dirname + '/letsencrypt.config'
     , approveRegistration: function (hostname, cb) {
-        cb(null, {
+        // TODO cache/report unauthorized
-          domains: [hostname]                 // TODO handle www and bare on the same cert
+        if (!hostname) {
-        , email: conf.letsencrypt.email
+          cb(new Error("[lex.approveRegistration] undefined hostname"), null);
-        , agreeTos: conf.letsencrypt.agreeTos
+          return;
+        }
+
+        walkLe(hostname).then(function (leAuth) {
+          // TODO should still check dns for hostname (and mx for email)
+          if (leAuth && leAuth.email && leAuth.agreeTos) {
+            cb(null, {
+              domains: [hostname]                 // TODO handle www and bare on the same cert
+            , email: leAuth.email
+            , agreeTos: leAuth.agreeTos
+            });
+          }
+          else {
+            // TODO report unauthorized
+            cb(new Error("Valid LetsEncrypt config with email and agreeTos not found for '" + hostname + "'"), null);
+          }
         });
         /*
         letsencrypt.getConfig({ domains: [domain] }, function (err, config) {
@@ -42,81 +110,92 @@ module.exports.create = function (opts) {
         */
       }
     });
-    //var letsencrypt = lex.letsencrypt;
+    conf.letsencrypt = lex.letsencrypt;
+    conf.lex = lex;
+    conf.walkLe = walkLe;
 
     return lex;
   }
 
-  function createAndBindServers(message, cb) {
+  function createAndBindServers(conf, getOrCreateHttpApp) {
     var lex;
 
-    if (message.conf.letsencrypt) {
+    if (conf.lexConf) {
-      lex = createLe(message.conf);
+      lex = createLe(conf.lexConf, conf);
     }
 
     // NOTE that message.conf[x] will be overwritten when the next message comes in
-    require('../lib/local-server').create(lex, message.conf.certPaths, message.conf.localPort, message, function (err, webserver) {
+    require('./local-server').create(lex, conf.certPaths, conf.localPort, conf, function (err, webserver) {
       if (err) {
         console.error('[ERROR] worker.js');
         console.error(err.stack);
         throw err;
       }
 
-      console.info("#" + id + " Listening on " + message.conf.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
+      console.info("#" + id + " Listening on " + conf.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
 
       // we don't need time to pass, just to be able to return
       process.nextTick(function () {
-        createAndBindInsecure(lex, message, cb);
+        createAndBindInsecure(lex, conf, getOrCreateHttpApp);
       });
 
       // we are returning the promise result to the caller
-      return cb(null, null, webserver, message);
+      return getOrCreateHttpApp(null, null, webserver, conf);
     });
   }
 
   //
   // Worker Mode
   //
-  function waitForConfig(message) {
+  function waitForConfig(realMessage) {
-    if ('walnut.init' !== message.type) {
+    if ('walnut.init' !== realMessage.type) {
       console.warn('[Worker] 0 got unexpected message:');
-      console.warn(message);
+      console.warn(realMessage);
       return;
     }
 
+    var conf = realMessage.conf;
     process.removeListener('message', waitForConfig);
 
     // NOTE: this callback must return a promise for an express app
-    createAndBindServers(message, function (err, insecserver, webserver, oldMessage) {
+
-      // TODO deep merge new message into old message
+    function getExpressApp(err, insecserver, webserver/*, newMessage*/) {
-      Object.keys(message.conf).forEach(function (key) {
-        oldMessage.conf[key] = message.conf[key];
-      });
       var PromiseA = require('bluebird');
+
       if (promiseApp) {
         return promiseApp;
       }
+
       promiseApp = new PromiseA(function (resolve) {
-        function initWebServer(srvmsg) {
+        function initHttpApp(srvmsg) {
           if ('walnut.webserver.onrequest' !== srvmsg.type) {
-            console.warn('[Worker] 1 got unexpected message:');
+            console.warn('[Worker] [onrequest] unexpected message:');
             console.warn(srvmsg);
             return;
           }
 
-          process.removeListener('message', initWebServer);
+          process.removeListener('message', initHttpApp);
 
-          resolve(require('../lib/worker').create(webserver, srvmsg.conf));
+          if (srvmsg.conf) {
+            Object.keys(srvmsg.conf).forEach(function (key) {
+              conf[key] = srvmsg.conf[key];
+            });
+          }
+
+          resolve(require('../lib/worker').create(webserver, conf));
         }
 
         process.send({ type: 'walnut.webserver.listening' });
-        process.on('message', initWebServer);
+        process.on('message', initHttpApp);
       }).then(function (app) {
         console.info('[Worker Ready]');
         return app;
       });
+
       return promiseApp;
-    });
+    }
+
+    createAndBindServers(conf, getExpressApp);
   }
 
   //
@@ -124,11 +203,13 @@ module.exports.create = function (opts) {
   //
   if (opts) {
     // NOTE: this callback must return a promise for an express app
-    createAndBindServers(opts, function (err, insecserver, webserver/*, message*/) {
+    createAndBindServers(opts, function (err, insecserver, webserver/*, conf*/) {
       var PromiseA = require('bluebird');
+
       if (promiseApp) {
         return promiseApp;
       }
+
       promiseApp = new PromiseA(function (resolve) {
         opts.getConfig(function (srvmsg) {
           resolve(require('../lib/worker').create(webserver, srvmsg));
@@ -137,6 +218,7 @@ module.exports.create = function (opts) {
         console.info('[Standalone Ready]');
         return app;
       });
+
       return promiseApp;
     });
   } else {

install.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sudo mkdir -p /srv/walnut/{certs,core,letsencrypt,lib}
+sudo mkdir -p /srv/walnut/{certs,core,letsencrypt,lib,config}
 sudo mkdir -p /srv/walnut/packages/{api,pages,services}
 sudo chown -R $(whoami):$(whoami) /srv/walnut
 
@@ -9,11 +9,23 @@ git clone https://github.com/Daplie/walnut.git /srv/walnut/core
 
 pushd /srv/walnut/core
 npm install
-sudo rsync -av /srv/walnut/core/etc/init/walnut.conf /etc/init/walnut.conf
-rsync -av /srv/walnut/core/etc/letsencrypt/ /srv/walnut/certs/
-
 popd
+
+sudo rsync -a /srv/walnut/core/etc/init/walnut.conf /etc/init/walnut.conf
+rsync -a /srv/walnut/core/etc/letsencrypt/ /srv/walnut/certs/
 mv /srv/walnut/core/node_modules /srv/walnut
 
+echo -n "Enter an email address to use for LetsEncrypt and press [ENTER]: "
+read LE_EMAIL
+node -e "
+  'use strict';
+
+  require('fs').writeFileSync('/srv/walnut/config.letsencrypt.json', JSON.stringify({
+    configDir: '/srv/walnut/letsencrypt'
+  , email: '$LE_EMAIL'
+  , agreeTos: true
+  }, null, '  '));
+"
+
 sudo service walnut stop
 sudo service walnut start

lib/apis.js

@@ -0,0 +1,250 @@
+'use strict';
+
+module.exports.create = function (xconfx, apiFactories, apiDeps) {
+  var PromiseA = apiDeps.Promise;
+  var express = require('express');
+  var fs = PromiseA.promisifyAll(require('fs'));
+  var path = require('path');
+  var localCache = { apis: {}, pkgs: {} };
+
+  // TODO xconfx.apispath
+  xconfx.apispath = path.join(__dirname, '..', '..', 'packages', 'apis');
+
+  function notConfigured(req, res) {
+    res.send({ error: { message: "api '" + req.apiId + "' not configured for domain '" + req.experienceId + "'" } });
+  }
+
+  function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
+    function handlePromise(p) {
+      return p.then(function (api) {
+        packagedApi._api = api;
+        return api;
+      });
+    }
+
+    if (!packagedApi._promise_api) {
+      packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
+    }
+
+    return handlePromise(packagedApi._promise_api);
+  }
+
+  function getApi(conf, pkgConf, pkgDeps, packagedApi) {
+    var PromiseA = pkgDeps.Promise;
+    var path = require('path');
+    var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
+
+    // TODO needs some version stuff (which would also allow hot-loading of updates)
+    // TODO version could be tied to sha256sum
+
+    return new PromiseA(function (resolve, reject) {
+      var myApp;
+      var ursa;
+      var promise;
+
+      // TODO dynamic requires are a no-no
+      // can we statically generate a require-er? on each install?
+      // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
+      // requirer[pkgpath]()
+      myApp = pkgDeps.express();
+      myApp.disable('x-powered-by');
+      if (pkgDeps.app.get('trust proxy')) {
+        myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
+      }
+      if (!pkgConf.pubkey) {
+        /*
+          return ursa.createPrivateKey(pem, password, encoding);
+          var pem = myKey.toPrivatePem();
+          return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
+          });
+        */
+        ursa = require('ursa');
+        pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
+        pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
+      }
+
+      try {
+        packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
+        packagedApi._apiname = packagedApi._apipkg.name;
+        if (packagedApi._apipkg.walnut) {
+          pkgpath += '/' + packagedApi._apipkg.walnut;
+        }
+        promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
+      } catch(e) {
+        reject(e);
+        return;
+      }
+
+      promise.then(function () {
+        // TODO give pub/priv pair for app and all public keys
+        // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
+        packagedApi._api = require('express-lazy')();
+        packagedApi._api_app = myApp;
+
+        //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
+        pkgDeps.getOauth3Controllers =
+        packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
+        require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
+
+        // DEBUG
+        //
+        /*
+        packagedApi._api.use('/', function (req, res, next) {
+          console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
+          next();
+        });
+        //*/
+
+        // TODO fix backwards compat
+
+        // /api/com.example.foo (no change)
+        packagedApi._api.use('/', packagedApi._api_app);
+
+        // /api/com.example.foo => /api
+        packagedApi._api.use('/', function (req, res, next) {
+          var priorUrl = req.url;
+          req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
+          // console.log('api mangle 3:', req.url);
+          packagedApi._api_app(req, res, function (err) {
+            req.url = priorUrl;
+            next(err);
+          });
+        });
+
+        // /api/com.example.foo => /
+        packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
+          // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
+          // console.log(packagedApi._api_app.toString());
+          packagedApi._api_app(req, res, next);
+        });
+
+        resolve(packagedApi._api);
+      }, reject);
+    });
+  }
+
+  // Read packages/apis/sub.sld.tld (forward dns) to find list of apis as tld.sld.sub (reverse dns)
+  // TODO packages/allowed_apis/sub.sld.tld (?)
+  // TODO auto-register org.oauth3.consumer for primaryDomain (and all sites?)
+  function loadApiHandler() {
+    return function handler(req, res, next) {
+      var name = req.experienceId;
+      var apiId = req.apiId;
+      var packagepath = path.join(xconfx.apispath, name);
+
+      return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
+        return text.trim().split(/\n/);
+      }, function () {
+        return [];
+      }).then(function (apis) {
+        return function (req, res, next) {
+          var apipath;
+
+          if (!apis.some(function (api) {
+            if (api === apiId) {
+              return true;
+            }
+          })) {
+            if (req.experienceId === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === apiId) {
+              // fallthrough
+            } else {
+              return null;
+            }
+          }
+
+          apipath = path.join(xconfx.apispath, apiId);
+
+          if (!localCache.pkgs[apiId]) {
+            return fs.readFileAsync(path.join(apipath, 'package.json'), 'utf8').then(function (text) {
+              var pkg = JSON.parse(text);
+              var deps = {};
+              var myApp;
+
+              if (pkg.walnut) {
+                apipath = path.join(apipath, pkg.walnut);
+              }
+
+              Object.keys(apiDeps).forEach(function (key) {
+                deps[key] = apiDeps[key];
+              });
+              Object.keys(apiFactories).forEach(function (key) {
+                deps[key] = apiFactories[key];
+              });
+
+              // TODO pull db stuff from package.json somehow and pass allowed data models as deps
+              //
+              // how can we tell which of these would be correct?
+              // deps.memstore = apiFactories.memstoreFactory.create(apiId);
+              // deps.memstore = apiFactories.memstoreFactory.create(req.experienceId);
+              // deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + apiId);
+
+              // let's go with this one for now and the api can choose to scope or not to scope
+              deps.memstore = apiFactories.memstoreFactory.create(apiId);
+
+              console.log('DEBUG apipath', apipath);
+              myApp = express();
+              //
+              // TODO handle /accounts/:accountId
+              //
+              return PromiseA.resolve(require(apipath).create({}/*pkgConf*/, deps/*pkgDeps*/, myApp/*myApp*/)).then(function (handler) {
+                localCache.pkgs[apiId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
+                localCache.pkgs[apiId].handler(req, res, next);
+              });
+            });
+          }
+          else {
+            localCache.pkgs[apiId].handler(req, res, next);
+            // TODO expire require cache
+            /*
+            if (Date.now() - localCache.pkgs[apiId].createdAt < (5 * 60 * 1000)) {
+              return;
+            }
+            */
+          }
+        };
+      }, function (/*err*/) {
+        return null;
+      }).then(function (handler) {
+
+        // keep object reference intact
+        // DO NOT cache non-existant api
+        if (handler) {
+          localCache.apis[name].handler = handler;
+        } else {
+          handler = notConfigured;
+        }
+        handler(req, res, next);
+      });
+    };
+  }
+
+  return function (req, res, next) {
+    var experienceId = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
+    var apiId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
+
+    Object.defineProperty(req, 'experienceId', {
+      enumerable: true
+    , configurable: false
+    , writable: false
+      // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
+      // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
+      // NOTE: probably best to alias the name logically
+    , value: experienceId
+    });
+    Object.defineProperty(req, 'apiId', {
+      enumerable: true
+    , configurable: false
+    , writable: false
+    , value: apiId
+    });
+
+    if (!localCache.apis[experienceId]) {
+      localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
+    }
+
+    localCache.apis[experienceId].handler(req, res, next);
+    if (Date.now() - localCache.apis[experienceId].createdAt > (5 * 60 * 1000)) {
+      localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
+    }
+  };
+};

lib/bootstrap.js

@@ -0,0 +1,306 @@
+'use strict';
+
+//
+// IMPORTANT !!!
+//
+// None of this is authenticated or encrypted
+//
+
+module.exports.create = function (app, xconfx, models) {
+  var PromiseA = require('bluebird');
+  var path = require('path');
+  var fs = PromiseA.promisifyAll(require('fs'));
+  var dns = PromiseA.promisifyAll(require('dns'));
+
+  function isInitialized() {
+    // TODO read from file only, not db
+    return models.ComDaplieWalnutConfig.get('config').then(function (conf) {
+      if (!conf || !conf.primaryDomain || !conf.primaryEmail) {
+        console.log('DEBUG incomplete conf', conf);
+        return false;
+      }
+
+      xconfx.primaryDomain = xconfx.primaryDomain || conf.primaryDomain;
+
+      var configname = conf.primaryDomain + '.json';
+      var configpath = path.join(__dirname, '..', '..', 'config', configname);
+
+      return fs.readFileAsync(configpath, 'utf8').then(function (text) {
+        return JSON.parse(text);
+      }, function (/*err*/) {
+        console.log('DEBUG not exists leconf', configpath);
+        return false;
+      }).then(function (data) {
+        if (!data || !data.email || !data.agreeTos) {
+          console.log('DEBUG incomplete leconf', data);
+          return false;
+        }
+
+        return true;
+      });
+    });
+  }
+
+  function initialize() {
+    var express = require('express');
+    var getIpAddresses = require('./ip-checker').getExternalAddresses;
+    var resolve;
+
+    function errorIfNotApi(req, res, next) {
+      // if it's not an ip address
+      if (/[a-z]+/.test(req.headers.host)) {
+        if (!/^api\./.test(req.headers.host)) {
+          console.log('req.headers.host');
+          console.log(req.headers.host);
+          res.send({ error: { message: "no api. subdomain prefix" } });
+          return;
+        }
+      }
+
+      next();
+    }
+
+    function errorIfApi(req, res, next) {
+      if (!/^api\./.test(req.headers.host)) {
+        next();
+        return;
+      }
+
+      // has api. hostname prefix
+
+      // doesn't have /api url prefix
+      if (!/^\/api\//.test(req.url)) {
+        res.send({ error: { message: "missing /api/ url prefix" } });
+        return;
+      }
+
+      res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
+    }
+
+    function getConfig(req, res) {
+      getIpAddresses().then(function (inets) {
+        var results = {
+          hostname: require('os').hostname()
+        , inets: inets.addresses.map(function (a) {
+            a.time = undefined;
+            return a;
+          })
+        };
+        //res.send({ inets: require('os').networkInterfaces() });
+        res.send(results);
+      });
+    }
+
+    function verifyIps(inets, hostname) {
+      var map = {};
+      var arr = [];
+
+      inets.forEach(function (addr) {
+        if (!map[addr.family]) {
+          map[addr.family] = true;
+          if (4 === addr.family) {
+            arr.push(dns.resolve4Async(hostname).then(function (arr) {
+              return arr;
+            }, function (/*err*/) {
+              return [];
+            }));
+          }
+          if (6 === addr.family) {
+            arr.push(dns.resolve6Async(hostname).then(function (arr) {
+              return arr;
+            }, function (/*err*/) {
+              return [];
+            }));
+          }
+        }
+      });
+
+      return PromiseA.all(arr).then(function (fams) {
+        console.log('DEBUG hostname', hostname);
+        var ips = [];
+
+        fams.forEach(function (addrs) {
+          console.log('DEBUG ipv46');
+          console.log(addrs);
+          addrs.forEach(function (addr) {
+            inets.forEach(function (a) {
+              if (a.address === addr) {
+                a.time = undefined;
+                ips.push(a);
+              }
+            });
+          });
+          console.log('');
+        });
+
+        return ips;
+      });
+    }
+
+    function setConfig(req, res) {
+      var config = req.body;
+      var results = {};
+
+      return PromiseA.resolve().then(function () {
+        if (!config.agreeTos && !config.tls) {
+          return PromiseA.reject(new Error("To enable encryption you must agree to the LetsEncrypt terms of service"));
+        }
+
+        if (!config.domain) {
+          return PromiseA.reject(new Error("You must specify a valid domain name"));
+        }
+        config.domain = config.domain.replace(/^www\./, '');
+
+        return getIpAddresses().then(function (inet) {
+          if (!inet.addresses.length) {
+            return PromiseA.reject(new Error("no ip addresses"));
+          }
+
+          results.inets = inet.addresses.map(function (a) {
+            a.time = undefined;
+            return a;
+          });
+
+          results.resolutions = [];
+          return PromiseA.all([
+            // for static content
+            verifyIps(inet.addresses, config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: config.domain, ips: ips });
+            })
+            // for redirects
+          , verifyIps(inet.addresses, 'www.' + config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: 'www.' + config.domain, ips: ips });
+            })
+            // for api
+          , verifyIps(inet.addresses, 'api.' + config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: 'api.' + config.domain, ips: ips });
+            })
+            // for protected assets
+          , verifyIps(inet.addresses, 'assets.' + config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: 'assets.' + config.domain, ips: ips });
+            })
+            // for the cloud management
+          , verifyIps(inet.addresses, 'cloud.' + config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: 'cloud.' + config.domain, ips: ips });
+            })
+          , verifyIps(inet.addresses, 'api.cloud.' + config.domain).then(function (ips) {
+              results.resolutions.push({ hostname: 'api.cloud.' + config.domain, ips: ips });
+            })
+          ]).then(function () {
+            if (!results.resolutions[0].ips.length) {
+              results.error = { message: "bare domain could not be resolved to this device" };
+            }
+            else if (!results.resolutions[2].ips.length) {
+              results.error = { message: "api subdomain could not be resolved to this device" };
+            }
+            /*
+            else if (!results.resolutions[1].ips.length) {
+              results.error = { message: "" }
+            }
+            else if (!results.resolutions[3].ips.length) {
+              results.error = { message: "" }
+            }
+            else if (!results.resolutions[4].ips.length || !results.resolutions[4].ips.length) {
+              results.error = { message: "cloud and api.cloud subdomains should be set up" };
+            }
+            */
+          });
+        });
+      }).then(function () {
+        if (results.error) {
+          return;
+        }
+
+        var configname = config.domain + '.json';
+        var configpath = path.join(__dirname, '..', '..', 'config', configname);
+        var leAuth = {
+          agreeTos: true
+        , email: config.email // TODO check email
+        , domain: config.domain
+        , createdAt: Date.now()
+        };
+
+        return dns.resolveMxAsync(config.email.replace(/.*@/, '')).then(function (/*addrs*/) {
+          // TODO allow private key to be uploaded
+          return fs.writeFileAsync(configpath, JSON.stringify(leAuth, null, '  '), 'utf8').then(function () {
+            return models.ComDaplieWalnutConfig.upsert('config', {
+              letsencrypt: leAuth
+            , primaryDomain: config.domain
+            , primaryEmail: config.email
+            });
+          });
+        }, function () {
+          return PromiseA.reject(new Error("invalid email address (MX record lookup failed)"));
+        });
+      }).then(function () {
+        if (!results.error && results.inets && resolve) {
+          resolve();
+          resolve = null;
+        }
+        res.send(results);
+      }, function (err) {
+        console.error('Error lib/bootstrap.js');
+        console.error(err.stack || err);
+        res.send({ error: { message: err.message || err.toString() } });
+      });
+    }
+
+    var CORS = require('connect-cors');
+    var cors = CORS({ credentials: true, headers: [
+      'X-Requested-With'
+    , 'X-HTTP-Method-Override'
+    , 'Content-Type'
+    , 'Accept'
+    , 'Authorization'
+    ], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
+
+    app.use('/', function (req, res, next) {
+      return isInitialized().then(function (initialized) {
+        if (!initialized) {
+          next();
+          return;
+        }
+
+        resolve(true);
+
+        // force page refresh
+        // TODO goto top of routes?
+        res.statusCode = 302;
+        res.setHeader('Location', req.url);
+        res.end();
+      });
+    });
+    app.use('/api', errorIfNotApi);
+    // NOTE Allows CORS access to API with ?access_token=
+    // TODO Access-Control-Max-Age: 600
+    // TODO How can we help apps handle this? token?
+    // TODO allow apps to configure trustedDomains, auth, etc
+    app.use('/api', cors);
+    app.get('/api/com.daplie.walnut.init', getConfig);
+    app.post('/api/com.daplie.walnut.init', setConfig);
+    app.use('/', errorIfApi);
+    app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'com.daplie.walnut.init')));
+
+    return new PromiseA(function (_resolve) {
+      resolve = _resolve;
+    });
+  }
+
+  return isInitialized().then(function (initialized) {
+    if (initialized) {
+      return true;
+    }
+
+    return initialize();
+  }, function (err) {
+    console.error('FATAL ERROR:');
+    console.error(err.stack || err);
+    app.use('/', function (req, res) {
+      res.send({
+        error: {
+          message: "Unrecoverable Error Requires manual server update: " + (err.message || err.toString())
+        }
+      });
+    });
+  });
+};

lib/insecure-server.js

@@ -1,113 +0,0 @@
-'use strict';
-
-module.exports.create = function (lex, securePort, insecurePort, info, serverCallback) {
-  var PromiseA = require('bluebird').Promise;
-  var appPromise;
-  //var app;
-  var http = require('http');
-  var redirectives;
-
-  function useAppInsecurely(req, res) {
-    if (!appPromise) {
-      return false;
-    }
-
-    appPromise.then(function (app) {
-      req._WALNUT_SECURITY_EXCEPTION = true;
-      app(req, res);
-    });
-
-    return true;
-  }
-
-  function redirectHttps(req, res) {
-    if (req.headers.host && /^\/.well-known\/acme-challenge/.test(req.url) && useAppInsecurely(req, res)) {
-      return true;
-    }
-    // TODO
-    // XXX NOTE: info.conf.redirects may or may not be loaded at first
-    // the object will be modified when the config is loaded
-    if (!redirectives && info.redirects || info.conf.redirects) {
-      redirectives = require('./hostname-redirects').compile(info.redirects || info.conf.redirects);
-    }
-    if (require('./no-www').scrubTheDub(req, res, redirectives)) {
-      return true;
-    }
-
-    // Let it do this once they visit the https site
-    // res.setHeader('Strict-Transport-Security', 'max-age=10886400; includeSubDomains; preload');
-
-    var host = req.headers.host || '';
-    var url = req.url;
-
-    // TODO
-    // allow exceptions for the case of arduino and whatnot that cannot handle https?
-    // http://evothings.com/is-it-possible-to-secure-micro-controllers-used-within-iot/
-    // needs ECDSA?
-
-    var escapeHtml = require('escape-html');
-    var newLocation = 'https://'
-      + host.replace(/:\d+/, ':' + securePort) + url
-      ;
-    var safeLocation = escapeHtml(newLocation);
-
-    var metaRedirect = ''
-      + '<html>\n'
-      + '<head>\n'
-      + '  <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
-      + '  <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
-      + '</head>\n'
-      + '<body style="display: none;">\n'
-      + '  <p>You requested an insecure resource. Please use this instead: \n'
-      + '    <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
-      + '</body>\n'
-      + '</html>\n'
-      ;
-
-    // DO NOT HTTP REDIRECT
-    /*
-    res.setHeader('Location', newLocation);
-    res.statusCode = 302;
-    */
-
-    // BAD NEWS BEARS
-    //
-    // When people are experimenting with the API and posting tutorials
-    // they'll use cURL and they'll forget to prefix with https://
-    // If we allow that, then many users will be sending private tokens
-    // and such with POSTs in clear text and, worse, it will work!
-    // To minimize this, we give browser users a mostly optimal experience,
-    // but people experimenting with the API get a message letting them know
-    // that they're doing it wrong and thus forces them to ensure they encrypt.
-    res.setHeader('Content-Type', 'text/html; charset=utf-8');
-    res.end(metaRedirect);
-  }
-
-  // TODO localhost-only server shutdown mechanism
-  // that closes all sockets, waits for them to finish,
-  // and then hands control over completely to respawned server
-
-  //
-  // Redirect HTTP to HTTPS
-  //
-  // This simply redirects from the current insecure location to the encrypted location
-  //
-  var insecureServer;
-  insecureServer = http.createServer();
-  insecureServer.listen(insecurePort, function () {
-    console.log("\nListening on http://localhost:" + insecureServer.address().port);
-    console.log("(handling any explicit redirects and redirecting all other traffic to https)\n");
-    if (serverCallback) {
-      appPromise = serverCallback(null, insecureServer);
-    }
-  });
-
-  if (lex) {
-    var LEX = require('letsencrypt-express');
-    insecureServer.on('request', LEX.createAcmeResponder(lex, redirectHttps));
-  } else {
-    insecureServer.on('request', redirectHttps);
-  }
-
-  return PromiseA.resolve(insecureServer);
-};

lib/ip-checker.js

@@ -0,0 +1,138 @@
+"use strict";
+
+var PromiseA = require('bluebird').Promise;
+var ifaces = require('os').networkInterfaces();
+var dns = PromiseA.promisifyAll(require('dns'));
+var https = require('https');
+
+function getExternalAddresses() {
+  var iftypes = {};
+  var ipv4check = 'api.ipify.org';
+  var ipv6check = 'myexternalip.com';
+
+  Object.keys(ifaces).forEach(function (ifname) {
+    ifaces[ifname].forEach(function (iface) {
+      // local addresses
+      if (iface.internal) {
+        return;
+      }
+      // auto address space
+      if (/^(fe80:|169\.)/.test(iface.address)) {
+        return;
+      }
+      /*
+      if (/^(fe80:|10\.|192\.168|172\.1[6-9]|172\.2[0-9]|172\.3[0-1])/.test(iface.address)) {
+        return;
+      }
+      */
+
+      iftypes[iface.family] = true;
+    });
+  });
+
+  console.log(iftypes);
+
+  var now = Date.now();
+
+  return PromiseA.all([
+    dns.lookupAsync(ipv4check, { family: 4/*, all: true*/ }).then(function (ans) {
+      iftypes.IPv4 = { address: ans[0], family: ans[1], time: Date.now() - now };
+    }).error(function () {
+      //console.log('no ipv4', Date.now() - now);
+      iftypes.IPv4 = false;
+    })
+    // curl -6 https://myexternalip.com/raw
+  , dns.lookupAsync(ipv6check, { family: 6/*, all: true*/ }).then(function (ans) {
+      iftypes.IPv6 = { address: ans[0], family: ans[1], time: Date.now() - now };
+    }).error(function (err) {
+      console.error('Error ip-checker.js');
+      console.error(err.stack || err);
+      //console.log('no ipv6', Date.now() - now);
+      iftypes.IPv6 = false;
+    })
+  ]).then(function () {
+    var requests = [];
+
+    if (iftypes.IPv4) {
+      requests.push(new PromiseA(function (resolve)  {
+        var req = https.request({
+          method: 'GET'
+        , hostname: iftypes.IPv4.address
+        , port: 443
+        , headers: {
+            Host: ipv4check
+          }
+        , path: '/'
+        //, family: 4
+        // TODO , localAddress: <<external_ipv4>>
+        }, function (res) {
+          var result = '';
+
+          res.on('error', function (/*err*/) {
+            resolve(null);
+          });
+
+          res.on('data', function (chunk) {
+            result += chunk.toString('utf8');
+          });
+
+          res.on('end', function () {
+            resolve({ address: result, family: 4/*, wan: result === iftypes.IPv4.localAddress*/, time: iftypes.IPv4.time });
+          });
+        });
+
+        req.on('error', function () {
+          resolve(null);
+        });
+        req.end();
+      }));
+    }
+
+    if (iftypes.IPv6) {
+      requests.push(new PromiseA(function (resolve)  {
+        var req = https.request({
+          method: 'GET'
+        , hostname: iftypes.IPv6.address
+        , port: 443
+        , headers: {
+            Host: ipv6check
+          }
+        , path: '/raw'
+        //, family: 6
+        // TODO , localAddress: <<external_ipv6>>
+        }, function (res) {
+          var result = '';
+
+          res.on('error', function (/*err*/) {
+            resolve(null);
+          });
+
+          res.on('data', function (chunk) {
+            result += chunk.toString('utf8').trim();
+          });
+          res.on('end', function () {
+            resolve({ address: result, family: 6/*, wan: result === iftypes.IPv6.localAaddress*/, time: iftypes.IPv4.time });
+          });
+        });
+
+        req.on('error', function () {
+          resolve(null);
+        });
+        req.end();
+      }));
+    }
+
+    return PromiseA.all(requests).then(function (ips) {
+      ips = ips.filter(function (ip) {
+        return ip;
+      });
+
+      return {
+        addresses: ips
+      , time: Date.now() - now
+      };
+    });
+  });
+}
+
+exports.getExternalAddresses = getExternalAddresses;

lib/local-server.js

@@ -2,7 +2,7 @@
 
 // Note the odd use of callbacks (instead of promises) here
 // It's to avoid loading bluebird yet (see sni-server.js for explanation)
-module.exports.create = function (lex, certPaths, port, info, serverCallback) {
+module.exports.create = function (certPaths, port, info, serverCallback) {
   function initServer(err, server) {
     var app;
     var promiseApp;
@@ -29,7 +29,7 @@ module.exports.create = function (lex, certPaths, port, info, serverCallback) {
     */
 
     // Get up and listening as absolutely quickly as possible
-    function onRequest(req, res) {
+    server.on('request', function (req, res) {
       // this is a hot piece of code, so we cache the result
       if (app) {
         app(req, res);
@@ -41,18 +41,11 @@ module.exports.create = function (lex, certPaths, port, info, serverCallback) {
         app = _app;
         app(req, res);
       });
-    }
+    });
-
-    if (lex) {
-      var LEX = require('letsencrypt-express');
-      server.on('request', LEX.createAcmeResponder(lex, onRequest));
-    } else {
-      server.on('request', onRequest);
-    }
   }
 
   if (certPaths) {
-    require('./sni-server').create(lex, certPaths, initServer);
+    require('./sni-server').create(certPaths, initServer);
   } else {
     initServer(null, require('http').createServer());
   }

lib/main.js

@@ -0,0 +1,297 @@
+'use strict';
+
+module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
+  var PromiseA = require('bluebird');
+  var path = require('path');
+  var fs = PromiseA.promisifyAll(require('fs'));
+  // NOTE: each process has its own cache
+  var localCache = { le: {}, statics: {} };
+  var express = require('express');
+  var apiApp;
+  var setupDomain = xconfx.setupDomain = ('cloud.' + xconfx.primaryDomain);
+  var setupApp;
+
+  function redirectHttpsHelper(req, res) {
+    var host = req.hostname || req.headers.host || '';
+    var url = req.url;
+
+    // TODO
+    // allow exceptions for the case of arduino and whatnot that cannot handle https?
+    // http://evothings.com/is-it-possible-to-secure-micro-controllers-used-within-iot/
+    // needs ECDSA?
+
+    var escapeHtml = require('escape-html');
+    var newLocation = 'https://'
+      + host.replace(/:\d+/, ':' + xconfx.externalPort) + url
+      ;
+    var safeLocation = escapeHtml(newLocation);
+
+    var metaRedirect = ''
+      + '<html>\n'
+      + '<head>\n'
+      + '  <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+      + '  <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+      + '</head>\n'
+      + '<body style="display: none;">\n'
+      + '  <p>You requested an insecure resource. Please use this instead: \n'
+      + '    <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+      + '</body>\n'
+      + '</html>\n'
+      ;
+
+    // DO NOT HTTP REDIRECT
+    /*
+    res.setHeader('Location', newLocation);
+    res.statusCode = 302;
+    */
+
+    // BAD NEWS BEARS
+    //
+    // When people are experimenting with the API and posting tutorials
+    // they'll use cURL and they'll forget to prefix with https://
+    // If we allow that, then many users will be sending private tokens
+    // and such with POSTs in clear text and, worse, it will work!
+    // To minimize this, we give browser users a mostly optimal experience,
+    // but people experimenting with the API get a message letting them know
+    // that they're doing it wrong and thus forces them to ensure they encrypt.
+    res.setHeader('Content-Type', 'text/html; charset=utf-8');
+    res.end(metaRedirect);
+  }
+
+  function redirectHttps(req, res) {
+    if (localCache.le[req.hostname]) {
+      if (localCache.le[req.hostname].conf) {
+        redirectHttpsHelper(req, res);
+        return;
+      }
+      else {
+        // TODO needs IPC to expire cache
+        redirectSetup(req.hostname, req, res);
+        return;
+        /*
+        if (Date.now() - localCache.le[req.hostname].createdAt < (5 * 60 * 1000)) {
+          // TODO link to dbconf.primaryDomain
+          res.send({ error: { message: "Security Error: Encryption for '" + req.hostname + "' has not been configured."
+              + " Please use the management interface to set up ACME / Let's Encrypt (or another solution)." } });
+          return;
+        }
+        */
+      }
+    }
+
+    return xconfx.walkLe(req.hostname).then(function (leAuth) {
+      if (!leAuth) {
+        redirectSetup(req.hostname, req, res);
+        return;
+      }
+
+      localCache.le[req.hostname] = { conf: leAuth, createdAt: Date.now() };
+      redirectHttps(req, res);
+    });
+  }
+
+  function disallowSymLinks(req, res) {
+    res.end(
+      "Symbolic Links are not supported on all platforms and are therefore disallowed."
+    + " Instead, simply create a file of the same name as the link with a single line of text"
+    + " which should be the relative or absolute path to the target directory."
+    );
+  }
+
+  function disallowNonFiles(req, res) {
+    res.end(
+      "Pipes, Blocks, Sockets, FIFOs, and other such nonsense are not permitted."
+    + " Instead please create a directory from which to read or create a file "
+    + " with a single line of text which should be the target directory to read from."
+    );
+  }
+
+  function securityError(req, res) {
+    res.end("Security Error: Link points outside of packages/pages");
+  }
+
+  function notConfigured(req, res, next) {
+    if (setupDomain !== req.hostname) {
+      redirectSetup(req.hostname, req, res);
+      return;
+    }
+
+    if (!setupApp) {
+      setupApp = express.static(path.join(xconfx.staticpath, 'com.daplie.walnut'));
+    }
+    setupApp(req, res, function () {
+      if ('/' === req.url) {
+        res.end('Sanity Fail: Configurator not found');
+        return;
+      }
+      next();
+    });
+  }
+
+  function loadHandler(name) {
+    return function handler(req, res, next) {
+      var packagepath = path.join(xconfx.staticpath, name);
+
+      return fs.lstatAsync(packagepath).then(function (stat) {
+        if (stat.isSymbolicLink()) {
+          return disallowSymLinks;
+        }
+
+        if (stat.isDirectory()) {
+          return express.static(packagepath);
+        }
+
+        if (!stat.isFile()) {
+          return disallowNonFiles;
+        }
+
+        return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
+          // TODO allow cascading
+          text = text.trim().split(/\n/)[0];
+
+          // TODO rerun the above, disallowing link-style (or count or memoize to prevent infinite loop)
+          // TODO make safe
+          packagepath = path.resolve(xconfx.staticpath, text);
+          if (0 !== packagepath.indexOf(xconfx.staticpath)) {
+            return securityError;
+          }
+
+          return express.static(packagepath);
+        });
+      }, function (/*err*/) {
+        return notConfigured;
+      }).then(function (handler) {
+
+        // keep object reference intact
+        localCache.statics[name].handler = handler;
+        handler(req, res, next);
+      });
+    };
+  }
+
+  function staticHelper(appId, opts) {
+    // TODO inter-process cache expirey
+    // TODO add to xconfx.staticpath
+    xconfx.staticpath = path.join(__dirname, '..', '..', 'packages', 'pages');
+    return fs.readdirAsync(xconfx.staticpath).then(function (nodes) {
+      if (opts && opts.clear) {
+        localCache.statics = {};
+      }
+
+      // longest to shortest
+      function shortToLong(a, b) {
+        return b.length - a.length;
+      }
+      nodes.sort(shortToLong);
+
+      nodes.forEach(function (name) {
+        if (!localCache.statics[name]) {
+          localCache.statics[name] = { handler: loadHandler(name), createdAt: Date.now() };
+        }
+      });
+
+      // Secure Matching
+      // apple.com#blah#  apple.com#blah#
+      // apple.com.us#    apple.com#foo#
+      // apple.com#       apple.com#foo#
+      nodes.some(function (name) {
+        if (0 === (name + '#').indexOf(appId + '#')) {
+          if (appId !== name) {
+            localCache.statics[appId] = localCache.statics[name];
+          }
+          return true;
+        }
+      });
+
+      if (!localCache.statics[appId]) {
+        localCache.statics[appId] = { handler: notConfigured, createdAt: Date.now() };
+      }
+
+      localCache.staticsKeys = Object.keys(localCache.statics).sort(shortToLong);
+      return localCache.statics[appId];
+    });
+  }
+
+  function redirectSetup(reason, req, res/*, next*/) {
+    var url = 'https://cloud.' + xconfx.primaryDomain;
+
+    if (443 !== xconfx.externalPort) {
+      url += ':' + xconfx.externalPort;
+    }
+
+    url += '#referrer=' + reason;
+
+    res.statusCode = 302;
+    res.setHeader('Location', url);
+    res.end();
+  }
+
+  function serveStatic(req, res, next) {
+    // If we get this far we can be pretty confident that
+    // the domain was already set up because it's encrypted
+    var appId = req.hostname + req.url.replace(/\/+/g, '#').replace(/#$/, '');
+    var appIdParts = appId.split('#');
+    var appIdPart;
+
+    if (!req.secure) {
+      // did not come from https
+      if (/\.(appcache|manifest)\b/.test(req.url)) {
+        require('./unbrick-appcache').unbrick(req, res);
+        return;
+      }
+      return redirectHttps(req, res);
+    }
+
+    // TODO configuration for allowing www
+    if (/^www\./.test(req.hostname)) {
+      // NOTE: acme responder and appcache unbricker must come before scrubTheDub
+      if (/\.(appcache|manifest)\b/.test(req.url)) {
+        require('./unbrick-appcache').unbrick(req, res);
+        return;
+      }
+      require('./no-www').scrubTheDub(req, res);
+      return;
+    }
+    /*
+    if (!redirectives && config.redirects) {
+      redirectives = require('./hostname-redirects').compile(config.redirects);
+    }
+    */
+
+    // TODO assets.example.com/sub/assets/com.example.xyz/
+    if (/^api\./.test(req.hostname) && /\/api(\/|$)/.test(req.url)) {
+      // supports api.example.com/sub/app/api/com.example.xyz/
+      if (!apiApp) {
+        apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
+      }
+      apiApp(req, res, next);
+      return;
+    }
+
+    while (appIdParts.length) {
+      // TODO needs IPC to expire cache
+      appIdPart = appIdParts.join('#');
+      if (localCache.statics[appIdPart]) {
+        break;
+      }
+      // TODO test via staticsKeys
+
+      appIdParts.pop();
+    }
+
+    if (!appIdPart || !localCache.statics[appIdPart]) {
+      return staticHelper(appId).then(function () {
+        localCache.statics[appId].handler(req, res, next);
+      });
+    }
+
+    localCache.statics[appIdPart].handler(req, res, next);
+    if (Date.now() - localCache.statics[appIdPart].createdAt > (5 * 60 * 1000)) {
+      staticHelper(appId, { clear: true });
+    }
+  }
+
+  app.use('/', serveStatic);
+
+  return PromiseA.resolve();
+};

lib/master.js

@@ -2,18 +2,17 @@
 
 var cluster = require('cluster');
 var PromiseA = require('bluebird');
-// TODO
-// var rootMasterKey;
 
 function init(conf, state) {
+  var newConf = {};
   if (!conf.ipcKey) {
-    conf.ipcKey = require('crypto').randomBytes(16).toString('base64');
+    conf.ipcKey = newConf.ipcKey = require('crypto').randomBytes(16).toString('base64');
   }
   if (!conf.sqlite3Sock) {
-    conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
+    conf.sqlite3Sock = newConf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
   }
   if (!conf.memstoreSock) {
-    conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
+    conf.memstoreSock = newConf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
   }
 
   try {
@@ -49,15 +48,14 @@ function init(conf, state) {
       verbose: null
     , sock: conf.sqlite3Sock
     , ipcKey: conf.ipcKey
-    })
+    })/*.then(function () {
+      var sqlite3 = require('sqlite3-cluster/client');
+      return sqliet3.createClientFactory(...);
+    })*/
   ]).then(function (args) {
     state.memstore = args[0];
-    state.sqlstore = args[1];
+    //state.sqlstore = args[1];
-    return {
+    return newConf;
-      conf: conf
-    , memstore: args[0]
-    , sqlstore: args[1]
-    };
   });
 
   return promise;
@@ -69,10 +67,10 @@ function touch(conf, state) {
   }
 
   // TODO if no xyz worker, start on xyz worker (unlock, for example)
-  return state.initialize.then(function () {
+  return state.initialize.then(function (newConf) {
     // TODO conf.locked = true|false;
     conf.initialized = true;
-    return conf;
+    return newConf;
   });
 }
 

lib/package-server-apis.js

@@ -0,0 +1,215 @@
+'use strict';
+
+var escapeStringRegexp = require('escape-string-regexp');
+//var apiHandlers = {};
+
+function getApi(conf, pkgConf, pkgDeps, packagedApi) {
+  var PromiseA = pkgDeps.Promise;
+  var path = require('path');
+  var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
+
+  // TODO needs some version stuff (which would also allow hot-loading of updates)
+  // TODO version could be tied to sha256sum
+
+  return new PromiseA(function (resolve, reject) {
+    var myApp;
+    var ursa;
+    var promise;
+
+    // TODO dynamic requires are a no-no
+    // can we statically generate a require-er? on each install?
+    // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
+    // requirer[pkgpath]()
+    myApp = pkgDeps.express();
+    myApp.disable('x-powered-by');
+    if (pkgDeps.app.get('trust proxy')) {
+      myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
+    }
+    if (!pkgConf.pubkey) {
+      /*
+        return ursa.createPrivateKey(pem, password, encoding);
+        var pem = myKey.toPrivatePem();
+        return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
+        });
+      */
+      ursa = require('ursa');
+      pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
+      pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
+    }
+
+    try {
+      packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
+      packagedApi._apiname = packagedApi._apipkg.name;
+      if (packagedApi._apipkg.walnut) {
+        pkgpath += '/' + packagedApi._apipkg.walnut;
+      }
+      promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
+    } catch(e) {
+      reject(e);
+      return;
+    }
+
+    promise.then(function () {
+      // TODO give pub/priv pair for app and all public keys
+      // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
+      packagedApi._api = require('express-lazy')();
+      packagedApi._api_app = myApp;
+
+      //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
+      pkgDeps.getOauth3Controllers =
+      packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
+      require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
+
+      // DEBUG
+      //
+      /*
+      packagedApi._api.use('/', function (req, res, next) {
+        console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
+        next();
+      });
+      //*/
+
+      // TODO fix backwards compat
+
+      // /api/com.example.foo (no change)
+      packagedApi._api.use('/', packagedApi._api_app);
+
+      // /api/com.example.foo => /api
+      packagedApi._api.use('/', function (req, res, next) {
+        var priorUrl = req.url;
+        req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
+        // console.log('api mangle 3:', req.url);
+        packagedApi._api_app(req, res, function (err) {
+          req.url = priorUrl;
+          next(err);
+        });
+      });
+
+      // /api/com.example.foo => /
+      packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
+        // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
+        // console.log(packagedApi._api_app.toString());
+        packagedApi._api_app(req, res, next);
+      });
+
+      resolve(packagedApi._api);
+    }, reject);
+  });
+}
+
+function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
+  function handlePromise(p) {
+    return p.then(function (api) {
+      packagedApi._api = api;
+      return api;
+    });
+  }
+
+  if (!packagedApi._promise_api) {
+    packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
+  }
+
+  return handlePromise(packagedApi._promise_api);
+}
+
+function runApi(opts, router, req, res, next) {
+  var path = require('path');
+  var pkgConf = opts.config;
+  var pkgDeps = opts.deps;
+  //var Services = opts.Services;
+  var packagedApi;
+  var pathname;
+
+  // TODO compile packagesMap
+  // TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name)
+  router.packagedApis.some(function (_packagedApi) {
+    // console.log('[DEBUG _packagedApi.id]', _packagedApi.id);
+    pathname = router.pathname;
+    if ('/' === pathname) {
+      pathname = '';
+    }
+    // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
+    if (!_packagedApi._api_re) {
+      _packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)');
+      //console.log('[api re 2]', _packagedApi._api_re);
+    }
+    if (_packagedApi._api_re.test(req.url)) {
+      packagedApi = _packagedApi;
+      return true;
+    }
+  });
+
+  if (!packagedApi) {
+    console.log("[ODD] no api for '" + req.url + "'");
+    next();
+    return;
+  }
+
+  // Reaching this point means that there are APIs for this pathname
+  // it is important to identify this host + pathname (example.com/foo) as the app
+  Object.defineProperty(req, 'experienceId', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
+    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
+    // NOTE: probably best to alias the name logically
+  , value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '')
+  });
+  Object.defineProperty(req, 'escapedExperienceId', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
+    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
+    // NOTE: probably best to alias the name logically
+  , value: req.experienceId.replace(/\//g, ':')
+  });
+  // packageId should mean hash(api.id + host + path) - also called "api"
+  Object.defineProperty(req, 'packageId', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
+    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
+    // NOTE: probably best to alias the name logically
+  , value: packagedApi.domain.id
+  });
+  Object.defineProperty(req, 'appConfig', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+  , value: {}       // TODO just the app-scoped config
+  });
+  Object.defineProperty(req, 'appDeps', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+  , value: {}       // TODO app-scoped deps
+                    // i.e. when we need to use things such as stripe id
+                    // without exposing them to the app
+  });
+
+  //
+  // TODO user authentication should go right about here
+  //
+
+  //
+  // TODO freeze objects for passing them into app
+  //
+
+  if (packagedApi._api) {
+    packagedApi._api(req, res, next);
+    return;
+  }
+
+  // console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id);
+  loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) {
+    api(req, res, next);
+  }, function (err) {
+    console.error('[App Promise Error]');
+    next(err);
+  });
+}
+
+module.exports.runApi = runApi;

lib/package-server-static.js

@@ -0,0 +1,87 @@
+'use strict';
+
+var staticHandlers = {};
+
+function loadPages(pkgConf, packagedPage, req, res, next) {
+  var PromiseA = require('bluebird');
+  var fs = require('fs');
+  var path = require('path');
+  var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || ''));
+
+  // TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc)
+
+  function handlePromise(p) {
+    p.then(function (app) {
+      app(req, res, next);
+      packagedPage._page = app;
+    }, function (err) {
+      console.error('[App Promise Error]');
+      next(err);
+    });
+  }
+
+  if (staticHandlers[pkgpath]) {
+    packagedPage._page = staticHandlers[pkgpath];
+    packagedPage._page(req, res, next);
+    return;
+  }
+
+  if (!packagedPage._promise_page) {
+    packagedPage._promise_page = new PromiseA(function (resolve, reject) {
+      fs.exists(pkgpath, function (exists) {
+        var staticServer;
+
+        if (!exists) {
+          reject(new Error("package '" + pkgpath + "' is registered but does not exist"));
+          return;
+        }
+
+        //console.log('[static mount]', pkgpath);
+        // https://github.com/expressjs/serve-static/issues/54
+        // https://github.com/pillarjs/send/issues/91
+        // https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx
+        staticServer = require('serve-static')(pkgpath, { dotfiles: undefined });
+        resolve(staticServer);
+      });
+    });
+  }
+
+  handlePromise(packagedPage._promise_page);
+}
+
+function layerItUp(pkgConf, router, req, res, next) {
+  var nexti = -1;
+  // Layers exist so that static apps can use them like a virtual filesystem
+  // i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html
+  // or perhaps some dynamic content (like application cache)
+  function nextify(err) {
+    var packagedPage;
+    nexti += 1;
+
+    if (err) {
+      next(err);
+      return;
+    }
+
+    // shortest to longest
+    //route = packages.pop();
+    // longest to shortest
+    packagedPage = router.packagedPages[nexti];
+    if (!packagedPage) {
+      next();
+      return;
+    }
+
+    if (packagedPage._page) {
+      packagedPage._page(req, res, nextify);
+      return;
+    }
+
+    // could attach to req.{ pkgConf, pkgDeps, Services}
+    loadPages(pkgConf, packagedPage, req, res, next);
+  }
+
+  nextify();
+}
+
+module.exports.layerItUp = layerItUp;

lib/package-server.js

@@ -1,8 +1,8 @@
 'use strict';
 
 var escapeStringRegexp = require('escape-string-regexp');
-var staticHandlers = {};
+var runApi = require('./package-server-apis').runApi;
-//var apiHandlers = {};
+var layerItUp = require('./package-server-static').layerItUp;
 
 function compileVhosts(vhostsMap) {
   var results = {
@@ -62,297 +62,6 @@ function compileVhosts(vhostsMap) {
   return results;
 }
 
-function loadPages(pkgConf, packagedPage, req, res, next) {
-  var PromiseA = require('bluebird');
-  var fs = require('fs');
-  var path = require('path');
-  var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || ''));
-
-  // TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc)
-
-  function handlePromise(p) {
-    p.then(function (app) {
-      app(req, res, next);
-      packagedPage._page = app;
-    }, function (err) {
-      console.error('[App Promise Error]');
-      next(err);
-    });
-  }
-
-  if (staticHandlers[pkgpath]) {
-    packagedPage._page = staticHandlers[pkgpath];
-    packagedPage._page(req, res, next);
-    return;
-  }
-
-  if (!packagedPage._promise_page) {
-    packagedPage._promise_page = new PromiseA(function (resolve, reject) {
-      fs.exists(pkgpath, function (exists) {
-        var staticServer;
-
-        if (!exists) {
-          reject(new Error("package '" + pkgpath + "' is registered but does not exist"));
-          return;
-        }
-
-        //console.log('[static mount]', pkgpath);
-        // https://github.com/expressjs/serve-static/issues/54
-        // https://github.com/pillarjs/send/issues/91
-        // https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx
-        staticServer = require('serve-static')(pkgpath, { dotfiles: undefined });
-        resolve(staticServer);
-      });
-    });
-  }
-
-  handlePromise(packagedPage._promise_page);
-}
-
-function getApi(conf, pkgConf, pkgDeps, packagedApi) {
-  var PromiseA = pkgDeps.Promise;
-  var path = require('path');
-  var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
-
-  // TODO needs some version stuff (which would also allow hot-loading of updates)
-  // TODO version could be tied to sha256sum
-
-  return new PromiseA(function (resolve, reject) {
-    var myApp;
-    var ursa;
-    var promise;
-
-    // TODO dynamic requires are a no-no
-    // can we statically generate a require-er? on each install?
-    // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
-    // requirer[pkgpath]()
-    myApp = pkgDeps.express();
-    myApp.disable('x-powered-by');
-    if (pkgDeps.app.get('trust proxy')) {
-      myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
-    }
-    if (!pkgConf.pubkey) {
-      /*
-        return ursa.createPrivateKey(pem, password, encoding);
-        var pem = myKey.toPrivatePem();
-        return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
-        });
-      */
-      ursa = require('ursa');
-      pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
-      pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
-    }
-
-    try {
-      packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
-      packagedApi._apiname = packagedApi._apipkg.name;
-      if (packagedApi._apipkg.walnut) {
-        pkgpath += '/' + packagedApi._apipkg.walnut;
-      }
-      promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
-    } catch(e) {
-      reject(e);
-      return;
-    }
-
-    promise.then(function () {
-      // TODO give pub/priv pair for app and all public keys
-      // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
-      packagedApi._api = require('express-lazy')();
-      packagedApi._api_app = myApp;
-
-      //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
-      pkgDeps.getOauth3Controllers =
-      packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
-      require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
-
-      // DEBUG
-      //
-      /*
-      packagedApi._api.use('/', function (req, res, next) {
-        console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
-        next();
-      });
-      //*/
-
-      // TODO fix backwards compat
-
-      // /api/com.example.foo (no change)
-      packagedApi._api.use('/', packagedApi._api_app);
-
-      // /api/com.example.foo => /api
-      packagedApi._api.use('/', function (req, res, next) {
-        var priorUrl = req.url;
-        req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
-        // console.log('api mangle 3:', req.url);
-        packagedApi._api_app(req, res, function (err) {
-          req.url = priorUrl;
-          next(err);
-        });
-      });
-
-      // /api/com.example.foo => /
-      packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
-        // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
-        // console.log(packagedApi._api_app.toString());
-        packagedApi._api_app(req, res, next);
-      });
-
-      resolve(packagedApi._api);
-    }, reject);
-  });
-}
-
-function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
-  function handlePromise(p) {
-    return p.then(function (api) {
-      packagedApi._api = api;
-      return api;
-    });
-  }
-
-  if (!packagedApi._promise_api) {
-    packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
-  }
-
-  return handlePromise(packagedApi._promise_api);
-}
-
-function layerItUp(pkgConf, router, req, res, next) {
-  var nexti = -1;
-  // Layers exist so that static apps can use them like a virtual filesystem
-  // i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html
-  // or perhaps some dynamic content (like application cache)
-  function nextify(err) {
-    var packagedPage;
-    nexti += 1;
-
-    if (err) {
-      next(err);
-      return;
-    }
-
-    // shortest to longest
-    //route = packages.pop();
-    // longest to shortest
-    packagedPage = router.packagedPages[nexti];
-    if (!packagedPage) {
-      next();
-      return;
-    }
-
-    if (packagedPage._page) {
-      packagedPage._page(req, res, nextify);
-      return;
-    }
-
-    // could attach to req.{ pkgConf, pkgDeps, Services}
-    loadPages(pkgConf, packagedPage, req, res, next);
-  }
-
-  nextify();
-}
-
-function runApi(opts, router, req, res, next) {
-  var path = require('path');
-  var pkgConf = opts.config;
-  var pkgDeps = opts.deps;
-  //var Services = opts.Services;
-  var packagedApi;
-  var pathname;
-
-  // TODO compile packagesMap
-  // TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name)
-  router.packagedApis.some(function (_packagedApi) {
-    // console.log('[DEBUG _packagedApi.id]', _packagedApi.id);
-    pathname = router.pathname;
-    if ('/' === pathname) {
-      pathname = '';
-    }
-    // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
-    if (!_packagedApi._api_re) {
-      _packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)');
-      //console.log('[api re 2]', _packagedApi._api_re);
-    }
-    if (_packagedApi._api_re.test(req.url)) {
-      packagedApi = _packagedApi;
-      return true;
-    }
-  });
-
-  if (!packagedApi) {
-    console.log("[ODD] no api for '" + req.url + "'");
-    next();
-    return;
-  }
-
-  // Reaching this point means that there are APIs for this pathname
-  // it is important to identify this host + pathname (example.com/foo) as the app
-  Object.defineProperty(req, 'experienceId', {
-    enumerable: true
-  , configurable: false
-  , writable: false
-    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
-    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
-    // NOTE: probably best to alias the name logically
-  , value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '')
-  });
-  Object.defineProperty(req, 'escapedExperienceId', {
-    enumerable: true
-  , configurable: false
-  , writable: false
-    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
-    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
-    // NOTE: probably best to alias the name logically
-  , value: req.experienceId.replace(/\//g, ':')
-  });
-  // packageId should mean hash(api.id + host + path) - also called "api"
-  Object.defineProperty(req, 'packageId', {
-    enumerable: true
-  , configurable: false
-  , writable: false
-    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
-    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
-    // NOTE: probably best to alias the name logically
-  , value: packagedApi.domain.id
-  });
-  Object.defineProperty(req, 'appConfig', {
-    enumerable: true
-  , configurable: false
-  , writable: false
-  , value: {}       // TODO just the app-scoped config
-  });
-  Object.defineProperty(req, 'appDeps', {
-    enumerable: true
-  , configurable: false
-  , writable: false
-  , value: {}       // TODO app-scoped deps
-                    // i.e. when we need to use things such as stripe id
-                    // without exposing them to the app
-  });
-
-  //
-  // TODO user authentication should go right about here
-  //
-
-  //
-  // TODO freeze objects for passing them into app
-  //
-
-  if (packagedApi._api) {
-    packagedApi._api(req, res, next);
-    return;
-  }
-
-  // console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id);
-  loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) {
-    api(req, res, next);
-  }, function (err) {
-    console.error('[App Promise Error]');
-    next(err);
-  });
-}
-
 function mapToApp(opts, req, res, next) {
   // opts = { config, deps, services }
   var vhost;
@@ -450,6 +159,5 @@ function mapToApp(opts, req, res, next) {
   return runApi(opts, router, req, res, next);
 }
 
-module.exports.runApi = runApi;
 module.exports.compileVhosts = compileVhosts;
 module.exports.mapToApp = mapToApp;

lib/worker.js

@@ -1,137 +1,50 @@
 'use strict';
 
-module.exports.create = function (webserver, conf, state) {
+module.exports.create = function (webserver, xconfx, state) {
+  console.log('DEBUG create worker');
+
   if (!state) {
     state = {};
   }
 
   var PromiseA = state.Promise || require('bluebird');
-  var path = require('path');
-  //var vhostsdir = path.join(__dirname, 'vhosts');
-  var express = require('express-lazy');
-  var app = express();
   var memstore;
   var sqlstores = {};
-  var models = {};
   var systemFactory = require('sqlite3-cluster/client').createClientFactory({
-      dirname: path.join(__dirname, '..', '..', 'var') // TODO conf
+      dirname: xconfx.varpath
-    , prefix: 'com.example.'
+    , prefix: 'com.daplie.walnut.'
     //, dbname: 'config'
     , suffix: ''
     , ext: '.sqlite3'
-    , sock: conf.sqlite3Sock
+    , sock: xconfx.sqlite3Sock
-    , ipcKey: conf.ipcKey
+    , ipcKey: xconfx.ipcKey
   });
+  /*
   var clientFactory = require('sqlite3-cluster/client').createClientFactory({
       algorithm: 'aes'
     , bits: 128
     , mode: 'cbc'
-    , dirname: path.join(__dirname, '..', '..', 'var') // TODO conf
+    , dirname: xconfx.varpath // TODO conf
-    , prefix: 'com.example.'
+    , prefix: 'com.daplie.walnut.'
     //, dbname: 'cluster'
     , suffix: ''
     , ext: '.sqlcipher'
-    , sock: conf.sqlite3Sock
+    , sock: xconfx.sqlite3Sock
-    , ipcKey: conf.ipcKey
+    , ipcKey: xconfx.ipcKey
   });
-  var cstore = require('cluster-store');
-  var redirectives;
-
-  app.disable('x-powered-by');
-  if (conf.trustProxy) {
-    console.info('[Trust Proxy]');
-    app.set('trust proxy', ['loopback']);
-    //app.set('trust proxy', function (ip) { console.log('[ip]', ip); return true; });
-  } else {
-    console.info('[DO NOT trust proxy]');
-    // TODO make sure the gzip module loads if there isn't a proxy gzip-ing for us
-    // app.use(compression())
-  }
-
-  /*
-  function unlockDevice(conf, state) {
-    return require('./lib/unlock-device').create().then(function (result) {
-      result.promise.then(function (_rootMasterKey) {
-        process.send({
-          type: 'walnut.keys.root'
-          conf: {
-            rootMasterKey: _rootMasterkey
-          }
-        });
-        conf.locked = false;
-        if (state.caddy) {
-          state.caddy.update(conf);
-        }
-        conf.rootMasterKey = _rootMasterKey;
-      });
-
-      return result.app;
-    });
-  }
   */
-
+  var cstore = require('cluster-store');
-  // TODO handle insecure to actual redirect
-  // blog.coolaj86.com -> coolaj86.com/blog
-  // hmm... that won't really matter with hsts
-  // I guess I just needs letsencrypt
-
-  function scrubTheDub(req, res, next) {
-    var host = req.hostname;
-
-    if (!host || 'string' !== typeof host) {
-      next();
-      return;
-    }
-
-    // TODO test if this is even necessary
-    host = host.toLowerCase();
-
-    // TODO this should be hot loadable / changeable
-    if (!redirectives && conf.redirects) {
-      redirectives = require('./hostname-redirects').compile(conf.redirects);
-    }
-
-    if (!/^www\./.test(host) && !redirectives) {
-      next();
-      return;
-    }
-
-    // TODO misnomer, handles all exact redirects
-    if (!require('./no-www').scrubTheDub(req, res, redirectives)) {
-      next();
-      return;
-    }
-  }
-
-  function caddyBugfix(req, res, next) {
-    // workaround for Caddy
-    // https://github.com/mholt/caddy/issues/341
-    if (app.get('trust proxy')) {
-      if (req.headers['x-forwarded-proto']) {
-        req.headers['x-forwarded-proto'] = (req.headers['x-forwarded-proto'] || '').split(/,\s+/g)[0] || undefined;
-      }
-      if (req.headers['x-forwarded-host']) {
-        req.headers['x-forwarded-host'] = (req.headers['x-forwarded-host'] || '').split(/,\s+/g)[0] || undefined;
-      }
-    }
-
-    next();
-  }
-
-  // TODO misnomer, this can handle nowww, yeswww, and exact hostname redirects
-  app.use('/', scrubTheDub);
-  app.use('/', caddyBugfix);
 
   return PromiseA.all([
     // TODO security on memstore
     // TODO memstoreFactory.create
     cstore.create({
-      sock: conf.memstoreSock
+      sock: xconfx.memstoreSock
-    , connect: conf.memstoreSock
+    , connect: xconfx.memstoreSock
       // TODO implement
-    , key: conf.ipcKey
+    , key: xconfx.ipcKey
     }).then(function (_memstore) {
-      memstore = _memstore;
+      memstore = PromiseA.promisifyAll(_memstore);
       return memstore;
     })
     // TODO mark a device as lost, stolen, missing in DNS records
@@ -140,101 +53,127 @@ module.exports.create = function (webserver, conf, state) {
         init: true
       , dbname: 'config'
     })
-  , clientFactory.create({
-        init: true
-      , key: '00000000000000000000000000000000'
-      // TODO only complain if the values are different
-      //, algo: 'aes'
-      , dbname: 'auth'
-    })
-  , clientFactory.create({
-        init: false
-      , dbname: 'system'
-    })
   ]).then(function (args) {
     memstore = args[0];
     sqlstores.config = args[1];
-    sqlstores.auth = args[2];
-    sqlstores.system = args[3];
-    sqlstores.create = clientFactory.create;
 
-    return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) {
+    var wrap = require('masterquest-sqlite3');
-      models.Config = tables;
+    var dir = [
-      return models.Config.Config.get().then(function (vhostsMap) {
+      { tablename: 'com_daplie_walnut_config'
-        // TODO the core needs to be replacable in one shot
+      , idname: 'id'
-        // rm -rf /tmp/walnut/; tar xvf -C /tmp/walnut/; mv /srv/walnut /srv/walnut.{{version}}; mv /tmp/walnut /srv/
+      , unique: [ 'id' ]
-        // this means that any packages must be outside, perhaps /srv/walnut/{boot,core,packages}
+      , indices: [ 'createdAt', 'updatedAt' ]
-        var pkgConf = {
+      }
-          pagespath: path.join(__dirname, '..', '..', 'packages', 'pages') + path.sep
+    , { tablename: 'com_daplie_walnut_redirects'
-        , apipath: path.join(__dirname, '..', '..', 'packages', 'apis') + path.sep
+      , idname: 'id'      // blog.example.com:sample.net/blog
-        , servicespath: path.join(__dirname, '..', '..', 'packages', 'services')
+      , unique: [ 'id' ]
-        , vhostsMap: vhostsMap
+      , indices: [ 'createdAt', 'updatedAt' ]
-        , vhostPatterns: null
+      }
-        , server: webserver
+    ];
-        , externalPort: conf.externalPort
-        , privkey: conf.privkey
-        , pubkey: conf.pubkey
-        , redirects: conf.redirects
-        , apiPrefix: '/api'
-        , 'org.oauth3.consumer': conf['org.oauth3.consumer']
-        , 'org.oauth3.provider': conf['org.oauth3.provider']
-        , keys: conf.keys
-        };
-        var pkgDeps = {
-          memstore: memstore
-        , sqlstores: sqlstores
-        , clientSqlFactory: clientFactory
-        , systemSqlFactory: systemFactory
-        //, handlePromise: require('./lib/common').promisableRequest;
-        //, handleRejection: require('./lib/common').rejectableRequest;
-        //, localPort: conf.localPort
-        , Promise: PromiseA
-        , express: express
-        , app: app
-        //, oauthmodels: require('oauthcommon/example-oauthmodels').create(conf)
-        };
-        var Services = require('./services-loader').create(pkgConf, {
-          memstore: memstore
-        , sqlstores: sqlstores
-        , clientSqlFactory: clientFactory
-        , systemSqlFactory: systemFactory
-        , Promise: PromiseA
-        });
-        var recase = require('connect-recase')({
-          // TODO allow explicit and or default flag
-          explicit: false
-        , default: 'snake'
-        , prefixes: ['/api']
-          // TODO allow exclude
-        //, exclusions: [config.oauthPrefix]
-        , exceptions: {}
-        //, cancelParam: 'camel'
-        });
 
-        function handlePackages(req, res, next) {
+    function scopeMemstore(expId) {
-          // TODO move to caddy parser?
+      var scope = expId + '|';
-          if (/(^|\.)proxyable\./.test(req.hostname)) {
+      return {
-            // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
+        getAsync: function (id) {
-            // proxyable.myapp.mydomain.com => myapp.mydomain.com
+          return memstore.getAsync(scope + id);
-            // TODO myapp.mydomain.com.example.proxyable.com => myapp.mydomain.com
+        }
-            req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
+      , setAsync: function (id, data) {
-          }
+          return memstore.setAsync(scope + id, data);
-
+        }
-          require('./package-server').mapToApp({
+      , touchAsync: function (id, data) {
-            config: pkgConf
+          return memstore.touchAsync(scope + id, data);
-          , deps: pkgDeps
+        }
-          , services: Services
+      , destroyAsync: function (id) {
-          , conf: conf
+          return memstore.destroyAsync(scope + id);
-          }, req, res, next);
         }
 
-        // TODO recase
+      // helpers
+      , allAsync: function () {
+          return memstore.allASync().then(function (db) {
+            return Object.keys(db).filter(function (key) {
+              return 0 === key.indexOf(scope);
+            }).map(function (key) {
+              return db[key];
+            });
+          });
+        }
+      , lengthAsync: function () {
+          return memstore.allASync().then(function (db) {
+            return Object.keys(db).filter(function (key) {
+              return 0 === key.indexOf(scope);
+            }).length;
+          });
+        }
+      , clearAsync: function () {
+          return memstore.allASync().then(function (db) {
+            return Object.keys(db).filter(function (key) {
+              return 0 === key.indexOf(scope);
+            }).map(function (key) {
+              return memstore.destroyAsync(key);
+            });
+          }).then(function () {
+            return null;
+          });
+        }
+      };
+    }
 
-        //
+    return wrap.wrap(sqlstores.config, dir).then(function (models) {
-        // Generic Template API
+      return models.ComDaplieWalnutConfig.find(null, { limit: 100 }).then(function (results) {
-        //
+        return models.ComDaplieWalnutConfig.find(null, { limit: 10000 }).then(function (redirects) {
-        app
+          var express = require('express-lazy');
-          .use('/api', require('body-parser').json({
+          var app = express();
+          var recase = require('connect-recase')({
+            // TODO allow explicit and or default flag
+            explicit: false
+          , default: 'snake'
+          , prefixes: ['/api']
+            // TODO allow exclude
+          //, exclusions: [config.oauthPrefix]
+          , exceptions: {}
+          //, cancelParam: 'camel'
+          });
+          var bootstrapApp;
+          var mainApp;
+          var apiDeps = {
+            models: models
+            // TODO don't let packages use this directly
+          , Promise: PromiseA
+          };
+          var apiFactories = {
+            memstoreFactory: { create: scopeMemstore }
+          , systemSqlFactory: systemFactory
+          };
+
+          function log(req, res, next) {
+            console.log('[worker/log]', req.method, req.headers.host, req.url);
+            next();
+          }
+
+          function setupMain() {
+            mainApp = express();
+            require('./main').create(mainApp, xconfx, apiFactories, apiDeps).then(function () {
+              // TODO process.send({});
+            });
+          }
+
+          if (!bootstrapApp) {
+            bootstrapApp = express();
+            require('./bootstrap').create(bootstrapApp, xconfx, models).then(function () {
+              // TODO process.send({});
+              setupMain();
+            });
+          }
+
+          process.on('message', function (data) {
+            if ('com.daplie.walnut.bootstrap' === data.type) {
+              setupMain();
+            }
+          });
+
+          app.disable('x-powered-by');
+          app.use('/', log);
+          app.use('/api', require('body-parser').json({
             strict: true // only objects and arrays
           , inflate: true
             // limited to due performance issues with JSON.parse and JSON.stringify
@@ -244,38 +183,40 @@ module.exports.create = function (webserver, conf, state) {
           , reviver: undefined
           , type: 'json'
           , verify: undefined
-          }))
+          }));
-          // DO NOT allow urlencoded at any point, it is expressly forbidden
+          app.use('/api', recase);
-          //.use(require('body-parser').urlencoded({
-          //  extended: true
-          //, inflate: true
-          //, limit: 100 * 1024
-          //, type: 'urlencoded'
-          //, verify: undefined
-          //}))
-          .use(require('connect-send-error').error())
-          ;
 
-        app.use('/api', recase);
+          app.use('/', function (req, res) {
+            if (!req.secure) {
+              // did not come from https
+              if (/\.(appcache|manifest)\b/.test(req.url)) {
+                require('./unbrick-appcache').unbrick(req, res);
+                return;
+              }
+            }
 
-        app.use('/', handlePackages);
+            if (xconfx.lex && /\.well-known\/acme-challenge\//.test(req.url)) {
-        app.use('/', function (err, req, res, next) {
+              var LEX = require('letsencrypt-express');
-          console.error('[Error Handler]');
+              xconfx.lex.debug = true;
-          console.error(err.stack);
+              xconfx.acmeResponder = xconfx.acmeResponder || LEX.createAcmeResponder(xconfx.lex/*, next*/);
-          if (req.xhr) {
+              xconfx.acmeResponder(req, res);
-            res.send({ error: { message: "kinda unknownish error" } });
+              return;
-          } else {
+            }
-            res.send('<html><head><title>ERROR</title></head><body>Error</body></html>');
-          }
 
-          // sadly express uses arity checking
+            // TODO check https://letsencrypt.status.io to see if https certification is not available
-          // so the fourth parameter must exist
+
-          if (false) {
+            if (mainApp) {
-            next();
+              mainApp(req, res);
-          }
+              return;
+            }
+            else {
+              bootstrapApp(req, res);
+              return;
+            }
+          });
+
+          return app;
         });
-
-        return app;
       });
     });
   });

setup-dev-deps.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+pushd node_modules/authentication-microservice/ || git clone git@github.com:coolaj86/node-authentication-microservice node_modules/authentication-microservice
+  git pull
+popd
+
+pushd node_modules/oauthclient-microservice/ || git clone git@github.com:OAuth3/node-oauth3clients.git node_modules/oauthclient-microservice
+  git pull
+popd
+
+pushd node_modules/oauthcommon/ || git clone git@github.com:coolaj86/node-oauthcommon.git node_modules/oauthcommon
+  git pull
+popd