now does redirects, static apps, and api mounting

boot/master.js

@@ -17,8 +17,8 @@ var workers = [];
 var caddypath = '/usr/local/bin/caddy';
 var useCaddy = require('fs').existsSync(caddypath);
 var conf = {
-  localPort: process.argv[2] || (useCaddy ? 4080 : 443)   // system / local network
+  localPort: process.argv[2] || (useCaddy ? 4080 : 6443)   // system / local network
-, insecurePort: process.argv[3] || (useCaddy ? 80 : 80)   // meh
+, insecurePort: process.argv[3] || (useCaddy ? 80 : 65080)   // meh
 , externalPort: 443                                       // world accessible
 // TODO externalInsecurePort?
 , locked: false // TODO XXX
@@ -42,8 +42,9 @@ function fork() {
 cluster.on('online', function (worker) {
   var path = require('path');
   // TODO XXX Should these be configurable? If so, where?
-  var certPaths = [path.join(__dirname, '..', 'certs', 'live')];
+  var certPaths = [path.join(__dirname, '..', '..', 'certs', 'live')];
   var info;
+  var config = require('../../config');
 
   console.info('[MASTER] Worker ' + worker.process.pid + ' is online');
   fork();
@@ -59,6 +60,8 @@ cluster.on('online', function (worker) {
     , trustProxy: useCaddy ? true : false
     , certPaths: useCaddy ? null : certPaths
     , ipcKey: null
+      // TODO let this load after server is listening
+    , redirects: config.redirects
     }
   };
   worker.send(info);
@@ -85,6 +88,7 @@ cluster.on('online', function (worker) {
       // TODO get this from db config instead
       info.conf.privkey = config.privkey;
       info.conf.pubkey = config.pubkey;
+      info.conf.redirects = config.redirects;
       worker.send(info);
     });
   }

boot/worker.js

@@ -2,24 +2,37 @@
 
 module.exports.create = function (opts) {
   var id = '0';
+  var promiseApp;
+
+  function createAndBindInsecure(message, cb) {
+    // TODO conditional if 80 is being served by caddy
+    require('../lib/insecure-server').create(message.conf.externalPort, message.conf.insecurePort, message, function (err, webserver) {
+      console.info("#" + id + " Listening on http://" + webserver.address().address + ":" + webserver.address().port, '\n');
+
+      // we are returning the promise result to the caller
+      return cb(null, webserver, null, message);
+    });
+  }
 
   function createAndBindServers(message, cb) {
-    var msg = message.conf;
+    // NOTE that message.conf[x] will be overwritten when the next message comes in
-
+    require('../lib/local-server').create(message.conf.certPaths, message.conf.localPort, message, function (err, webserver) {
-    require('../lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) {
       if (err) {
         console.error('[ERROR] worker.js');
         console.error(err.stack);
         throw err;
       }
 
-      console.info("#" + id + " Listening on " + msg.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
+      console.info("#" + id + " Listening on " + message.conf.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
 
-      return cb(webserver);
+      // we don't need time to pass, just to be able to return
+      process.nextTick(function () {
+        createAndBindInsecure(message, cb);
+      });
+
+      // we are returning the promise result to the caller
+      return cb(null, null, webserver, message);
     });
-
-    // TODO conditional if 80 is being served by caddy
-    require('../lib/insecure-server').create(msg.externalPort, msg.insecurePort);
   }
 
   //
@@ -35,9 +48,16 @@ module.exports.create = function (opts) {
     process.removeListener('message', waitForConfig);
 
     // NOTE: this callback must return a promise for an express app
-    createAndBindServers(message, function (webserver) {
+    createAndBindServers(message, function (err, insecserver, webserver, oldMessage) {
+      // TODO deep merge new message into old message
+      Object.keys(message.conf).forEach(function (key) {
+        oldMessage.conf[key] = message.conf[key];
+      });
       var PromiseA = require('bluebird');
-      return new PromiseA(function (resolve) {
+      if (promiseApp) {
+        return promiseApp;
+      }
+      promiseApp = new PromiseA(function (resolve) {
         function initWebServer(srvmsg) {
           if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) {
             console.warn('[Worker] 1 got unexpected message:');
@@ -56,6 +76,7 @@ module.exports.create = function (opts) {
         console.info('[Worker Ready]');
         return app;
       });
+      return promiseApp;
     });
   }
 
@@ -64,9 +85,12 @@ module.exports.create = function (opts) {
   //
   if (opts) {
     // NOTE: this callback must return a promise for an express app
-    createAndBindServers(opts, function (webserver) {
+    createAndBindServers(opts, function (err, insecserver, webserver/*, message*/) {
       var PromiseA = require('bluebird');
-      return new PromiseA(function (resolve) {
+      if (promiseApp) {
+        return promiseApp;
+      }
+      promiseApp = new PromiseA(function (resolve) {
         opts.getConfig(function (srvmsg) {
           resolve(require('../lib/worker').create(webserver, srvmsg));
         });
@@ -74,6 +98,7 @@ module.exports.create = function (opts) {
         console.info('[Standalone Ready]');
         return app;
       });
+      return promiseApp;
     });
   } else {
     // we are in cluster mode, as opposed to standalone mode

lib/api-server.js

@@ -1,159 +0,0 @@
-'use strict';
-
-// TODO handle static app urls?
-// NOTE rejecting non-api urls should happen before this
-module.exports.create = function (conf, deps/*, Services*/) {
-  var PromiseA = deps.Promise;
-  var app = deps.app;
-  var express = deps.express;
-  var escapeStringRegexp = require('escape-string-regexp');
-  var vhostsMap = conf.vhostsMap;
-
-  function getApi(route) {
-    // TODO don't modify route, modify some other variable instead
-
-    var path = require('path');
-    // TODO needs some version stuff (which would also allow hot-loading of updates)
-    // TODO version could be tied to sha256sum
-    var pkgpath = path.join(conf.apipath, (route.api.package || route.api.id), (route.api.version || ''));
-
-    return new PromiseA(function (resolve, reject) {
-      var myApp;
-      var ursa;
-
-      try {
-        // TODO dynamic requires are a no-no
-        // can we statically generate a require-er? on each install?
-        // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
-        // requirer[pkgpath]()
-        myApp = express();
-        myApp.disable('x-powered-by');
-        if (app.get('trust proxy')) {
-          myApp.set('trust proxy', app.get('trust proxy'));
-        }
-        if (!conf.pubkey) {
-          /*
-            return ursa.createPrivateKey(pem, password, encoding);
-            var pem = myKey.toPrivatePem();
-            return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
-            });
-          */
-          ursa = require('ursa');
-          conf.keypair = ursa.createPrivateKey(conf.privkey, 'ascii');
-          conf.pubkey = ursa.createPublicKey(conf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
-        }
-        // TODO give pub/priv pair for app and all public keys
-        route.route = require(pkgpath).create(conf, deps, myApp);
-      } catch(e) {
-        reject(e);
-        return;
-      }
-
-      resolve(route.route);
-    });
-  }
-
-  function api(req, res, next) {
-    var apps; 
-
-    if (!vhostsMap[req.hostname]) {
-      // TODO keep track of match-only vhosts, such as '*.example.com',
-      // separate from exact matches
-      next(new Error("this domain is not registered"));
-      return;
-    }
-
-    vhostsMap[req.hostname].pathnames.some(function (route) {
-      var pathname = route.pathname;
-      if ('/' === pathname) {
-        pathname = '/api';
-      }
-      if (-1 === pathname.indexOf('/api')) {
-        // TODO needs namespace for current api
-        pathname = '/api' + pathname;
-      }
-      // pathname += '.local';
-
-      if (!route.re) {
-        route.re = new RegExp(escapeStringRegexp(pathname) + '(#|\\/|\\?|$)');
-      }
-      // re.test("/api")
-      // re.test("/api?")
-      // re.test("/api/")
-      // re.test("/api/foo")
-      // re.test("/apifoo") // false
-      if (route.re.test(req.url)) {
-        // make a copy
-        apps = route.apps.slice(0);
-        return true;
-      }
-    });
-
-    if (!apps) {
-      next();
-      return;
-    }
-
-    function nextify(err) {
-      var route;
-
-      if (err) {
-        next(err);
-        return;
-      }
-      
-      // shortest to longest
-      //route = apps.pop();
-      // longest to shortest
-      route = apps.shift();
-      if (!route) {
-        next();
-        return;
-      }
-
-      if (route.route) {
-        if (route.route.then) {
-          route.route.then(function (expressApp) {
-            expressApp(req, res, nextify);
-          });
-          return;
-        }
-        route.route(req, res, nextify);
-        return;
-      }
-
-      if (route._errored) {
-        nextify(new Error("couldn't load api"));
-        return;
-      }
-
-      if (!route.api) {
-        console.error('missing route:', req.url);
-        nextify(new Error("no api available for this route"));
-        return;
-      }
-
-      return getApi(route).then(function (expressApp) {
-        try {
-          expressApp(req, res, nextify);
-          route.route = expressApp;
-        } catch(e) {
-          route._errored = true;
-          console.error('[App Load Error]');
-          nextify(new Error("couldn't load api"));
-        }
-
-        return expressApp;
-      }, function (err) {
-        console.error('[App Promise Error]');
-        nextify(err);
-      });
-    }
-
-    nextify();
-  }
-
-  return {
-    api: api
-  };
-};

lib/hostname-redirects.js

@@ -2,8 +2,7 @@
 
 // TODO detect infinite redirects
 
-module.exports.compile = module.exports.sortOpts = function (opts) {
+module.exports.compile = module.exports.sortOpts = function (redirects) {
-  var redirects = opts.redirects;
   var dups = {};
   var results = {
     conflicts: {}

lib/insecure-server.js

@@ -1,50 +1,51 @@
 'use strict';
 
-module.exports.create = function (securePort, insecurePort, redirects) {
+module.exports.create = function (securePort, insecurePort, info, serverCallback) {
   var PromiseA = require('bluebird').Promise;
+  var appPromise;
+  //var app;
   var http = require('http');
-  var escapeRe;
+  var redirectives;
+
+  function useAppInsecurely(req, res) {
+    if (!appPromise) {
+      return false;
+    }
+
+    appPromise.then(function (app) {
+      req._WALNUT_SECURITY_EXCEPTION = true;
+      app(req, res);
+    });
+
+    return true;
+  }
 
   function redirectHttps(req, res) {
-    res.setHeader('Strict-Transport-Security', 'max-age=10886400; includeSubDomains; preload');
+    // Let it do this once they visit the https site
+    // res.setHeader('Strict-Transport-Security', 'max-age=10886400; includeSubDomains; preload');
 
-    var insecureRedirects;
     var host = req.headers.host || '';
     var url = req.url;
 
-    if (require('./unbrick-appcache').unbrick(req, res)) {
+    // TODO
-      return;
+    // XXX NOTE: info.conf.redirects may or may not be loaded at first
+    // the object will be modified when the config is loaded
+    if (!redirectives && info.conf.redirects) {
+      redirectives = require('./hostname-redirects').compile(info.conf.redirects);
+    }
+    if (require('./no-www').scrubTheDub(req, res, redirectives)) {
+      return true;
     }
 
-    // because I have domains for which I don't want to pay for SSL certs
+    // TODO
-    insecureRedirects = (redirects||[]).sort(function (a, b) {
+    // allow exceptions for the case of arduino and whatnot that cannot handle https?
-      var hlen = b.from.hostname.length - a.from.hostname.length;
+    // http://evothings.com/is-it-possible-to-secure-micro-controllers-used-within-iot/
-      var plen;
+    // needs ECDSA?
-      if (!hlen) {
-        plen = b.from.path.length - a.from.path.length;
-        return plen;
-      }
-      return hlen;
-    }).forEach(function (redirect) {
-      var origHost = host;
 
-      if (!escapeRe) {
+    console.warn('HARD-CODED HTTPS EXCEPTION in insecure-server.js');
-        escapeRe = require('escape-string-regexp');
+    if (/redirect-www.org/.test(host) && useAppInsecurely(req, res)) {
-      }
+      return true;
-
+    }
-      // TODO if '*' === hostname[0], omit '^'
-      host = host.replace(
-        new RegExp('^' + escapeRe(redirect.from.hostname))
-      , redirect.to.hostname
-      );
-      if (host === origHost) {
-        return;
-      }
-      url = url.replace(
-        new RegExp('^' + escapeRe(redirect.from.path))
-      , redirect.to.path
-      );
-    });
 
     var escapeHtml = require('escape-html');
     var newLocation = 'https://'
@@ -95,11 +96,14 @@ module.exports.create = function (securePort, insecurePort, redirects) {
   //
   var insecureServer;
   insecureServer = http.createServer();
-  insecureServer.on('request', redirectHttps);
   insecureServer.listen(insecurePort, function () {
-    console.log("\nListening on https://localhost:" + insecureServer.address().port);
+    console.log("\nListening on http://localhost:" + insecureServer.address().port);
-    console.log("(redirecting all traffic to https)\n");
+    console.log("(handling any explicit redirects and redirecting all other traffic to https)\n");
+    if (serverCallback) {
+      appPromise = serverCallback(null, insecureServer);
+    }
   });
+  insecureServer.on('request', redirectHttps);
 
   return PromiseA.resolve(insecureServer);
 };

lib/local-server.js

@@ -2,7 +2,7 @@
 
 // Note the odd use of callbacks (instead of promises) here
 // It's to avoid loading bluebird yet (see sni-server.js for explanation)
-module.exports.create = function (certPaths, port, serverCallback) {
+module.exports.create = function (certPaths, port, info, serverCallback) {
   function initServer(err, server) {
     var app;
     var promiseApp;
@@ -45,7 +45,7 @@ module.exports.create = function (certPaths, port, serverCallback) {
   }
 
   if (certPaths) {
-    require('./sni-server').create(certPaths, port, initServer);
+    require('./sni-server').create(certPaths, initServer);
   } else {
     initServer(null, require('http').createServer());
   }

lib/no-www.js

@@ -11,14 +11,16 @@ module.exports.scrubTheDub = function (req, res, redirectives) {
   var escapeHtml = require('escape-html');
   var newLocation;
   var safeLocation;
+  // TODO req.hostname
+  var hostname = (req.headers.host||'').split(':')[0];
 
   if (redirectives) {
-    newLocation = require('./hostname-redirects').redirectTo(req.hostname, redirectives);
+    newLocation = require('./hostname-redirects').redirectTo(hostname, redirectives);
     if (!newLocation) {
       return false;
     }
   } else {
-    newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url;
+    newLocation = 'https://' + hostname.replace(/^www\./, '') + req.url;
   }
   safeLocation = escapeHtml(newLocation);
 

lib/package-server.js

@@ -0,0 +1,385 @@
+'use strict';
+
+var escapeStringRegexp = require('escape-string-regexp');
+var staticHandlers = {};
+//var apiHandlers = {};
+
+function compileVhosts(vhostsMap) {
+  var results = {
+    patterns: []
+  , conflictsMap: {}
+  , matchesMap: {}
+  };
+
+  // compli
+  Object.keys(vhostsMap).forEach(function (key) {
+    var vhost = vhostsMap[key];
+    var bare;
+    var www;
+
+    if ('.' === vhost.hostname[0]) {
+      // for consistency
+      // TODO this should happen at the database level
+      vhost.hostname = '*' + vhost.hostname;
+    }
+
+    if ('*' === vhost.hostname[0]) {
+      // TODO check that we are not trying to redirect a tld (.com, .co.uk, .org, etc)
+      // tlds should follow the global policy
+      if (vhost.hostname[1] && '.' !== vhost.hostname[1]) {
+        // this is not a good place to throw as the consequences of a bug would be
+        // very bad, but errors should never be silent, so we'll compromise
+        console.warn("[NON-FATAL ERROR]: ignoring pattern '" + vhost.hostname + "'");
+        results.conflictsMap[vhost.hostname] = vhost;
+      }
+
+      // nix the '*' for easier matching
+      vhost.hostname = vhost.hostname.slice(1);
+      // except the default
+      if (!vhost.hostname) {
+        vhost.hostname = '*';
+      }
+      if (results.conflictsMap[vhost.hostname]) {
+        console.warn("[NON-FATAL ERROR]: duplicate entry for pattern '" + vhost.hostname + "'");
+      }
+
+      results.conflictsMap[vhost.hostname] = vhost;
+      results.patterns.push(vhost);
+      return;
+    }
+
+    //console.log('[vhost]');
+    //console.log(vhost);
+    bare = vhost.hostname.replace(/^www\./i, '');
+    www = vhost.hostname.replace(/^(www\.)?/i, 'www.');
+
+    results.matchesMap[bare] = vhost;
+    results.matchesMap[www] = vhost;
+  });
+
+  results.patterns.sort(function (a, b) {
+    return b.id.length - a.id.length;
+  });
+
+  return results;
+}
+
+function loadPages(pkgConf, route, req, res, next) {
+  var PromiseA = require('bluebird');
+  var fs = require('fs');
+  var path = require('path');
+  var pkgpath = path.join(pkgConf.apppath, (route.app.package || route.app.id), (route.app.version || ''));
+
+  // TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc)
+
+  function handlePromise(p) {
+    p.then(function (app) {
+      app(req, res, next);
+      route._app = app;
+    }, function (err) {
+      console.error('[App Promise Error]');
+      next(err);
+    });
+  }
+
+  if (staticHandlers[pkgpath]) {
+    route._app = staticHandlers[pkgpath];
+    route._app(req, res, next);
+    return;
+  }
+
+  if (!route._promise_app) {
+    route._promise_app = new PromiseA(function (resolve, reject) {
+      fs.exists(pkgpath, function (exists) {
+        if (!exists) {
+          reject(new Error("package is registered but does not exist"));
+          return;
+        }
+
+        //console.log('[static mount]', pkgpath);
+        resolve(require('serve-static')(pkgpath));
+      });
+    });
+  }
+
+  handlePromise(route._promise_app);
+}
+
+function getApi(pkgConf, pkgDeps, route) {
+  var PromiseA = require('bluebird');
+  var path = require('path');
+  var pkgpath = path.join(pkgConf.apipath, route.api.id/*, (route.api.version || '')*/);
+
+  // TODO needs some version stuff (which would also allow hot-loading of updates)
+  // TODO version could be tied to sha256sum
+
+  return new PromiseA(function (resolve, reject) {
+    var myApp;
+    var ursa;
+    var promise;
+
+    // TODO dynamic requires are a no-no
+    // can we statically generate a require-er? on each install?
+    // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
+    // requirer[pkgpath]()
+    myApp = pkgDeps.express();
+    myApp.disable('x-powered-by');
+    if (pkgDeps.app.get('trust proxy')) {
+      myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
+    }
+    if (!pkgConf.pubkey) {
+      /*
+        return ursa.createPrivateKey(pem, password, encoding);
+        var pem = myKey.toPrivatePem();
+        return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
+        });
+      */
+      ursa = require('ursa');
+      pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
+      pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
+    }
+
+    try {
+      route._apipkg = require(path.join(pkgpath, 'package.json'));
+      route._apiname = route._apipkg.name;
+      promise = require(pkgpath).create(pkgConf, pkgDeps, myApp);
+    } catch(e) {
+      reject(e);
+      return;
+    }
+
+    promise.then(function () {
+      // TODO give pub/priv pair for app and all public keys
+      // route._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
+      route._api = require('express')();
+      route._api_app = myApp;
+      // TODO fix backwards compat
+      // /api/com.example.foo (no change)
+      route._api.use('/', route._api_app);
+      // /api/com.example.foo => /
+      route._api.use('/api/' + route.api.id, function (req, res, next) {
+        //console.log('api mangle 2:', '/api/' + route.api.id, req.url);
+        route._api_app(req, res, next);
+      });
+      // /api/com.example.foo => /api
+      route._api.use('/', function (req, res, next) {
+        req.url = '/api' + req.url.slice(('/api/' + route.api.id).length);
+        //console.log('api mangle 3:', req.url);
+        route._api_app(req, res, next);
+      });
+      resolve(route._api);
+    }, reject);
+  });
+}
+
+function loadApi(pkgConf, pkgDeps, route) {
+  function handlePromise(p) {
+    return p.then(function (api) {
+      route._api = api;
+      return api;
+    });
+  }
+
+  if (!route._promise_api) {
+    route._promise_api = getApi(pkgConf, pkgDeps, route);
+  }
+
+  return handlePromise(route._promise_api);
+}
+
+function layerItUp(pkgConf, router, req, res, next) {
+  var nexti = -1;
+  // Layers exist so that static apps can use them like a virtual filesystem
+  // i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html
+  // or perhaps some dynamic content (like application cache)
+  function nextify(err) {
+    var route;
+    nexti += 1;
+
+    if (err) {
+      next(err);
+      return;
+    }
+
+    // shortest to longest
+    //route = packages.pop();
+    // longest to shortest
+    route = router.packages[nexti];
+    if (!route) {
+      next();
+      return;
+    }
+
+    if (!route.app) {
+      // new Error("no Static App is registered for the specified path")
+      nextify();
+      return;
+    }
+    if (route._app) {
+      route._app(req, res, nextify);
+      return;
+    }
+
+    // could attach to req.{ pkgConf, pkgDeps, Services}
+    loadPages(pkgConf, route, req, res, next);
+  }
+
+  nextify();
+}
+
+function runApi(opts, router, req, res, next) {
+  var pkgConf = opts.config;
+  var pkgDeps = opts.deps;
+  //var Services = opts.Services;
+  var route;
+
+  // TODO compile packagesMap
+  // TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name)
+  router.packages.some(function (_route) {
+    var pathname = router.pathname;
+    if ('/' === pathname) {
+      pathname = '';
+    }
+
+    // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
+    if (!_route._api_re) {
+      _route._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _route.api.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)');
+      //console.log('[api re 2]', _route._api_re);
+    }
+    if (_route._api_re.test(req.url)) {
+      route = _route;
+      return true;
+    }
+  });
+
+  if (!route) {
+    //console.log('[no api route]');
+    next();
+    return;
+  }
+  Object.defineProperty(req, 'appId', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+    // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
+    // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
+  , value: route.id
+  });
+  Object.defineProperty(req, 'appConfig', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+  , value: {}       // TODO just the app-scoped config
+  });
+  Object.defineProperty(req, 'appDeps', {
+    enumerable: true
+  , configurable: false
+  , writable: false
+  , value: {}       // TODO app-scoped deps
+                    // i.e. when we need to use things such as stripe id
+                    // without exposing them to the app
+  });
+
+  //
+  // TODO user authentication should go right about here
+  //
+
+  //
+  // TODO freeze objects for passing them into app
+  //
+
+  if (route._api) {
+    route._api(req, res, next);
+    return;
+  }
+
+  loadApi(pkgConf, pkgDeps, route).then(function (api) {
+    api(req, res, next);
+  }, function (err) {
+    console.error('[App Promise Error]');
+    next(err);
+  });
+}
+
+function mapToApp(opts, req, res, next) {
+  // opts = { config, deps, services }
+  var vhost;
+  var router;
+  var pkgConf = opts.config;
+
+  if (!pkgConf.vhostConf) {
+    pkgConf.vhostConf = compileVhosts(pkgConf.vhostsMap);
+  }
+
+  //console.log('req.hostname');
+  //console.log(req.hostname);
+
+  //console.log(Object.keys(pkgConf.vhostConf.matchesMap));
+
+  // TODO www vs no-www?
+  vhost = pkgConf.vhostConf.matchesMap[req.hostname];
+
+  if (!vhost) {
+    pkgConf.vhostConf.patterns.some(function (pkg) {
+      if ('*' === pkg.id || pkg.id === req.hostname.slice(req.hostname.length - pkg.id.length)) {
+        vhost = pkg;
+        return true;
+      }
+    });
+  }
+
+  if (!vhost) {
+    next();
+    return;
+  }
+
+  //console.log('vhost');
+  //console.log(vhost);
+
+  // TODO don't modify route here (or in subloaders), modify some other variable instead
+  // TODO precompile RegExps and pre-sort app vs api
+  vhost.pathnames.some(function (routes) {
+    var pathname = routes.pathname;
+    if ('/' === pathname) {
+      pathname = '';
+    }
+
+    if (!routes._re_app) {
+      routes._re_app = new RegExp(escapeStringRegexp(pathname) + '(#|\\/|\\?|$)');
+      //console.log('[static re]', routes._re_app);
+    }
+
+    if (!routes._re_api) {
+      // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
+      routes._re_api = new RegExp(escapeStringRegexp(pathname + '/api/') + '([\\w\\.\\-]+)(\\/|\\?|$)');
+      //console.log('[api re]', routes._re_api);
+    }
+
+    if (routes._re_app.test(req.url)) {
+      router = routes;
+      return true;
+    }
+
+    // no need to test for api yet as it is a postfix
+  });
+
+  if (!router) {
+    //console.log('[no router for]', req.url);
+    next();
+    return;
+  }
+
+  if (!router._re_api.test(req.url)) {
+    //console.log('[static router]');
+    //console.log(router._re_api, req.url);
+    layerItUp(pkgConf, router, req, res, next);
+    return;
+  }
+
+  //console.log('[api router]', req.url);
+  return runApi(opts, router, req, res, next);
+}
+
+module.exports.runApi = runApi;
+module.exports.compileVhosts = compileVhosts;
+module.exports.mapToApp = mapToApp;

lib/schemes-config.js

@@ -72,29 +72,29 @@ function deserialize(results) {
   return config;
 }
 
+function sortApps(a, b) {
+  // hlen isn't important in this current use of the sorter,
+  // but is important for an alternate version
+  var hlen = b.hostname.length - a.hostname.length;
+  var plen = b.pathname.length - a.pathname.length;
+
+  // A directory could be named example.com, example.com# example.com##
+  // to indicate order of preference (for API addons, for example)
+  var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
+
+  if (!hlen) {
+    if (!plen) {
+      return dlen;
+    }
+    return plen;
+  }
+  return hlen;
+}
+
 function getVhostsMap(config) {
   var vhosts = [];
   var vhostsMap = {};
 
-  function sortApps(a, b) {
-    // hlen isn't important in this current use of the sorter,
-    // but is important for an alternate version
-    var hlen = b.hostname.length - a.hostname.length;
-    var plen = b.pathname.length - a.pathname.length;
-
-    // A directory could be named example.com, example.com# example.com##
-    // to indicate order of preference (for API addons, for example)
-    var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
-
-    if (!hlen) {
-      if (!plen) {
-        return dlen;
-      }
-      return plen;
-    }
-    return hlen;
-  }
-
   Object.keys(config.domains).forEach(function (domainname) {
     var domain = config.domains[domainname];
     var info = getDomainInfo(domainname);
@@ -110,15 +110,15 @@ function getVhostsMap(config) {
 
   vhosts.forEach(function (domain) {
     if (!vhostsMap[domain.hostname]) {
-      vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] };
+      vhostsMap[domain.hostname] = { hostname: domain.hostname, id: domain.id, pathnamesMap: {}, pathnames: [] };
     }
 
     if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) {
-      vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] };
+      vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, packages: [] };
       vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]);
     }
 
-    vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain);
+    vhostsMap[domain.hostname].pathnamesMap[domain.pathname].packages.push(domain);
   });
 
   return vhostsMap;
@@ -135,7 +135,7 @@ module.exports.create = function (db) {
     //
     { tablename: 'apis'
     , idname: 'id'      // io.lds.auth, com.daplie.radio
-    , unique: ['id'] 
+    , unique: ['id']
       // name // LDS Account, Radio
     , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name']
     }

lib/worker.js

@@ -10,8 +10,6 @@ module.exports.create = function (webserver, info, state) {
   //var vhostsdir = path.join(__dirname, 'vhosts');
   var express = require('express-lazy');
   var app = express();
-  var apiHandler;
-  var Services;
   var memstore;
   var sqlstores = {};
   var models = {};
@@ -37,6 +35,7 @@ module.exports.create = function (webserver, info, state) {
     , ipcKey: info.conf.ipcKey
   });
   var cstore = require('cluster-store');
+  var redirectives;
 
   app.disable('x-powered-by');
   if (info.conf.trustProxy) {
@@ -45,6 +44,8 @@ module.exports.create = function (webserver, info, state) {
     //app.set('trust proxy', function (ip) { console.log('[ip]', ip); return true; });
   } else {
     console.info('[DO NOT trust proxy]');
+    // TODO make sure the gzip module loads if there isn't a proxy gzip-ing for us
+    // app.use(compression())
   }
 
   /*
@@ -85,12 +86,21 @@ module.exports.create = function (webserver, info, state) {
     // TODO test if this is even necessary
     host = host.toLowerCase();
 
-    if (!/^www\./.test(host)) {
+    // TODO this should be hot loadable / changeable
+    if (!redirectives && info.conf.redirects) {
+      redirectives = require('./hostname-redirects').compile(info.conf.redirects);
+    }
+
+    if (!/^www\./.test(host) && !redirectives) {
       next();
       return;
     }
 
-    require('./no-www').scrubTheDub(req, res);
+    // TODO misnomer, handles all exact redirects
+    if (!require('./no-www').scrubTheDub(req, res, redirectives)) {
+      next();
+      return;
+    }
   }
 
   function caddyBugfix(req, res, next) {
@@ -108,6 +118,7 @@ module.exports.create = function (webserver, info, state) {
     next();
   }
 
+  // TODO misnomer, this can handle nowww, yeswww, and exact hostname redirects
   app.use('/', scrubTheDub);
   app.use('/', caddyBugfix);
 
@@ -153,21 +164,34 @@ module.exports.create = function (webserver, info, state) {
         // TODO the core needs to be replacable in one shot
         // rm -rf /tmp/walnut/; tar xvf -C /tmp/walnut/; mv /srv/walnut /srv/walnut.{{version}}; mv /tmp/walnut /srv/
         // this means that any packages must be outside, perhaps /srv/walnut/{boot,core,packages}
-        var apiConf = {
+        var pkgConf = {
           apppath: path.join(__dirname, '..', '..', 'packages', 'apps') + path.sep
         , apipath: path.join(__dirname, '..', '..', 'packages', 'apis') + path.sep
         , servicespath: path.join(__dirname, '..', '..', 'packages', 'services')
         , vhostsMap: vhostsMap
+        , vhostPatterns: null
         , server: webserver
         , externalPort: info.conf.externalPort
         , primaryNameserver: info.conf.primaryNameserver
         , nameservers: info.conf.nameservers
         , privkey: info.conf.privkey
         , pubkey: info.conf.pubkey
+        , redirects: info.conf.redirects
         , apiPrefix: '/api'
         };
-
+        var pkgDeps = {
-        Services = require('./services-loader').create(apiConf, {
+          memstore: memstore
+        , sqlstores: sqlstores
+        , clientSqlFactory: clientFactory
+        , systemSqlFactory: systemFactory
+        //, handlePromise: require('./lib/common').promisableRequest;
+        //, handleRejection: require('./lib/common').rejectableRequest;
+        //, localPort: info.conf.localPort
+        , Promise: PromiseA
+        , express: express
+        , app: app
+        };
+        var Services = require('./services-loader').create(pkgConf, {
           memstore: memstore
         , sqlstores: sqlstores
         , clientSqlFactory: clientFactory
@@ -175,12 +199,7 @@ module.exports.create = function (webserver, info, state) {
         , Promise: PromiseA
         });
 
-        function handleApi(req, res, next) {
+        function handlePackages(req, res, next) {
-          if (!/^\/api/.test(req.url)) {
-            next();
-            return;
-          }
-
           // TODO move to caddy parser?
           if (/(^|\.)proxyable\./.test(req.hostname)) {
             // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
@@ -189,34 +208,11 @@ module.exports.create = function (webserver, info, state) {
             req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
           }
 
-          if (apiHandler) {
+          require('./package-server').mapToApp({
-            /*
+            config: pkgConf
-            if (apiHandler.then) {
+          , deps: pkgDeps
-              apiHandler.then(function (myApp) {
+          , services: Services
-                myApp(req, res, next);
+          }, req, res, next);
-              });
-              return;
-            }
-            */
-
-            apiHandler(req, res, next);
-            return;
-          }
-
-          apiHandler = require('./api-server').create(apiConf, {
-            memstore: memstore
-          , sqlstores: sqlstores
-          , clientSqlFactory: clientFactory
-          , systemSqlFactory: systemFactory
-          //, handlePromise: require('./lib/common').promisableRequest;
-          //, handleRejection: require('./lib/common').rejectableRequest;
-          //, localPort: info.conf.localPort
-          , Promise: PromiseA
-          , express: express
-          , app: app
-          }, Services).api;
-
-          apiHandler(req, res, next);
         }
 
         // TODO recase
@@ -246,7 +242,8 @@ module.exports.create = function (webserver, info, state) {
           //}))
           .use(require('connect-send-error').error())
           ;
-        app.use('/', handleApi);
+
+        app.use('/', handlePackages);
         app.use('/', function (err, req, res, next) {
           console.error('[Error Handler]');
           console.error(err.stack);

tests/hostname-redirects.js

@@ -55,9 +55,9 @@ var domains = {
 , 'www.example.com': false
 };
 
-var redirects = sortOpts(opts);
+var redirects = sortOpts(opts.redirects);
 
-console.log(redirects);
+//console.log(redirects);
 
 Object.keys(domains).forEach(function (domain, i) {
   var redir = domains[domain];
@@ -68,5 +68,8 @@ Object.keys(domains).forEach(function (domain, i) {
   }
 });
 
+console.log("TODO: we do not yet detect infinite loop redirects");
+console.log("");
+console.log("");
 console.log("Didn't throw any errors. Must have worked, eh?");
-console.log("TODO: detect and report infinite redirects");
+console.log("");