lazier load, faster https listen

lib/vhost-sni-server.js

@@ -1,125 +1,31 @@
 'use strict';
  
-var https           = require('https');
-var PromiseA        = require('bluebird').Promise;
-var forEachAsync    = require('foreachasync').forEachAsync.create(PromiseA);
-var fs              = require('fs');
-var path            = require('path');
-var crypto          = require('crypto');
-var connect         = require('connect');
-var vhost           = require('vhost');
-
 module.exports.create = function (securePort, certsPath, vhostsdir) {
-  function getDomainInfo(apppath) {
+  var PromiseA = require('bluebird').Promise;
-    var parts = apppath.split(/[#%]+/);
+  var https = require('https');
-    var hostname = parts.shift();
+  var fs = require('fs');
-    var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, '');
+  var path = require('path');
-
-    return {
-      hostname: hostname
-    , pathname: pathname
-    , dirpathname: parts.join('#')
-    , dirname: apppath
-    , isRoot: apppath === hostname
-    };
-  }
-
-  function loadDomainMounts(domaininfo) {
-    var appContext;
-
-    // should order and group by longest domain, then longest path
-    if (!domainMergeMap[domaininfo.hostname]) {
-      // create an connect / express app exclusive to this domain
-      // TODO express??
-      domainMergeMap[domaininfo.hostname] = {
-        hostname: domaininfo.hostname
-      , apps: connect()
-      , mountsMap: {}
-      };
-      domainMerged.push(domainMergeMap[domaininfo.hostname]);
-    }
-
-    if (domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname]) {
-      return;
-    }
-
-    domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] = function (req, res, next) {
-      if (appContext) {
-        appContext(req, res, next);
-        return;
-      }
-
-      console.log('[log] LOADING "' + domaininfo.hostname + '/' + domaininfo.pathname + '"');
-      getAppContext(domaininfo).then(function (localApp) {
-        // Note: pathname should NEVER have a leading '/' on its own
-        // we always add it explicitly
-        try {
-          domainMergeMap[domaininfo.hostname].apps.use('/' + domaininfo.pathname, localApp);
-          console.info('Loaded ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname);
-          appContext = localApp;
-          appContext(req, res, next);
-        } catch(e) {
-          console.error('[ERROR] ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname);
-          console.error(e);
-          res.send('{ "error": { "message": "[ERROR] could not load '
-            + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname
-            + 'or default error app." } }');
-        }
-      });
-    };
-    domainMergeMap[domaininfo.hostname].apps.use(
-      '/' + domaininfo.pathname
-    , domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] 
-    );
-
-    return PromiseA.resolve();
-  }
-
-  function loadDomainVhosts() {
-    domainMerged.forEach(function (domainApp) {
-      console.log('[log] merged ' + domainApp.hostname);
-      app.use(vhost(domainApp.hostname, domainApp.apps));
-      app.use(vhost('www.' + domainApp.hostname, domainApp.apps));
-    });
-  }
-
-  function readNewVhosts() {
-    return fs.readdirSync(vhostsdir).filter(function (node) {
-      // not a hidden or private file
-      return '.' !== node[0] && '_' !== node[0];
-    }).map(getDomainInfo).sort(function (a, b) {
-      var hlen = b.hostname.length - a.hostname.length;
-      var plen = b.pathname.length - a.pathname.length;
-
-      // A directory could be named example.com, example.com# example.com##
-      // to indicate order of preference (for API addons, for example)
-      var dlen = b.dirname.length - a.dirname.length;
-      if (!hlen) {
-        if (!plen) {
-          return dlen;
-        }
-        return plen;
-      }
-      return plen;
-    });
-  }
-
-    // connect / express app
-  var app             = connect();
-
-    // SSL Server
-  var secureContexts  = {};
   var dummyCerts;
-  var secureOpts;
+  var secureContexts  = {};
-  var secureServer;
 
-  /*
+  function loadDummyCerts() {
-  var rootDomains     = domains.filter(function (domaininfo) {
+    if (dummyCerts) {
-                          return domaininfo.isRoot;
+      return dummyCerts;
-                        });
+    }
-  */
+
-  var domainMergeMap  = {};
+    dummyCerts = {
-  var domainMerged    = [];
+      key:          fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.key.pem'))
+    , cert:         fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.crt.pem'))
+    , ca:           fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) {
+                      return /crt\.pem$/.test(node);
+                    }).map(function (node) {
+                      console.log('[log dummy ca]', node);
+                      return fs.readFileSync(path.join(certsPath, 'ca', node));
+                    })
+    };
+
+    return dummyCerts;
+  }
 
   function createSecureContext(certs) {
     // workaround for v0.12 / v1.2 backwards compat
@@ -130,61 +36,205 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
     }
   }
 
-  function getDummyAppContext(err, msg) {
+  function createPromiseApps(secureServer) {
-    console.error('[ERROR] getDummyAppContext');
+    return new PromiseA(function (resolve) {
-    console.error(err);
+      var forEachAsync    = require('foreachasync').forEachAsync.create(PromiseA);
-    console.error(msg);
+      var connect         = require('connect');
-    return function (req, res) {
+      var app             = connect();
-      res.end('{ "error": { "message": "' + msg + '" } }');
+      var vhost           = require('vhost');
-    }
-  }
 
-  function getAppContext(domaininfo) {
+      var domainMergeMap  = {};
-    var localApp;
+      var domainMerged    = [];
 
-    try {
+      function getDomainInfo(apppath) {
-      // TODO live reload required modules
+        var parts = apppath.split(/[#%]+/);
-      localApp = require(path.join(vhostsdir, domaininfo.dirname, 'app.js'));
+        var hostname = parts.shift();
-      if (localApp.create) {
+        var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, '');
-        // TODO read local config.yml and pass it in
+
-        // TODO pass in websocket
+        return {
-        localApp = localApp.create(/*config*/);
+          hostname: hostname
-        if (!localApp) {
+        , pathname: pathname
-          return getDummyAppContext(null, "[ERROR] no app was returned by app.js for " + domaininfo.driname);
+        , dirpathname: parts.join('#')
-        }
+        , dirname: apppath
+        , isRoot: apppath === hostname
+        };
       }
-      if (!localApp.then) {
+
-        localApp = PromiseA.resolve(localApp);
+      function loadDomainMounts(domaininfo) {
-      } else {
+        var appContext;
-        return localApp.catch(function (e) {
+
-          return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname);
+        // should order and group by longest domain, then longest path
+        if (!domainMergeMap[domaininfo.hostname]) {
+          // create an connect / express app exclusive to this domain
+          // TODO express??
+          domainMergeMap[domaininfo.hostname] = {
+            hostname: domaininfo.hostname
+          , apps: connect()
+          , mountsMap: {}
+          };
+          domainMerged.push(domainMergeMap[domaininfo.hostname]);
+        }
+
+        if (domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname]) {
+          return;
+        }
+
+        domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] = function (req, res, next) {
+          if (appContext) {
+            appContext(req, res, next);
+            return;
+          }
+
+          console.log('[log] LOADING "' + domaininfo.hostname + '/' + domaininfo.pathname + '"');
+          getAppContext(domaininfo).then(function (localApp) {
+            // Note: pathname should NEVER have a leading '/' on its own
+            // we always add it explicitly
+            try {
+              domainMergeMap[domaininfo.hostname].apps.use('/' + domaininfo.pathname, localApp);
+              console.info('Loaded ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname);
+              appContext = localApp;
+              appContext(req, res, next);
+            } catch(e) {
+              console.error('[ERROR] ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname);
+              console.error(e);
+              res.end('{ "error": { "message": "[ERROR] could not load '
+                + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname
+                + 'or default error app." } }');
+            }
+          });
+        };
+        domainMergeMap[domaininfo.hostname].apps.use(
+          '/' + domaininfo.pathname
+        , domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] 
+        );
+
+        return PromiseA.resolve();
+      }
+
+      function readNewVhosts() {
+        return fs.readdirSync(vhostsdir).filter(function (node) {
+          // not a hidden or private file
+          return '.' !== node[0] && '_' !== node[0];
+        }).map(getDomainInfo).sort(function (a, b) {
+          var hlen = b.hostname.length - a.hostname.length;
+          var plen = b.pathname.length - a.pathname.length;
+
+          // A directory could be named example.com, example.com# example.com##
+          // to indicate order of preference (for API addons, for example)
+          var dlen = b.dirname.length - a.dirname.length;
+          if (!hlen) {
+            if (!plen) {
+              return dlen;
+            }
+            return plen;
+          }
+          return plen;
         });
       }
-    } catch(e) {
-      localApp = getDummyAppContext(e, "[ERROR] could not load app.js for " + domaininfo.dirname);
-      localApp = PromiseA.resolve(localApp);
 
-      return localApp;
+      function getDummyAppContext(err, msg) {
-    }
+        console.error('[ERROR] getDummyAppContext');
+        console.error(err);
+        console.error(msg);
+        return function (req, res) {
+          res.end('{ "error": { "message": "' + msg + '" } }');
+        }
+      }
 
-    return localApp;
+      function getAppContext(domaininfo) {
-  }
+        var localApp;
 
-  function loadDummyCerts() {
+        try {
-    var certs = {
+          // TODO live reload required modules
-      key:          fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.key.pem'))
+          localApp = require(path.join(vhostsdir, domaininfo.dirname, 'app.js'));
-    , cert:         fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.crt.pem'))
+          if (localApp.create) {
-    , ca:           fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) {
+            // TODO read local config.yml and pass it in
-                      return /crt\.pem$/.test(node);
+            // TODO pass in websocket
-                    }).map(function (node) {
+            localApp = localApp.create(/*config*/);
-                      console.log('[log dummy ca]', node);
+            if (!localApp) {
-                      return fs.readFileSync(path.join(certsPath, 'ca', node));
+              return getDummyAppContext(null, "[ERROR] no app was returned by app.js for " + domaininfo.driname);
-                    })
+            }
-    };
+          }
-    secureContexts.dummy = createSecureContext(dummyCerts);
+          if (!localApp.then) {
-    dummyCerts = certs;
+            localApp = PromiseA.resolve(localApp);
-    return certs
+          } else {
-  }
+            return localApp.catch(function (e) {
+              return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname);
+            });
+          }
+        } catch(e) {
+          localApp = getDummyAppContext(e, "[ERROR] could not load app.js for " + domaininfo.dirname);
+          localApp = PromiseA.resolve(localApp);
+
+          return localApp;
+        }
+
+        return localApp;
+      }
+
+      function loadDomainVhosts() {
+        domainMerged.forEach(function (domainApp) {
+          console.log('[log] loaded mounts for domain ' + domainApp.hostname);
+          app.use(vhost(domainApp.hostname, domainApp.apps));
+          app.use(vhost('www.' + domainApp.hostname, domainApp.apps));
+        });
+      }
+
+      function hotloadApp(req, res, next) {
+        var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA);
+        var vhost = (req.headers.host || '').split(':')[0];
+
+        // the matching domain didn't catch it
+        if (domainMergeMap[vhost]) {
+          next();
+          return;
+        }
+
+        return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(function () {
+          // no matching domain was added
+          if (!domainMergeMap[vhost]) {
+            next();
+            return;
+          }
+
+          return forEachAsync(domainMergeMap[vhost].apps, function (fn) {
+            return new PromiseA(function (resolve) {
+              try {
+                fn(req, res, function (err) {
+                  if (err) {
+                    reject(err);
+                  }
+                  resolve();
+                });
+              } catch(e) {
+                reject(e);
+              }
+            });
+          }).catch(function (e) {
+            next(e);
+          });
+        });
+
+        /*
+        // TODO loop through mounts and see if any fit
+        domainMergeMap[vhost].mountsMap['/' + domaininfo.dirpathname]
+        if (!domainMergeMap[domaininfo.hostname]) {
+          // TODO reread directories
+        }
+        */
+      }
+
+
+      // TODO pre-cache these once the server has started?
+      // return forEachAsync(rootDomains, loadCerts);
+      // TODO load these even more lazily
+      return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(function () {
+        app.use(hotloadApp);
+        resolve(app);
+        return;
+      });
+    });
+  };
 
   function loadCerts(domainname) {
     // TODO make async
@@ -219,7 +269,7 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
     }
 
     try {
-      secureContexts[domainname] = crypto.createCredentials(secOpts).context;
+      secureContexts[domainname] = createSecureContext(secOpts).context;
     } catch(err) {
       console.error("[ERROR] Certificates in '" + certsPath + "' could not be used:");
       console.error(err);
@@ -229,36 +279,9 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
     return secureContexts[domainname];
   }
 
-  // TODO pre-cache these once the server has started?
+  function createSecureServer() {
-  // return forEachAsync(rootDomains, loadCerts);
+    var dummyCerts = loadDummyCerts();
-  // TODO load these even more lazily
+    var secureOpts = {
-  return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(runServer);
-
-  function hotloadApp(req, res, next) {
-    var vhost = (req.headers.host || '').split(':')[0];
-
-    if (!domainMergeMap[vhost]) {
-      // TODO reread directories
-    }
-
-    /*
-    // TODO loop through mounts and see if any fit
-    domainMergeMap[vhost].mountsMap['/' + domaininfo.dirpathname]
-    if (!domainMergeMap[domaininfo.hostname]) {
-      // TODO reread directories
-    }
-    */
-    // TODO hot load all-the-things
-    next();
-  }
-
-  app.use(hotloadApp);
-
-  function runServer() {
-    //provide a SNICallback when you create the options for the https server
-
-    loadDummyCerts();
-    secureOpts = {
                     // fallback / default dummy certs
       key:          dummyCerts.key
     , cert:         dummyCerts.cert
@@ -270,10 +293,14 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
       secureOpts.SNICallback = function (domainname, cb) {
         console.log('[log] SNI:', domainname);
 
+        if (!secureContexts.dummy) {
+          secureContexts.dummy = createSecureContext(dummyCerts);
+        }
+
         var secureContext = secureContexts[domainname]
           || loadCerts(domainname)
           || secureContexts.dummy
-          || createSecureContext(dummyCerts)
+          //|| createSecureContext(dummyCerts)
           //|| createSecureContext(loadDummyCerts())
           ;
 
@@ -286,28 +313,47 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
           secureContext = createSecureContext(loadDummyCerts());
         }
 
-        console.log('[log]', secureContext);
-
         // workaround for v0.12 / v1.2 backwards compat bug
         if ('function' === typeof cb) {
-          console.log('using sni callback callback');
           cb(null, secureContext);
         } else {
-          console.log('NOT using sni callback callback');
           return secureContext;
         }
       };
     }
+
     addSniWorkaroundCallback();
-
+    return https.createServer(secureOpts);
-    secureServer = https.createServer(secureOpts);
-    secureServer.on('request', function (req, res) {
-      app(req, res);
-    });
-    secureServer.listen(securePort, function () {
-      console.log("Listening on https://localhost:" + secureServer.address().port);
-    });
-
-    return PromiseA.resolve();
   }
+
+  function runServer() {
+    return new PromiseA(function (resolve) {
+      var secureServer = createSecureServer();
+      var promiseApps;
+
+      function loadPromise() {
+        if (!promiseApps) {
+          promiseApps = createPromiseApps(secureServer);
+        }
+        return promiseApps;
+      }
+
+      secureServer.listen(securePort, function () {
+        resolve(secureServer);
+        console.log("Listening on https://localhost:" + secureServer.address().port, '\n');
+        loadPromise();
+      });
+
+      // Get up and listening as absolutely quickly as possible
+      secureServer.on('request', function (req, res) {
+        loadPromise().then(function (app) {
+          app(req, res);
+        });
+      });
+
+      return secureServer;
+    });
+  }
+
+  return runServer();
 }

package.json

@@ -51,6 +51,7 @@
     "express": "^4.11.2",
     "express-session": "^1.10.3",
     "foreachasync": "^5.0.5",
+    "http-proxy": "^1.8.1",
     "human-readable-ids": "^1.0.1",
     "nat-pmp": "0.0.3",
     "node-acme": "0.0.1",

walnut.js

@@ -37,8 +37,6 @@ function phoneHome() {
     }
   ];
 
-  //
-  /*
   // TODO return a middleware
   holepunch.run(require('./redirects.json').reduce(function (all, redirect) {
     if (!all[redirect.from.hostname]) {
@@ -54,7 +52,8 @@ function phoneHome() {
   }, []), ports).catch(function () {
     console.error("Couldn't phone home. Oh well");
   });
-  //*/
 }
 require('./lib/insecure-server').create(securePort, insecurePort, redirects);
-require('./lib/vhost-sni-server.js').create(securePort, certsPath, vhostsdir).then(phoneHome);
+require('./lib/vhost-sni-server.js').create(securePort, certsPath, vhostsdir)
+  //.then(phoneHome)
+  ;