From 90640ee8e7cd4f80418327416fe2449c6caae000 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Fri, 20 Feb 2015 21:21:37 +0000 Subject: [PATCH] lazier load, faster https listen --- lib/vhost-sni-server.js | 466 ++++++++++++++++++++++------------------ package.json | 1 + walnut.js | 7 +- 3 files changed, 260 insertions(+), 214 deletions(-) diff --git a/lib/vhost-sni-server.js b/lib/vhost-sni-server.js index 3fc41c2..2cd0774 100644 --- a/lib/vhost-sni-server.js +++ b/lib/vhost-sni-server.js @@ -1,125 +1,31 @@ 'use strict'; -var https = require('https'); -var PromiseA = require('bluebird').Promise; -var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA); -var fs = require('fs'); -var path = require('path'); -var crypto = require('crypto'); -var connect = require('connect'); -var vhost = require('vhost'); - module.exports.create = function (securePort, certsPath, vhostsdir) { - function getDomainInfo(apppath) { - var parts = apppath.split(/[#%]+/); - var hostname = parts.shift(); - var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, ''); - - return { - hostname: hostname - , pathname: pathname - , dirpathname: parts.join('#') - , dirname: apppath - , isRoot: apppath === hostname - }; - } - - function loadDomainMounts(domaininfo) { - var appContext; - - // should order and group by longest domain, then longest path - if (!domainMergeMap[domaininfo.hostname]) { - // create an connect / express app exclusive to this domain - // TODO express?? - domainMergeMap[domaininfo.hostname] = { - hostname: domaininfo.hostname - , apps: connect() - , mountsMap: {} - }; - domainMerged.push(domainMergeMap[domaininfo.hostname]); - } - - if (domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname]) { - return; - } - - domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] = function (req, res, next) { - if (appContext) { - appContext(req, res, next); - return; - } - - console.log('[log] LOADING "' + domaininfo.hostname + '/' + domaininfo.pathname + '"'); - getAppContext(domaininfo).then(function (localApp) { - // Note: pathname should NEVER have a leading '/' on its own - // we always add it explicitly - try { - domainMergeMap[domaininfo.hostname].apps.use('/' + domaininfo.pathname, localApp); - console.info('Loaded ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname); - appContext = localApp; - appContext(req, res, next); - } catch(e) { - console.error('[ERROR] ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname); - console.error(e); - res.send('{ "error": { "message": "[ERROR] could not load ' - + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname - + 'or default error app." } }'); - } - }); - }; - domainMergeMap[domaininfo.hostname].apps.use( - '/' + domaininfo.pathname - , domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] - ); - - return PromiseA.resolve(); - } - - function loadDomainVhosts() { - domainMerged.forEach(function (domainApp) { - console.log('[log] merged ' + domainApp.hostname); - app.use(vhost(domainApp.hostname, domainApp.apps)); - app.use(vhost('www.' + domainApp.hostname, domainApp.apps)); - }); - } - - function readNewVhosts() { - return fs.readdirSync(vhostsdir).filter(function (node) { - // not a hidden or private file - return '.' !== node[0] && '_' !== node[0]; - }).map(getDomainInfo).sort(function (a, b) { - var hlen = b.hostname.length - a.hostname.length; - var plen = b.pathname.length - a.pathname.length; - - // A directory could be named example.com, example.com# example.com## - // to indicate order of preference (for API addons, for example) - var dlen = b.dirname.length - a.dirname.length; - if (!hlen) { - if (!plen) { - return dlen; - } - return plen; - } - return plen; - }); - } - - // connect / express app - var app = connect(); - - // SSL Server - var secureContexts = {}; + var PromiseA = require('bluebird').Promise; + var https = require('https'); + var fs = require('fs'); + var path = require('path'); var dummyCerts; - var secureOpts; - var secureServer; + var secureContexts = {}; - /* - var rootDomains = domains.filter(function (domaininfo) { - return domaininfo.isRoot; - }); - */ - var domainMergeMap = {}; - var domainMerged = []; + function loadDummyCerts() { + if (dummyCerts) { + return dummyCerts; + } + + dummyCerts = { + key: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.key.pem')) + , cert: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.crt.pem')) + , ca: fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) { + return /crt\.pem$/.test(node); + }).map(function (node) { + console.log('[log dummy ca]', node); + return fs.readFileSync(path.join(certsPath, 'ca', node)); + }) + }; + + return dummyCerts; + } function createSecureContext(certs) { // workaround for v0.12 / v1.2 backwards compat @@ -130,61 +36,205 @@ module.exports.create = function (securePort, certsPath, vhostsdir) { } } - function getDummyAppContext(err, msg) { - console.error('[ERROR] getDummyAppContext'); - console.error(err); - console.error(msg); - return function (req, res) { - res.end('{ "error": { "message": "' + msg + '" } }'); - } - } + function createPromiseApps(secureServer) { + return new PromiseA(function (resolve) { + var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA); + var connect = require('connect'); + var app = connect(); + var vhost = require('vhost'); - function getAppContext(domaininfo) { - var localApp; + var domainMergeMap = {}; + var domainMerged = []; - try { - // TODO live reload required modules - localApp = require(path.join(vhostsdir, domaininfo.dirname, 'app.js')); - if (localApp.create) { - // TODO read local config.yml and pass it in - // TODO pass in websocket - localApp = localApp.create(/*config*/); - if (!localApp) { - return getDummyAppContext(null, "[ERROR] no app was returned by app.js for " + domaininfo.driname); - } + function getDomainInfo(apppath) { + var parts = apppath.split(/[#%]+/); + var hostname = parts.shift(); + var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, ''); + + return { + hostname: hostname + , pathname: pathname + , dirpathname: parts.join('#') + , dirname: apppath + , isRoot: apppath === hostname + }; } - if (!localApp.then) { - localApp = PromiseA.resolve(localApp); - } else { - return localApp.catch(function (e) { - return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname); + + function loadDomainMounts(domaininfo) { + var appContext; + + // should order and group by longest domain, then longest path + if (!domainMergeMap[domaininfo.hostname]) { + // create an connect / express app exclusive to this domain + // TODO express?? + domainMergeMap[domaininfo.hostname] = { + hostname: domaininfo.hostname + , apps: connect() + , mountsMap: {} + }; + domainMerged.push(domainMergeMap[domaininfo.hostname]); + } + + if (domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname]) { + return; + } + + domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] = function (req, res, next) { + if (appContext) { + appContext(req, res, next); + return; + } + + console.log('[log] LOADING "' + domaininfo.hostname + '/' + domaininfo.pathname + '"'); + getAppContext(domaininfo).then(function (localApp) { + // Note: pathname should NEVER have a leading '/' on its own + // we always add it explicitly + try { + domainMergeMap[domaininfo.hostname].apps.use('/' + domaininfo.pathname, localApp); + console.info('Loaded ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname); + appContext = localApp; + appContext(req, res, next); + } catch(e) { + console.error('[ERROR] ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname); + console.error(e); + res.end('{ "error": { "message": "[ERROR] could not load ' + + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname + + 'or default error app." } }'); + } + }); + }; + domainMergeMap[domaininfo.hostname].apps.use( + '/' + domaininfo.pathname + , domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] + ); + + return PromiseA.resolve(); + } + + function readNewVhosts() { + return fs.readdirSync(vhostsdir).filter(function (node) { + // not a hidden or private file + return '.' !== node[0] && '_' !== node[0]; + }).map(getDomainInfo).sort(function (a, b) { + var hlen = b.hostname.length - a.hostname.length; + var plen = b.pathname.length - a.pathname.length; + + // A directory could be named example.com, example.com# example.com## + // to indicate order of preference (for API addons, for example) + var dlen = b.dirname.length - a.dirname.length; + if (!hlen) { + if (!plen) { + return dlen; + } + return plen; + } + return plen; }); } - } catch(e) { - localApp = getDummyAppContext(e, "[ERROR] could not load app.js for " + domaininfo.dirname); - localApp = PromiseA.resolve(localApp); - return localApp; - } + function getDummyAppContext(err, msg) { + console.error('[ERROR] getDummyAppContext'); + console.error(err); + console.error(msg); + return function (req, res) { + res.end('{ "error": { "message": "' + msg + '" } }'); + } + } - return localApp; - } + function getAppContext(domaininfo) { + var localApp; - function loadDummyCerts() { - var certs = { - key: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.key.pem')) - , cert: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.crt.pem')) - , ca: fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) { - return /crt\.pem$/.test(node); - }).map(function (node) { - console.log('[log dummy ca]', node); - return fs.readFileSync(path.join(certsPath, 'ca', node)); - }) - }; - secureContexts.dummy = createSecureContext(dummyCerts); - dummyCerts = certs; - return certs - } + try { + // TODO live reload required modules + localApp = require(path.join(vhostsdir, domaininfo.dirname, 'app.js')); + if (localApp.create) { + // TODO read local config.yml and pass it in + // TODO pass in websocket + localApp = localApp.create(/*config*/); + if (!localApp) { + return getDummyAppContext(null, "[ERROR] no app was returned by app.js for " + domaininfo.driname); + } + } + if (!localApp.then) { + localApp = PromiseA.resolve(localApp); + } else { + return localApp.catch(function (e) { + return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname); + }); + } + } catch(e) { + localApp = getDummyAppContext(e, "[ERROR] could not load app.js for " + domaininfo.dirname); + localApp = PromiseA.resolve(localApp); + + return localApp; + } + + return localApp; + } + + function loadDomainVhosts() { + domainMerged.forEach(function (domainApp) { + console.log('[log] loaded mounts for domain ' + domainApp.hostname); + app.use(vhost(domainApp.hostname, domainApp.apps)); + app.use(vhost('www.' + domainApp.hostname, domainApp.apps)); + }); + } + + function hotloadApp(req, res, next) { + var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA); + var vhost = (req.headers.host || '').split(':')[0]; + + // the matching domain didn't catch it + if (domainMergeMap[vhost]) { + next(); + return; + } + + return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(function () { + // no matching domain was added + if (!domainMergeMap[vhost]) { + next(); + return; + } + + return forEachAsync(domainMergeMap[vhost].apps, function (fn) { + return new PromiseA(function (resolve) { + try { + fn(req, res, function (err) { + if (err) { + reject(err); + } + resolve(); + }); + } catch(e) { + reject(e); + } + }); + }).catch(function (e) { + next(e); + }); + }); + + /* + // TODO loop through mounts and see if any fit + domainMergeMap[vhost].mountsMap['/' + domaininfo.dirpathname] + if (!domainMergeMap[domaininfo.hostname]) { + // TODO reread directories + } + */ + } + + + // TODO pre-cache these once the server has started? + // return forEachAsync(rootDomains, loadCerts); + // TODO load these even more lazily + return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(function () { + app.use(hotloadApp); + resolve(app); + return; + }); + }); + }; function loadCerts(domainname) { // TODO make async @@ -219,7 +269,7 @@ module.exports.create = function (securePort, certsPath, vhostsdir) { } try { - secureContexts[domainname] = crypto.createCredentials(secOpts).context; + secureContexts[domainname] = createSecureContext(secOpts).context; } catch(err) { console.error("[ERROR] Certificates in '" + certsPath + "' could not be used:"); console.error(err); @@ -229,36 +279,9 @@ module.exports.create = function (securePort, certsPath, vhostsdir) { return secureContexts[domainname]; } - // TODO pre-cache these once the server has started? - // return forEachAsync(rootDomains, loadCerts); - // TODO load these even more lazily - return forEachAsync(readNewVhosts(), loadDomainMounts).then(loadDomainVhosts).then(runServer); - - function hotloadApp(req, res, next) { - var vhost = (req.headers.host || '').split(':')[0]; - - if (!domainMergeMap[vhost]) { - // TODO reread directories - } - - /* - // TODO loop through mounts and see if any fit - domainMergeMap[vhost].mountsMap['/' + domaininfo.dirpathname] - if (!domainMergeMap[domaininfo.hostname]) { - // TODO reread directories - } - */ - // TODO hot load all-the-things - next(); - } - - app.use(hotloadApp); - - function runServer() { - //provide a SNICallback when you create the options for the https server - - loadDummyCerts(); - secureOpts = { + function createSecureServer() { + var dummyCerts = loadDummyCerts(); + var secureOpts = { // fallback / default dummy certs key: dummyCerts.key , cert: dummyCerts.cert @@ -270,10 +293,14 @@ module.exports.create = function (securePort, certsPath, vhostsdir) { secureOpts.SNICallback = function (domainname, cb) { console.log('[log] SNI:', domainname); + if (!secureContexts.dummy) { + secureContexts.dummy = createSecureContext(dummyCerts); + } + var secureContext = secureContexts[domainname] || loadCerts(domainname) || secureContexts.dummy - || createSecureContext(dummyCerts) + //|| createSecureContext(dummyCerts) //|| createSecureContext(loadDummyCerts()) ; @@ -286,28 +313,47 @@ module.exports.create = function (securePort, certsPath, vhostsdir) { secureContext = createSecureContext(loadDummyCerts()); } - console.log('[log]', secureContext); - // workaround for v0.12 / v1.2 backwards compat bug if ('function' === typeof cb) { - console.log('using sni callback callback'); cb(null, secureContext); } else { - console.log('NOT using sni callback callback'); return secureContext; } }; } + addSniWorkaroundCallback(); - - secureServer = https.createServer(secureOpts); - secureServer.on('request', function (req, res) { - app(req, res); - }); - secureServer.listen(securePort, function () { - console.log("Listening on https://localhost:" + secureServer.address().port); - }); - - return PromiseA.resolve(); + return https.createServer(secureOpts); } + + function runServer() { + return new PromiseA(function (resolve) { + var secureServer = createSecureServer(); + var promiseApps; + + function loadPromise() { + if (!promiseApps) { + promiseApps = createPromiseApps(secureServer); + } + return promiseApps; + } + + secureServer.listen(securePort, function () { + resolve(secureServer); + console.log("Listening on https://localhost:" + secureServer.address().port, '\n'); + loadPromise(); + }); + + // Get up and listening as absolutely quickly as possible + secureServer.on('request', function (req, res) { + loadPromise().then(function (app) { + app(req, res); + }); + }); + + return secureServer; + }); + } + + return runServer(); } diff --git a/package.json b/package.json index 2b8face..b96f3aa 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "express": "^4.11.2", "express-session": "^1.10.3", "foreachasync": "^5.0.5", + "http-proxy": "^1.8.1", "human-readable-ids": "^1.0.1", "nat-pmp": "0.0.3", "node-acme": "0.0.1", diff --git a/walnut.js b/walnut.js index 37517a0..0ae897e 100644 --- a/walnut.js +++ b/walnut.js @@ -37,8 +37,6 @@ function phoneHome() { } ]; - // - /* // TODO return a middleware holepunch.run(require('./redirects.json').reduce(function (all, redirect) { if (!all[redirect.from.hostname]) { @@ -54,7 +52,8 @@ function phoneHome() { }, []), ports).catch(function () { console.error("Couldn't phone home. Oh well"); }); - //*/ } require('./lib/insecure-server').create(securePort, insecurePort, redirects); -require('./lib/vhost-sni-server.js').create(securePort, certsPath, vhostsdir).then(phoneHome); +require('./lib/vhost-sni-server.js').create(securePort, certsPath, vhostsdir) + //.then(phoneHome) + ;