From a8724cc5023a3058ba4d5b289ee5363e71b38f24 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Fri, 6 Nov 2015 11:05:32 +0000 Subject: [PATCH] Near-minimal app boot Aside from a few external process calls there are now zero external dependencies required as part of the node.js boot process. Yay! --- .gitignore | 1 + boot/README.md | 48 +++++++ lib/load-certs.js | 68 ++++++++++ lib/local-server.js | 45 ++++--- lib/master.js | 138 ++++++++++++++++++++ lib/sni-server.js | 140 +++++---------------- lib/spawn-caddy.js | 135 +++++++++++--------- lib/unlock-device.js | 3 - lib/worker.js | 119 ++++++++++++++++++ master.js | 290 ++++++++++++------------------------------- worker.js | 259 ++++++++++---------------------------- 11 files changed, 658 insertions(+), 588 deletions(-) create mode 100644 boot/README.md create mode 100644 lib/load-certs.js create mode 100644 lib/master.js create mode 100644 lib/worker.js diff --git a/.gitignore b/.gitignore index 9586ea8..5087394 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ redirects.json +Caddyfile sites-available sites-enabled dyndns-token.js diff --git a/boot/README.md b/boot/README.md new file mode 100644 index 0000000..0f90f20 --- /dev/null +++ b/boot/README.md @@ -0,0 +1,48 @@ +Small and Fast +============== + +We're targetting very tiny systems, so we have to +be really small and really fast. + +We want to get from 0 to a listening socket as quickly +as possible, so we have this little folder of boot +code that uses no external modules and as few internal +modules as reasonably possible. + +* fs.readFileSync is fast (< 1ms) +* v8's parser is pretty fast +* v8's fast compiler is slow +* v8's optimizer happens just-in-time + +Master +====== + +Master has a few jobs: + +* spin up the reverse proxy (caddy in this case) +* spin up the workers (as many as CPU cores) +* manage shared key/value store +* manage shared sqlite3 +* perform one-off processes once boot is complete + * SIGUSR1 (normally SIGHUP) to caddy + * watch and update ip address + * watch and update router unpn / pmp-nat + * watch and update Reverse VPN + +Worker +====== + +Workers are the ones that master spins up to do the hard +core stuff. They run the apis of the apps. + +Low Mem +======= + +We need to profile very low memory devices and see if +it is better to have just one process, or if master and +worker is still okay over time. + +The working suspision is that by occasionally starting +up a new worker and killing the old one when memory usage +starts to rise should fair pretty well and keeping +the system stable. diff --git a/lib/load-certs.js b/lib/load-certs.js new file mode 100644 index 0000000..5731340 --- /dev/null +++ b/lib/load-certs.js @@ -0,0 +1,68 @@ +'use strict'; + +function loadCerts(secureContexts, certPaths, domainname, prevdomainname) { + var PromiseA = require('bluebird'); + var fs = PromiseA.promisifyAll(require('fs')); + var path = require('path'); + + if (/(^|\.)proxyable\./.test(domainname)) { + // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com + // proxyable.myapp.mydomain.com => myapp.mydomain.com + // TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com + domainname = domainname.replace(/.*\.?proxyable\./, ''); + } + + if (secureContexts[domainname]) { + return PromiseA.resolve(secureContexts[domainname]); + } + + return PromiseA.some(certPaths.map(function (pathname) { + return PromiseA.all([ + fs.readFileAsync(path.join(pathname, domainname, 'privkey.pem'), 'ascii') + , fs.readFileAsync(path.join(pathname, domainname, 'fullchain.pem'), 'ascii') + ]); + }), 1).then(function (some) { + var one = some[0]; + secureContexts[domainname] = require('tls').createSecureContext({ + key: one[0] + , cert: one[1] + // https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ + // https://nodejs.org/api/tls.html + // removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU + , ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256' + , honorCipherOrder: true + }); + + // guard against race condition on Promise.some + if (prevdomainname && !secureContexts[prevdomainname]) { + // TODO XXX make sure that letsencrypt www. domains handle the bare domains also (and vice versa) + secureContexts[prevdomainname] = secureContexts[domainname]; + } + + return secureContexts[domainname]; + }, function (/*err*/) { + // AggregateError means both promises failed + // TODO check ENOENT + + // test "is this server <>?" + // try letsencrypt + // fail with www.example.com + if (/^www\./i.test(domainname)) { + return loadCerts(secureContexts, certPaths, domainname.replace(/^www\./i, ''), domainname); + } + + return (secureContexts['www.example.com'] || secureContexts['example.com']); + }).then(function (ctx) { + // TODO generate some self-signed certs? + if (!ctx) { + console.error("[loadCerts()] Could not load default HTTPS certificates!!!"); + return PromiseA.reject({ + message: "No default certificates for https" + , code: 'E_NO_DEFAULT_CERTS' + }); + } + + return ctx; + }); +} +module.exports.load = loadCerts; diff --git a/lib/local-server.js b/lib/local-server.js index 9c6fa7b..f0801e4 100644 --- a/lib/local-server.js +++ b/lib/local-server.js @@ -1,30 +1,43 @@ 'use strict'; -module.exports.create = function (port, promiseApp) { - var PromiseA = require('bluebird'); +// Note the odd use of callbacks (instead of promises) here +// It's to avoid loading bluebird yet (see sni-server.js for explanation) +module.exports.create = function (certPaths, port, serverCallback) { + function initServer(err, server) { + var app; + var promiseApp; - return new PromiseA(function (resolve, reject) { - var server = require('http').createServer(); + if (err) { + serverCallback(err); + return; + } - server.on('error', reject); - server.listen(port, 'localhost', function () { - console.log("Listening", server.address()); - resolve(server); + server.on('error', serverCallback); + server.listen(port, function () { + // is it even theoritically possible for + // a request to come in before this callback has fired? + // I'm assuming this event must fire before any request event + promiseApp = serverCallback(null, server); }); // Get up and listening as absolutely quickly as possible server.on('request', function (req, res) { - // TODO move to caddy parser? - if (/(^|\.)proxyable\./.test(req.headers.host)) { - // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com - // proxyable.myapp.mydomain.com => myapp.mydomain.com - // TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com - req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, ''); + // this is a hot piece of code, so we cache the result + if (app) { + app(req, res); + return; } - promiseApp().then(function (app) { + promiseApp.then(function (_app) { + app = _app; app(req, res); }); }); - }); + } + + if (certPaths) { + require('./sni-server').create(certPaths, port, initServer); + } else { + initServer(null, require('http').createServer()); + } }; diff --git a/lib/master.js b/lib/master.js new file mode 100644 index 0000000..9e7d1a7 --- /dev/null +++ b/lib/master.js @@ -0,0 +1,138 @@ +'use strict'; + +var cluster = require('cluster'); +var PromiseA = require('bluebird'); +var memstore; +// TODO +// var rootMasterKey; + +function updateIps() { + console.log('[UPDATE IP]'); + require('./ddns-updater').update().then(function (results) { + results.forEach(function (result) { + if (result.error) { + console.error(result); + } else { + console.log('[SUCCESS]', result.service.hostname); + } + }); + }).error(function (err) { + console.error('[UPDATE IP] ERROR'); + console.error(err); + }); +} + +function init(conf/*, state*/) { + if (!conf.ipcKey) { + conf.ipcKey = require('crypto').randomBytes(16).toString('base64'); + } + + var memstoreOpts = { + sock: conf.memstoreSock || '/tmp/memstore.sock' + + // If left 'null' or 'undefined' this defaults to a similar memstore + // with no special logic for 'cookie' or 'expires' + , store: cluster.isMaster && null //new require('express-session/session/memory')() + + // a good default to use for instances where you might want + // to cluster or to run standalone, but with the same API + , serve: cluster.isMaster + , connect: cluster.isWorker + //, standalone: (1 === numCores) // overrides serve and connect + // TODO implement + , key: conf.ipcKey + }; + try { + require('fs').unlinkSync(memstoreOpts.sock); + } catch(e) { + if ('ENOENT' !== e.code) { + console.error(e.stack); + console.error(JSON.stringify(e)); + } + // ignore + } + + var cstore = require('cluster-store'); + var memstorePromise = cstore.create(memstoreOpts).then(function (_memstore) { + memstore = _memstore; + }); + + // TODO check the IP every 5 minutes and update it every hour + setInterval(updateIps, 60 * 60 * 1000); + // we don't want this to load right away (extra procesing time) + setTimeout(updateIps, 1); + + return memstorePromise; +} + +function touch(conf, state) { + if (!state.initialize) { + state.initialize = init(conf, state); + } + + // TODO if no xyz worker, start on xyz worker (unlock, for example) + return state.initialize.then(function () { + // TODO conf.locked = true|false; + conf.initialized = true; + return conf; + }); + + /* + setInterval(function () { + console.log('SIGUSR1 to caddy'); + return caddy.update(caddyConf); + }, 10 * 60 * 1000); + */ +} + + //var config = require('./device.json'); + + // require('ssl-root-cas').inject(); + + /* + function phoneHome() { + var holepunch = require('./holepunch/beacon'); + var ports; + + ports = [ + { private: 65022 + , public: 65022 + , protocol: 'tcp' + , ttl: 0 + , test: { service: 'ssh' } + , testable: false + } + , { private: 650443 + , public: 650443 + , protocol: 'tcp' + , ttl: 0 + , test: { service: 'https' } + } + , { private: 65080 + , public: 65080 + , protocol: 'tcp' + , ttl: 0 + , test: { service: 'http' } + } + ]; + + // TODO return a middleware + holepunch.run(require('./redirects.json').reduce(function (all, redirect) { + if (!all[redirect.from.hostname]) { + all[redirect.from.hostname] = true; + all.push(redirect.from.hostname); + } + if (!all[redirect.to.hostname]) { + all[redirect.to.hostname] = true; + all.push(redirect.to.hostname); + } + + return all; + }, []), ports).catch(function () { + console.error("Couldn't phone home. Oh well"); + }); + } + */ + +module.exports.init = init; +module.exports.touch = touch; diff --git a/lib/sni-server.js b/lib/sni-server.js index d3fa09f..7a7d692 100644 --- a/lib/sni-server.js +++ b/lib/sni-server.js @@ -1,116 +1,44 @@ 'use strict'; -module.exports.create = function (certPaths, securePort, promiseApp) { - var https = require('https'); - // there are a few things that must exist on every core anyway +// Note the odd use of callbacks here. +// We're targetting low-power platforms and so we're trying to +// require everything as lazily as possible until our server +// is actually listening on the socket. Bluebird is heavy. +// Even the built-in modules can take dozens of milliseconds to require +module.exports.create = function (certPaths, serverCallback) { + // Recognize that this secureContexts cache is local to this CPU core var secureContexts = {}; - function loadCerts(domainname, prevdomainname) { - var PromiseA = require('bluebird'); - var fs = PromiseA.promisifyAll(require('fs')); - var path = require('path'); - - if (secureContexts[domainname]) { - return PromiseA.resolve(secureContexts[domainname]); - } - - return PromiseA.some(certPaths.map(function (pathname) { - return PromiseA.all([ - fs.readFileAsync(path.join(pathname, domainname, 'privkey.pem'), 'ascii') - , fs.readFileAsync(path.join(pathname, domainname, 'fullchain.pem'), 'ascii') - ]); - }), 1).then(function (some) { - var one = some[0]; - secureContexts[domainname] = require('tls').createSecureContext({ - key: one[0] - , cert: one[1] - // https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ - // https://nodejs.org/api/tls.html - // removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU - , ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256' - , honorCipherOrder: true - }); - - // guard against race condition on Promise.some - if (prevdomainname && !secureContexts[prevdomainname]) { - // TODO XXX make sure that letsencrypt www. domains handle the bare domains also (and vice versa) - secureContexts[prevdomainname] = secureContexts[domainname]; - } - - return secureContexts[domainname]; - }, function (/*err*/) { - // AggregateError means both promises failed - // TODO check ENOENT - - // test "is this server <>?" - // try letsencrypt - // fail with www.example.com - if (/^www\./i.test(domainname)) { - return loadCerts(domainname.replace(/^www\./i, ''), domainname); - } - - return (secureContexts['www.example.com'] || secureContexts['example.com']); - }).then(function (ctx) { - // TODO generate some self-signed certs? - if (!ctx) { - console.error("[loadCerts()] Could not load default HTTPS certificates!!!"); - return PromiseA.reject({ - message: "No default certificates for https" - , code: 'E_NO_DEFAULT_CERTS' - }); - } - - return ctx; - }); - } - function createSecureServer() { - return loadCerts('www.example.com').then(function (secureOpts) { + var domainname = 'www.example.com'; + var fs = require('fs'); + var secureOpts = { + // TODO create backup file just in case this one is ever corrupted + // NOTE synchronous is faster in this case of initialization + // NOTE certsPath[0] must be the default (LE) directory (another may be used for OV and EV certs) + key: fs.readFileSync(certPaths[0] + '/' + domainname + '/privkey.pem', 'ascii') + , cert: fs.readFileSync(certPaths[0] + '/' + domainname + '/fullchain.pem', 'ascii') + // https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ + // https://nodejs.org/api/tls.html + // removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU + , ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256' + , honorCipherOrder: true + }; - //SNICallback is passed the domain name, see NodeJS docs on TLS - secureOpts.SNICallback = function (domainname, cb) { - if (/(^|\.)proxyable\./.test(domainname)) { - // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com - // proxyable.myapp.mydomain.com => myapp.mydomain.com - // TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com - domainname = domainname.replace(/.*\.?proxyable\./, ''); - } + //SNICallback is passed the domain name, see NodeJS docs on TLS + secureOpts.SNICallback = function (domainname, cb) { + // NOTE: '*.proxyable.*' domains will be truncated + require('./load-certs').load(secureContexts, certPaths, domainname).then(function (context) { + cb(null, context); + }, function (err) { + console.error('[SNI Callback]'); + console.error(err.stack); + cb(err); + }); + }; - loadCerts(domainname).then(function (context) { - cb(null, context); - }, function (err) { - console.error('[SNI Callback]'); - console.error(err.stack); - cb(err); - }); - }; - - return https.createServer(secureOpts); - }); + serverCallback(null, require('https').createServer(secureOpts)); } - return createSecureServer().then(function (secureServer) { - var PromiseA = require('bluebird'); - - return new PromiseA(function (resolve, reject) { - secureServer.on('error', reject); - secureServer.listen(securePort, function () { - resolve(secureServer); - }); - - // Get up and listening as absolutely quickly as possible - secureServer.on('request', function (req, res) { - if (/(^|\.)proxyable\./.test(req.headers.host)) { - // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com - // proxyable.myapp.mydomain.com => myapp.mydomain.com - // TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com - req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, ''); - } - - promiseApp().then(function (app) { - app(req, res); - }); - }); - }); - }); + createSecureServer(); }; diff --git a/lib/spawn-caddy.js b/lib/spawn-caddy.js index a0d0d2a..b9cb96a 100644 --- a/lib/spawn-caddy.js +++ b/lib/spawn-caddy.js @@ -1,78 +1,89 @@ 'use strict'; -module.exports.create = function (/*config*/) { - var PromiseA = require('bluebird'); +function tplCaddyfile(conf) { + var contents = []; + + conf.domains.forEach(function (hostname) { + var content = ""; + + content+= "https://" + hostname + " {\n" + + " gzip\n" + + " tls " + + "/srv/walnut/certs/live/" + hostname + "/fullchain.pem " + + "/srv/walnut/certs/live/" + hostname + "/privkey.pem\n" + ; + + if (conf.locked) { + content += " root /srv/walnut/init.public/\n"; + } else { + content += " root /srv/walnut/sites-enabled/" + hostname + "/\n"; + } + + content += + " proxy /api http://localhost:" + conf.localPort.toString() + "\n" + // # TODO internal + + "}"; + + contents.push(content); + }); + + return contents.join('\n\n'); +} + +module.exports.tplCaddyfile = tplCaddyfile; +module.exports.create = function (config) { var spawn = require('child_process').spawn; - var path = require('path'); - var caddypath = '/usr/local/bin/caddy'; - var caddyfilepath = path.join(__dirname, '..', 'Caddyfile'); - var sitespath = path.join(__dirname, '..', 'sites-enabled'); + var caddypath = config.caddypath; + var caddyfilepath = config.caddyfilepath; + var sitespath = config.sitespath; var caddy; var fs = require('fs'); - // TODO this should be expanded to include proxies a la proxydyn - function writeCaddyfile(conf) { - return new PromiseA(function (resolve, reject) { - fs.readdir(sitespath, function (err, nodes) { - if (err) { - reject(err); + function writeCaddyfile(conf, cb) { + fs.readdir(sitespath, function (err, nodes) { + if (err) { + if (cb) { + cb(err); return; } + console.error('[writeCaddyFile] 0'); + console.error(err.stack); + throw err; + } - conf.domains = nodes.filter(function (node) { - return /\./.test(node) && !/(^\.)|([\/\:\\])/.test(node); - }); + conf.domains = nodes.filter(function (node) { + return /\./.test(node) && !/(^\.)|([\/\:\\])/.test(node); + }); - var contents = tplCaddyfile(conf); - fs.writeFile(caddyfilepath, contents, 'utf8', function (err) { - if (err) { - reject(err); + var contents = tplCaddyfile(conf); + fs.writeFile(caddyfilepath, contents, 'utf8', function (err) { + if (err) { + if (cb) { + cb(err); return; } + console.error('[writeCaddyFile] 1'); + console.error(err.stack); + throw err; + } - resolve(); - }); + if (cb) { cb(null); } }); }); } - function tplCaddyfile(conf) { - var contents = []; - - conf.domains.forEach(function (hostname) { - var content = ""; - - content+= "https://" + hostname + " {\n" - + " gzip\n" - + " tls " - + "/srv/walnut/certs/live/" + hostname + "/fullchain.pem " - + "/srv/walnut/certs/live/" + hostname + "/privkey.pem\n" - ; - - if (conf.locked) { - content += " root /srv/walnut/init.public/\n"; - } else { - content += " root /srv/walnut/sites-enabled/" + hostname + "/\n"; - } - - content += - " proxy /api http://localhost:" + conf.localPort.toString() + "\n" - // # TODO internal - + "}"; - - contents.push(content); - }); - - return contents.join('\n\n'); - } - - function spawnCaddy(conf) { + function spawnCaddy(conf, cb) { console.log('[CADDY] start'); - return writeCaddyfile(conf).then(function () { + writeCaddyfile(conf, function (err) { + if (err) { + console.error('[writeCaddyfile]'); + console.error(err.stack); + throw err; + } if (caddy) { caddy.kill('SIGUSR1'); - return; + return caddy; // TODO caddy.kill('SIGKILL'); if SIGTERM fails // https://github.com/mholt/caddy/issues/107 @@ -81,6 +92,13 @@ module.exports.create = function (/*config*/) { //caddy.kill('SIGTERM'); } + try { + require('child_process').execSync('killall caddy'); + } catch(e) { + // ignore + // Command failed: killall caddy + // caddy: no process found + } caddy = spawn(caddypath, ['-conf', caddyfilepath], { stdio: ['ignore', 'pipe', 'pipe'] }); caddy.stdout.on('data', function (str) { console.error('[Caddy]', str.toString('utf8')); @@ -100,7 +118,12 @@ module.exports.create = function (/*config*/) { }, 1 * 1000); }); - return caddy; + try { + if ('function' === typeof cb) { cb(null, caddy); } + } catch(e) { + console.error('ERROR: [spawn-caddy.js]'); + console.error(e.stack); + } }); } @@ -120,7 +143,7 @@ module.exports.create = function (/*config*/) { return { spawn: spawnCaddy , update: function (conf) { - return writeCaddyfile(conf).then(sighup); + return writeCaddyfile(conf, sighup); } , sighup: sighup }; diff --git a/lib/unlock-device.js b/lib/unlock-device.js index d676953..7e34cf9 100644 --- a/lib/unlock-device.js +++ b/lib/unlock-device.js @@ -15,7 +15,6 @@ module.exports.create = function () { //var rootMasterKey; app.use(function (req, res, next) { - console.log('yo yo yo soldya boy!', req.url); res.setHeader('Connection', 'close'); next(); }); @@ -51,7 +50,6 @@ module.exports.create = function () { }); app.use('/api', function (req, res) { - console.log('[d] /api'); res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.statusCode = 200; res.end(JSON.stringify({ @@ -66,7 +64,6 @@ module.exports.create = function () { // TODO break application cache? // TODO serve public sites? app.use('/', function (req, res, next) { - console.log('[pub] /'); if (!serveInitStatic) { serveStatic = require('serve-static'); serveInitStatic = serveStatic(path.join(__dirname, '..', 'init.public')); diff --git a/lib/worker.js b/lib/worker.js new file mode 100644 index 0000000..0f01d60 --- /dev/null +++ b/lib/worker.js @@ -0,0 +1,119 @@ +'use strict'; + +module.exports.create = function (webserver, info) { + var path = require('path'); + var vhostsdir = path.join(__dirname, 'vhosts'); + var app = require('express')(); + var apiHandler; + + /* + function unlockDevice(conf, state) { + return require('./lib/unlock-device').create().then(function (result) { + result.promise.then(function (_rootMasterKey) { + process.send({ + type: 'com.daplie.walnut.keys.root' + conf: { + rootMasterKey: _rootMasterkey + } + }); + conf.locked = false; + if (state.caddy) { + state.caddy.update(conf); + } + conf.rootMasterKey = _rootMasterKey; + }); + + return result.app; + }); + } + */ + + function scrubTheDubHelper(req, res/*, next*/) { + // hack for bricked app-cache + if (/\.appcache\b/.test(req.url)) { + res.setHeader('Content-Type', 'text/cache-manifest'); + res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*'); + return; + } + + // TODO port number for non-443 + var escapeHtml = require('escape-html'); + var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url; + var safeLocation = escapeHtml(newLocation); + + var metaRedirect = '' + + '\n' + + '\n' + + ' \n' + + ' \n' + + '\n' + + '\n' + + '

You requested an old resource. Please use this instead: \n' + + ' ' + safeLocation + '

\n' + + '\n' + + '\n' + ; + + // 301 redirects will not work for appcache + res.end(metaRedirect); + } + + function scrubTheDub(req, res, next) { + var host = req.hostname; + + if (!host || 'string' !== typeof host) { + next(); + return; + } + host = host.toLowerCase(); + + if (/^www\./.test(host)) { + scrubTheDubHelper(req, res, next); + return; + } + } + + function handleApi(req, res, next) { + if (!/^\/api/.test(req.url)) { + next(); + return; + } + + // TODO move to caddy parser? + if (/(^|\.)proxyable\./.test(req.hostname)) { + // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com + // proxyable.myapp.mydomain.com => myapp.mydomain.com + // TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com + req.hostname = req.hostname.replace(/.*\.?proxyable\./, ''); + } + + if (apiHandler) { + if (apiHandler.then) { + apiHandler.then(function (app) { + app(req, res, next); + }); + return; + } + + apiHandler(req, res, next); + return; + } + + apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app).then(function (app) { + // X-Forwarded-For + // X-Forwarded-Proto + console.log('api server', req.hostname, req.secure, req.ip); + apiHandler = app; + app(req, res, next); + }); + } + + if (info.trustProxy) { + app.set('trust proxy', ['loopback']); + //app.set('trust proxy', function (ip) { ... }); + } + app.use('/', scrubTheDub); + app.use('/', handleApi); + + return app; +}; diff --git a/master.js b/master.js index 664a41c..7bc7296 100644 --- a/master.js +++ b/master.js @@ -1,237 +1,105 @@ 'use strict'; +// TODO if RAM is very low we should not fork at all, +// but use a different process altogether + +console.log('pid:', process.pid); +console.log('title:', process.title); +console.log('arch:', process.arch); +console.log('platform:', process.platform); console.log('\n\n\n[MASTER] Welcome to WALNUT!'); -var PromiseA = require('bluebird'); -var fs = PromiseA.promisifyAll(require('fs')); var cluster = require('cluster'); -var numForks = 0; -var numCores = Math.min(2, require('os').cpus().length); -var securePort = process.argv[2] || 443; // 443 -var insecurePort = process.argv[3] || 80; // 80 -var localPort = securePort; -var caddy; -var masterServer; -var rootMasterKey; - -var redirects = require('./redirects.json'); var path = require('path'); +var minWorkers = 2; +var numCores = Math.max(minWorkers, require('os').cpus().length); +var workers = []; +var caddypath = '/usr/local/bin/caddy'; +var useCaddy = require('fs').existsSync(caddypath); +var conf = { + localPort: process.argv[2] || (useCaddy ? 4080 : 443) // system / local network +, insecurePort: process.argv[3] || (useCaddy ? 80 : 80) // meh +, externalPort: 443 // world accessible +// TODO externalInsecurePort? +, locked: false // TODO XXX +, ipcKey: null +, caddyfilepath: path.join(__dirname, 'Caddyfile') +, sitespath: path.join(__dirname, 'sites-enabled') +}; +var state = {}; +var caddy; - // force SSL upgrade server -var certPaths = [path.join(__dirname, 'certs', 'live')]; -var promiseServer; -var masterApp; -var caddyConf = { localPort: 4080, locked: true }; - -//console.log('\n.'); +if (useCaddy) { + conf.caddypath = caddypath; +} function fork() { - if (numForks < numCores) { - numForks += 1; - cluster.fork(); + if (workers.length < numCores) { + workers.push(cluster.fork()); } } -// Note that this function will be called async, after promiseServer is returned -// it seems like a circular dependency, but it isn't... not exactly anyway -function promiseApps() { - if (masterApp) { - return PromiseA.resolve(masterApp); - } - - masterApp = promiseServer.then(function (_masterServer) { - masterServer = _masterServer; - console.log("[MASTER] Listening on https://localhost:" + masterServer.address().port, '\n'); - - return require('./lib/unlock-device').create().then(function (result) { - result.promise.then(function (_rootMasterKey) { - var i; - caddyConf.locked = false; - if (caddy) { - caddy.update(caddyConf); - } - rootMasterKey = _rootMasterKey; - - if (numCores <= 2) { - // we're on one core, stagger the remaning - fork(); - return; - } - - for (i = 0; i < numCores; i += 1) { - fork(); - } - }); - - masterApp = result.app; - return result.app; - }); - }); - - return masterApp; -} - -// TODO have a fallback server than can download and apply an update? -require('./lib/insecure-server').create(securePort, insecurePort, redirects); -//console.log('\n.'); -promiseServer = fs.existsAsync('/usr/local/bin/caddy').then(function () { - console.log("Caddy is not present"); - // Caddy DOES NOT exist, use our node sni-server - return require('./lib/sni-server').create(certPaths, localPort, promiseApps); -}, function () { - console.log("Caddy is present (assumed running)"); - // Caddy DOES exist, use our http server without sni - localPort = caddyConf.localPort; - caddy = require('./lib/spawn-caddy').create(); - - return caddy.spawn(caddyConf).then(function () { - console.log("caddy has spawned"); - //return caddy.update(caddyConf).then(function () { - // console.log("caddy is updating"); - - setInterval(function () { - console.log('SIGUSR1 to caddy'); - return caddy.update(caddyConf); - }, 60 * 1000); - - return require('./lib/local-server').create(localPort, promiseApps); - //}); - }); -}); - -//console.log('\n.'); - cluster.on('online', function (worker) { + var path = require('path'); + // TODO XXX Should these be configurable? If so, where? + var certPaths = [path.join(__dirname, 'certs', 'live')]; + var info; + console.log('[MASTER] Worker ' + worker.process.pid + ' is online'); fork(); - if (masterServer) { - // NOTE: it's possible that this could survive idle for a while through keep-alive - // should default to connection: close - masterServer.close(); - masterServer = null; + info = { + type: 'com.daplie.walnut.init' + , conf: { + protocol: useCaddy ? 'http' : 'https' + , externalPort: conf.externalPort + , localPort: conf.localPort + , insecurePort: conf.insecurePort + , trustProxy: useCaddy ? true : false + , certPaths: useCaddy ? null : certPaths + , ipcKey: null + } + }; + worker.send(info); - setTimeout(function () { - // TODO use `id' to find user's uid / gid and set to file - // TODO set immediately? - if (!caddy) { - // TODO what about caddy - process.setgid(1000); - process.setuid(1000); - } - }, 1000); + function touchMaster(msg) { + if ('com.daplie.walnut.webserver.listening' !== msg.type) { + console.warn('[MASTER] received unexpected message from worker'); + console.warn(msg); + return; + } + + // calls init if init has not been called + state.caddy = caddy; + state.workers = workers; + require('./lib/master').touch(conf, state).then(function () { + info.type = 'com.daplie.walnut.webserver.onrequest'; + info.conf.ipcKey = conf.ipcKey; + worker.send(info); + }); } - - console.log("securePort", securePort); - worker.send({ - type: 'init' - , securePort: localPort - , certPaths: caddy ? null : certPaths - }); - - worker.on('message', function (msg) { - console.log('message from worker'); - console.log(msg); - }); + worker.on('message', touchMaster); }); cluster.on('exit', function (worker, code, signal) { - numForks -= 1; console.log('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal); + + workers = workers.map(function (w) { + if (worker !== w) { + return w; + } + return null; + }).filter(function (w) { + return w; + }); + fork(); }); -// TODO delegate to workers -function updateIps() { - console.log('[UPDATE IP]'); - require('./lib/ddns-updater').update().then(function (results) { - results.forEach(function (result) { - if (result.error) { - console.error(result); - } else { - console.log('[SUCCESS]', result.service.hostname); - } - }); - }).error(function (err) { - console.error('[UPDATE IP] ERROR'); - console.error(err); - }); +fork(); + +if (useCaddy) { + caddy = require('./lib/spawn-caddy').create(conf); + // relies on { localPort, locked } + caddy.spawn(conf); } -// TODO check the IP every 5 minutes and update it every hour -setInterval(updateIps, 60 * 60 * 1000); -// we don't want this to load right away (extra procesing time) -setTimeout(updateIps, 1); - -/* -worker.send({ - insecurePort: insecurePort -}); -*/ - - -/* -var fs = require('fs'); -var daplieReadFile = fs.readFileSync; -var time = 0; - -fs.readFileSync = function (filename) { - var now = Date.now(); - var data = daplieReadFile.apply(fs, arguments); - var t; - - t = (Date.now() - now); - time += t; - console.log('loaded "' + filename + '" in ' + t + 'ms (total ' + time + 'ms)'); - - return data; -}; -*/ - -//var config = require('./device.json'); - -// require('ssl-root-cas').inject(); - -/* -function phoneHome() { - var holepunch = require('./holepunch/beacon'); - var ports; - - ports = [ - { private: 65022 - , public: 65022 - , protocol: 'tcp' - , ttl: 0 - , test: { service: 'ssh' } - , testable: false - } - , { private: 650443 - , public: 650443 - , protocol: 'tcp' - , ttl: 0 - , test: { service: 'https' } - } - , { private: 65080 - , public: 65080 - , protocol: 'tcp' - , ttl: 0 - , test: { service: 'http' } - } - ]; - - // TODO return a middleware - holepunch.run(require('./redirects.json').reduce(function (all, redirect) { - if (!all[redirect.from.hostname]) { - all[redirect.from.hostname] = true; - all.push(redirect.from.hostname); - } - if (!all[redirect.to.hostname]) { - all[redirect.to.hostname] = true; - all.push(redirect.to.hostname); - } - - return all; - }, []), ports).catch(function () { - console.error("Couldn't phone home. Oh well"); - }); -} -*/ diff --git a/worker.js b/worker.js index 22c8ffc..5cf8355 100644 --- a/worker.js +++ b/worker.js @@ -2,205 +2,72 @@ var cluster = require('cluster'); var id = cluster.worker.id.toString(); -var path = require('path'); -var vhostsdir = path.join(__dirname, 'vhosts'); -console.log('[Worker #' + id + '] online!'); - -function init(info) { - var promiseServer; - var workerApp; - - function promiseApps() { - var PromiseA = require('bluebird'); - - if (workerApp) { - return PromiseA.resolve(workerApp); - } - - workerApp = promiseServer.then(function (secureServer) { - //secureServer = _secureServer; - console.log("#" + id + " Listening on https://localhost:" + secureServer.address().port, '\n'); - var app = require('express')(); - var apiHandler; - var staticHandlers = {}; - - app.use('/', function (req, res, next) { - if (!/^\/api/.test(req.url)) { - next(); - return; - } - - if (apiHandler) { - if (apiHandler.then) { - apiHandler.then(function (app) { - app(req, res, next); - }); - return; - } - - apiHandler(req, res, next); - return; - } - - apiHandler = require('./lib/vhost-server').create(info.securePort, vhostsdir).create(secureServer, app).then(function (app) { - apiHandler = app; - app(req, res, next); - }); - }); - - function scrubTheDub(req, res/*, next*/) { - // hack for bricked app-cache - if (/\.appcache\b/.test(req.url)) { - res.setHeader('Content-Type', 'text/cache-manifest'); - res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*'); - return; - } - - // TODO port number for non-443 - var escapeHtml = require('escape-html'); - var newLocation = 'https://' + req.headers.host.replace(/^www\./, '') + req.url; - var safeLocation = escapeHtml(newLocation); - - var metaRedirect = '' - + '\n' - + '\n' - + ' \n' - + ' \n' - + '\n' - + '\n' - + '

You requested an old resource. Please use this instead: \n' - + ' ' + safeLocation + '

\n' - + '\n' - + '\n' - ; - - // 301 redirects will not work for appcache - res.end(metaRedirect); - } - - app.use('/', function (req, res, next) { - if (/^\/api/.test(req.url)) { - next(); - return; - } - - // TODO block absolute urls for mounted apps? - // i.e. referer daplie.com/connect requests daplie.com/scripts/blah -> daplie.com/connect/scripts ? - var host = req.headers.host; - var invalidHost = /(\.\.)|[\\:\/\s\|>\*<]/; - - if (!host || 'string' !== typeof host) { - next(); - return; - } - host = host.toLowerCase(); - - if (/^www\./.test(host)) { - scrubTheDub(req, res, next); - return; - } - - function serveIt() { - // TODO redirect GET /favicon.ico to GET (req.headers.referer||'') + /favicon.ico - // TODO other common root things - robots.txt, app-icon, etc - staticHandlers[host].favicon(req, res, function (err) { - if (err) { - next(err); - return; - } - staticHandlers[host](req, res, next); - }); - } - - if (staticHandlers[host]) { - if (staticHandlers[host].then) { - staticHandlers[host].then(function () { - serveIt(); - }, function (err) { - res.send({ - error: { - message: err.message - , code: err.code - } - }); - }); - return; - } - - serveIt(); - return; - } - - staticHandlers[host] = PromiseA.resolve().then(function () { - var fs = PromiseA.promisifyAll(require('fs')); - - // host can be spoofed by the user, so lets be safe - // don't allow .. or / or whitespace - // RFC says domains must start with a-zA-Z0-9 and follow with normal characters - // HOWEVER, there are now Unicode character domains - // punycode? - // - if (invalidHost.test(host)) { - return PromiseA.reject({ - message: "invalid Host header" - , code: "E_INVALID_HOST" - }); - } - - return fs.readdirAsync(path.join(__dirname, 'sites-enabled')).then(function (nodes) { - nodes.forEach(function (node) { - if ('function' === typeof staticHandlers[host] && !staticHandlers[host].then) { - return; - } - - // ignore .gitkeep and folders without a . - if (0 === node.indexOf('.') || -1 === node.indexOf('.') || invalidHost.test(node)) { - return; - } - - console.log('vhost static'); - console.log(node); - staticHandlers[node] = require('serve-static')(path.join(__dirname, 'sites-enabled', node)); - try { - // TODO look for favicon - staticHandlers[node].favicon = require('serve-favicon')(path.join(__dirname, 'sites-enabled', node, 'favicon.ico')); - } catch(e) { - staticHandlers[node].favicon = function (req, res, next) { next(); }; - } - }); - - if (staticHandlers[host]) { - serveIt(); - } else { - next(); - } - - return staticHandlers[host]; - }); - }); - }); - - workerApp = app; - return app; - }); - - return workerApp; - } - - if (info.certPaths) { - promiseServer = require('./lib/sni-server').create(info.certPaths, info.securePort, promiseApps); - } else { - promiseServer = require('./lib/local-server').create(info.securePort, promiseApps); - } -} - -process.on('message', function (msg) { - if ('init' === msg.type) { - init(msg); +function waitForInit(message) { + if ('com.daplie.walnut.init' !== message.type) { + console.log('[Worker] 0 got unexpected message:'); + console.log(message); return; } - console.log('[Worker] got unexpected message:'); + var msg = message.conf; + process.removeListener('message', waitForInit); + + require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) { + if (err) { + console.log('[ERROR] worker.js'); + console.error(err.stack); + throw err; + } + + console.log("#" + id + " Listening on " + msg.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n'); + + var PromiseA = require('bluebird'); + return new PromiseA(function (resolve) { + function initWebServer(srvmsg) { + if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) { + console.log('[Worker] 1 got unexpected message:'); + console.log(srvmsg); + return; + } + + process.removeListener('message', initWebServer); + resolve(require('./lib/worker').create(webserver, srvmsg)); + } + process.send({ type: 'com.daplie.walnut.webserver.listening' }); + process.on('message', initWebServer); + }); + }); +} + +// We have to wait to get the configuration from the master process +// before we can start our webserver +console.log('[Worker #' + id + '] online!'); +process.on('message', waitForInit); + +// +// Debugging +// +process.on('exit', function (code) { + // only sync code can run here + console.log('uptime:', process.uptime()); + console.log(process.memoryUsage()); + console.log('[exit] process.exit() has been called (or master has killed us).'); + console.log(code); +}); +process.on('beforeExit', function (msg) { + // async can be scheduled here + console.log('[beforeExit] Event Loop is empty. Process will end.'); console.log(msg); }); +process.on('unhandledRejection', function (err) { + // this should always throw + // (it means somewhere we're not using bluebird by accident) + console.error('[unhandledRejection]'); + console.error(err.stack); + throw err; +}); +process.on('rejectionHandled', function (msg) { + console.error('[rejectionHandled]'); + console.error(msg); +});