coolaj86
/
walnut.js
1
1
Fork 1
Code Issues Pull Requests Releases Activity

Near-minimal app boot 

Aside from a few external process calls there are now
zero external dependencies required as part of the
node.js boot process. Yay!
This commit is contained in:
AJ ONeal 2015-11-06 11:05:32 +00:00
parent e14a6fd651
commit a8724cc502
11 changed files with 658 additions and 588 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
redirects.json
Caddyfile
sites-available
sites-enabled
dyndns-token.js

48
boot/README.md Normal file
View File

@ -0,0 +1,48 @@
Small and Fast
==============
We're targetting very tiny systems, so we have to
be really small and really fast.
We want to get from 0 to a listening socket as quickly
as possible, so we have this little folder of boot
code that uses no external modules and as few internal
modules as reasonably possible.
* fs.readFileSync is fast (< 1ms)
* v8's parser is pretty fast
* v8's fast compiler is slow
* v8's optimizer happens just-in-time
Master
======
Master has a few jobs:
* spin up the reverse proxy (caddy in this case)
* spin up the workers (as many as CPU cores)
* manage shared key/value store
* manage shared sqlite3
* perform one-off processes once boot is complete
* SIGUSR1 (normally SIGHUP) to caddy
* watch and update ip address
* watch and update router unpn / pmp-nat
* watch and update Reverse VPN
Worker
======
Workers are the ones that master spins up to do the hard
core stuff. They run the apis of the apps.
Low Mem
=======
We need to profile very low memory devices and see if
it is better to have just one process, or if master and
worker is still okay over time.
The working suspision is that by occasionally starting
up a new worker and killing the old one when memory usage
starts to rise should fair pretty well and keeping
the system stable.

68
lib/load-certs.js Normal file
View File

@ -0,0 +1,68 @@
'use strict';
function loadCerts(secureContexts, certPaths, domainname, prevdomainname) {
var PromiseA = require('bluebird');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
if (/(^|\.)proxyable\./.test(domainname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
domainname = domainname.replace(/.*\.?proxyable\./, '');
}
if (secureContexts[domainname]) {
return PromiseA.resolve(secureContexts[domainname]);
}
return PromiseA.some(certPaths.map(function (pathname) {
return PromiseA.all([
fs.readFileAsync(path.join(pathname, domainname, 'privkey.pem'), 'ascii')
, fs.readFileAsync(path.join(pathname, domainname, 'fullchain.pem'), 'ascii')
]);
}), 1).then(function (some) {
var one = some[0];
secureContexts[domainname] = require('tls').createSecureContext({
key: one[0]
, cert: one[1]
// https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
// https://nodejs.org/api/tls.html
// removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU
, ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256'
, honorCipherOrder: true
});
// guard against race condition on Promise.some
if (prevdomainname && !secureContexts[prevdomainname]) {
// TODO XXX make sure that letsencrypt www. domains handle the bare domains also (and vice versa)
secureContexts[prevdomainname] = secureContexts[domainname];
}
return secureContexts[domainname];
}, function (/*err*/) {
// AggregateError means both promises failed
// TODO check ENOENT
// test "is this server <<domainname>>?"
// try letsencrypt
// fail with www.example.com
if (/^www\./i.test(domainname)) {
return loadCerts(secureContexts, certPaths, domainname.replace(/^www\./i, ''), domainname);
}
return (secureContexts['www.example.com'] || secureContexts['example.com']);
}).then(function (ctx) {
// TODO generate some self-signed certs?
if (!ctx) {
console.error("[loadCerts()] Could not load default HTTPS certificates!!!");
return PromiseA.reject({
message: "No default certificates for https"
, code: 'E_NO_DEFAULT_CERTS'
});
}
return ctx;
});
}
module.exports.load = loadCerts;

45
lib/local-server.js
View File

@ -1,30 +1,43 @@
'use strict';
module.exports.create = function (port, promiseApp) {
var PromiseA = require('bluebird');
// Note the odd use of callbacks (instead of promises) here
// It's to avoid loading bluebird yet (see sni-server.js for explanation)
module.exports.create = function (certPaths, port, serverCallback) {
function initServer(err, server) {
var app;
var promiseApp;
return new PromiseA(function (resolve, reject) {
var server = require('http').createServer();
if (err) {
serverCallback(err);
return;
}
server.on('error', reject);
server.listen(port, 'localhost', function () {
console.log("Listening", server.address());
resolve(server);
server.on('error', serverCallback);
server.listen(port, function () {
// is it even theoritically possible for
// a request to come in before this callback has fired?
// I'm assuming this event must fire before any request event
promiseApp = serverCallback(null, server);
});
// Get up and listening as absolutely quickly as possible
server.on('request', function (req, res) {
// TODO move to caddy parser?
if (/(^|\.)proxyable\./.test(req.headers.host)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, '');
// this is a hot piece of code, so we cache the result
if (app) {
app(req, res);
return;
}
promiseApp().then(function (app) {
promiseApp.then(function (_app) {
app = _app;
app(req, res);
});
});
});
}
if (certPaths) {
require('./sni-server').create(certPaths, port, initServer);
} else {
initServer(null, require('http').createServer());
}
};

138
lib/master.js Normal file
View File

@ -0,0 +1,138 @@
'use strict';
var cluster = require('cluster');
var PromiseA = require('bluebird');
var memstore;
// TODO
// var rootMasterKey;
function updateIps() {
console.log('[UPDATE IP]');
require('./ddns-updater').update().then(function (results) {
results.forEach(function (result) {
if (result.error) {
console.error(result);
} else {
console.log('[SUCCESS]', result.service.hostname);
}
});
}).error(function (err) {
console.error('[UPDATE IP] ERROR');
console.error(err);
});
}
function init(conf/*, state*/) {
if (!conf.ipcKey) {
conf.ipcKey = require('crypto').randomBytes(16).toString('base64');
}
var memstoreOpts = {
sock: conf.memstoreSock || '/tmp/memstore.sock'
// If left 'null' or 'undefined' this defaults to a similar memstore
// with no special logic for 'cookie' or 'expires'
, store: cluster.isMaster && null //new require('express-session/session/memory')()
// a good default to use for instances where you might want
// to cluster or to run standalone, but with the same API
, serve: cluster.isMaster
, connect: cluster.isWorker
//, standalone: (1 === numCores) // overrides serve and connect
// TODO implement
, key: conf.ipcKey
};
try {
require('fs').unlinkSync(memstoreOpts.sock);
} catch(e) {
if ('ENOENT' !== e.code) {
console.error(e.stack);
console.error(JSON.stringify(e));
}
// ignore
}
var cstore = require('cluster-store');
var memstorePromise = cstore.create(memstoreOpts).then(function (_memstore) {
memstore = _memstore;
});
// TODO check the IP every 5 minutes and update it every hour
setInterval(updateIps, 60 * 60 * 1000);
// we don't want this to load right away (extra procesing time)
setTimeout(updateIps, 1);
return memstorePromise;
}
function touch(conf, state) {
if (!state.initialize) {
state.initialize = init(conf, state);
}
// TODO if no xyz worker, start on xyz worker (unlock, for example)
return state.initialize.then(function () {
// TODO conf.locked = true|false;
conf.initialized = true;
return conf;
});
/*
setInterval(function () {
console.log('SIGUSR1 to caddy');
return caddy.update(caddyConf);
}, 10 * 60 * 1000);
*/
}
//var config = require('./device.json');
// require('ssl-root-cas').inject();
/*
function phoneHome() {
var holepunch = require('./holepunch/beacon');
var ports;
ports = [
{ private: 65022
, public: 65022
, protocol: 'tcp'
, ttl: 0
, test: { service: 'ssh' }
, testable: false
}
, { private: 650443
, public: 650443
, protocol: 'tcp'
, ttl: 0
, test: { service: 'https' }
}
, { private: 65080
, public: 65080
, protocol: 'tcp'
, ttl: 0
, test: { service: 'http' }
}
];
// TODO return a middleware
holepunch.run(require('./redirects.json').reduce(function (all, redirect) {
if (!all[redirect.from.hostname]) {
all[redirect.from.hostname] = true;
all.push(redirect.from.hostname);
}
if (!all[redirect.to.hostname]) {
all[redirect.to.hostname] = true;
all.push(redirect.to.hostname);
}
return all;
}, []), ports).catch(function () {
console.error("Couldn't phone home. Oh well");
});
}
*/
module.exports.init = init;
module.exports.touch = touch;

140
lib/sni-server.js
View File

@ -1,116 +1,44 @@
'use strict';
module.exports.create = function (certPaths, securePort, promiseApp) {
var https = require('https');
// there are a few things that must exist on every core anyway
// Note the odd use of callbacks here.
// We're targetting low-power platforms and so we're trying to
// require everything as lazily as possible until our server
// is actually listening on the socket. Bluebird is heavy.
// Even the built-in modules can take dozens of milliseconds to require
module.exports.create = function (certPaths, serverCallback) {
// Recognize that this secureContexts cache is local to this CPU core
var secureContexts = {};
function loadCerts(domainname, prevdomainname) {
var PromiseA = require('bluebird');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
if (secureContexts[domainname]) {
return PromiseA.resolve(secureContexts[domainname]);
}
return PromiseA.some(certPaths.map(function (pathname) {
return PromiseA.all([
fs.readFileAsync(path.join(pathname, domainname, 'privkey.pem'), 'ascii')
, fs.readFileAsync(path.join(pathname, domainname, 'fullchain.pem'), 'ascii')
]);
}), 1).then(function (some) {
var one = some[0];
secureContexts[domainname] = require('tls').createSecureContext({
key: one[0]
, cert: one[1]
// https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
// https://nodejs.org/api/tls.html
// removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU
, ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256'
, honorCipherOrder: true
});
// guard against race condition on Promise.some
if (prevdomainname && !secureContexts[prevdomainname]) {
// TODO XXX make sure that letsencrypt www. domains handle the bare domains also (and vice versa)
secureContexts[prevdomainname] = secureContexts[domainname];
}
return secureContexts[domainname];
}, function (/*err*/) {
// AggregateError means both promises failed
// TODO check ENOENT
// test "is this server <<domainname>>?"
// try letsencrypt
// fail with www.example.com
if (/^www\./i.test(domainname)) {
return loadCerts(domainname.replace(/^www\./i, ''), domainname);
}
return (secureContexts['www.example.com'] || secureContexts['example.com']);
}).then(function (ctx) {
// TODO generate some self-signed certs?
if (!ctx) {
console.error("[loadCerts()] Could not load default HTTPS certificates!!!");
return PromiseA.reject({
message: "No default certificates for https"
, code: 'E_NO_DEFAULT_CERTS'
});
}
return ctx;
});
}
function createSecureServer() {
return loadCerts('www.example.com').then(function (secureOpts) {
var domainname = 'www.example.com';
var fs = require('fs');
var secureOpts = {
// TODO create backup file just in case this one is ever corrupted
// NOTE synchronous is faster in this case of initialization
// NOTE certsPath[0] must be the default (LE) directory (another may be used for OV and EV certs)
key: fs.readFileSync(certPaths[0] + '/' + domainname + '/privkey.pem', 'ascii')
, cert: fs.readFileSync(certPaths[0] + '/' + domainname + '/fullchain.pem', 'ascii')
// https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
// https://nodejs.org/api/tls.html
// removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU
, ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256'
, honorCipherOrder: true
};
//SNICallback is passed the domain name, see NodeJS docs on TLS
secureOpts.SNICallback = function (domainname, cb) {
if (/(^|\.)proxyable\./.test(domainname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
domainname = domainname.replace(/.*\.?proxyable\./, '');
}
//SNICallback is passed the domain name, see NodeJS docs on TLS
secureOpts.SNICallback = function (domainname, cb) {
// NOTE: '*.proxyable.*' domains will be truncated
require('./load-certs').load(secureContexts, certPaths, domainname).then(function (context) {
cb(null, context);
}, function (err) {
console.error('[SNI Callback]');
console.error(err.stack);
cb(err);
});
};
loadCerts(domainname).then(function (context) {
cb(null, context);
}, function (err) {
console.error('[SNI Callback]');
console.error(err.stack);
cb(err);
});
};
return https.createServer(secureOpts);
});
serverCallback(null, require('https').createServer(secureOpts));
}
return createSecureServer().then(function (secureServer) {
var PromiseA = require('bluebird');
return new PromiseA(function (resolve, reject) {
secureServer.on('error', reject);
secureServer.listen(securePort, function () {
resolve(secureServer);
});
// Get up and listening as absolutely quickly as possible
secureServer.on('request', function (req, res) {
if (/(^|\.)proxyable\./.test(req.headers.host)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, '');
}
promiseApp().then(function (app) {
app(req, res);
});
});
});
});
createSecureServer();
};

135
lib/spawn-caddy.js
View File

@ -1,78 +1,89 @@
'use strict';
module.exports.create = function (/*config*/) {
var PromiseA = require('bluebird');
function tplCaddyfile(conf) {
var contents = [];
conf.domains.forEach(function (hostname) {
var content = "";
content+= "https://" + hostname + " {\n"
+ " gzip\n"
+ " tls "
+ "/srv/walnut/certs/live/" + hostname + "/fullchain.pem "
+ "/srv/walnut/certs/live/" + hostname + "/privkey.pem\n"
;
if (conf.locked) {
content += " root /srv/walnut/init.public/\n";
} else {
content += " root /srv/walnut/sites-enabled/" + hostname + "/\n";
}
content +=
" proxy /api http://localhost:" + conf.localPort.toString() + "\n"
// # TODO internal
+ "}";
contents.push(content);
});
return contents.join('\n\n');
}
module.exports.tplCaddyfile = tplCaddyfile;
module.exports.create = function (config) {
var spawn = require('child_process').spawn;
var path = require('path');
var caddypath = '/usr/local/bin/caddy';
var caddyfilepath = path.join(__dirname, '..', 'Caddyfile');
var sitespath = path.join(__dirname, '..', 'sites-enabled');
var caddypath = config.caddypath;
var caddyfilepath = config.caddyfilepath;
var sitespath = config.sitespath;
var caddy;
var fs = require('fs');
// TODO this should be expanded to include proxies a la proxydyn
function writeCaddyfile(conf) {
return new PromiseA(function (resolve, reject) {
fs.readdir(sitespath, function (err, nodes) {
if (err) {
reject(err);
function writeCaddyfile(conf, cb) {
fs.readdir(sitespath, function (err, nodes) {
if (err) {
if (cb) {
cb(err);
return;
}
console.error('[writeCaddyFile] 0');
console.error(err.stack);
throw err;
}
conf.domains = nodes.filter(function (node) {
return /\./.test(node) && !/(^\.)|([\/\:\\])/.test(node);
});
conf.domains = nodes.filter(function (node) {
return /\./.test(node) && !/(^\.)|([\/\:\\])/.test(node);
});
var contents = tplCaddyfile(conf);
fs.writeFile(caddyfilepath, contents, 'utf8', function (err) {
if (err) {
reject(err);
var contents = tplCaddyfile(conf);
fs.writeFile(caddyfilepath, contents, 'utf8', function (err) {
if (err) {
if (cb) {
cb(err);
return;
}
console.error('[writeCaddyFile] 1');
console.error(err.stack);
throw err;
}
resolve();
});
if (cb) { cb(null); }
});
});
}
function tplCaddyfile(conf) {
var contents = [];
conf.domains.forEach(function (hostname) {
var content = "";
content+= "https://" + hostname + " {\n"
+ " gzip\n"
+ " tls "
+ "/srv/walnut/certs/live/" + hostname + "/fullchain.pem "
+ "/srv/walnut/certs/live/" + hostname + "/privkey.pem\n"
;
if (conf.locked) {
content += " root /srv/walnut/init.public/\n";
} else {
content += " root /srv/walnut/sites-enabled/" + hostname + "/\n";
}
content +=
" proxy /api http://localhost:" + conf.localPort.toString() + "\n"
// # TODO internal
+ "}";
contents.push(content);
});
return contents.join('\n\n');
}
function spawnCaddy(conf) {
function spawnCaddy(conf, cb) {
console.log('[CADDY] start');
return writeCaddyfile(conf).then(function () {
writeCaddyfile(conf, function (err) {
if (err) {
console.error('[writeCaddyfile]');
console.error(err.stack);
throw err;
}
if (caddy) {
caddy.kill('SIGUSR1');
return;
return caddy;
// TODO caddy.kill('SIGKILL'); if SIGTERM fails
// https://github.com/mholt/caddy/issues/107
@ -81,6 +92,13 @@ module.exports.create = function (/*config*/) {
//caddy.kill('SIGTERM');
}
try {
require('child_process').execSync('killall caddy');
} catch(e) {
// ignore
// Command failed: killall caddy
// caddy: no process found
}
caddy = spawn(caddypath, ['-conf', caddyfilepath], { stdio: ['ignore', 'pipe', 'pipe'] });
caddy.stdout.on('data', function (str) {
console.error('[Caddy]', str.toString('utf8'));
@ -100,7 +118,12 @@ module.exports.create = function (/*config*/) {
}, 1 * 1000);
});
return caddy;
try {
if ('function' === typeof cb) { cb(null, caddy); }
} catch(e) {
console.error('ERROR: [spawn-caddy.js]');
console.error(e.stack);
}
});
}
@ -120,7 +143,7 @@ module.exports.create = function (/*config*/) {
return {
spawn: spawnCaddy
, update: function (conf) {
return writeCaddyfile(conf).then(sighup);
return writeCaddyfile(conf, sighup);
}
, sighup: sighup
};

3
lib/unlock-device.js
View File

@ -15,7 +15,6 @@ module.exports.create = function () {
//var rootMasterKey;
app.use(function (req, res, next) {
console.log('yo yo yo soldya boy!', req.url);
res.setHeader('Connection', 'close');
next();
});
@ -51,7 +50,6 @@ module.exports.create = function () {
});
app.use('/api', function (req, res) {
console.log('[d] /api');
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.statusCode = 200;
res.end(JSON.stringify({
@ -66,7 +64,6 @@ module.exports.create = function () {
// TODO break application cache?
// TODO serve public sites?
app.use('/', function (req, res, next) {
console.log('[pub] /');
if (!serveInitStatic) {
serveStatic = require('serve-static');
serveInitStatic = serveStatic(path.join(__dirname, '..', 'init.public'));

119
lib/worker.js Normal file
View File

@ -0,0 +1,119 @@
'use strict';
module.exports.create = function (webserver, info) {
var path = require('path');
var vhostsdir = path.join(__dirname, 'vhosts');
var app = require('express')();
var apiHandler;
/*
function unlockDevice(conf, state) {
return require('./lib/unlock-device').create().then(function (result) {
result.promise.then(function (_rootMasterKey) {
process.send({
type: 'com.daplie.walnut.keys.root'
conf: {
rootMasterKey: _rootMasterkey
}
});
conf.locked = false;
if (state.caddy) {
state.caddy.update(conf);
}
conf.rootMasterKey = _rootMasterKey;
});
return result.app;
});
}
*/
function scrubTheDubHelper(req, res/*, next*/) {
// hack for bricked app-cache
if (/\.appcache\b/.test(req.url)) {
res.setHeader('Content-Type', 'text/cache-manifest');
res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*');
return;
}
// TODO port number for non-443
var escapeHtml = require('escape-html');
var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an old resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
// 301 redirects will not work for appcache
res.end(metaRedirect);
}
function scrubTheDub(req, res, next) {
var host = req.hostname;
if (!host || 'string' !== typeof host) {
next();
return;
}
host = host.toLowerCase();
if (/^www\./.test(host)) {
scrubTheDubHelper(req, res, next);
return;
}
}
function handleApi(req, res, next) {
if (!/^\/api/.test(req.url)) {
next();
return;
}
// TODO move to caddy parser?
if (/(^|\.)proxyable\./.test(req.hostname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com
req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
}
if (apiHandler) {
if (apiHandler.then) {
apiHandler.then(function (app) {
app(req, res, next);
});
return;
}
apiHandler(req, res, next);
return;
}
apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app).then(function (app) {
// X-Forwarded-For
// X-Forwarded-Proto
console.log('api server', req.hostname, req.secure, req.ip);
apiHandler = app;
app(req, res, next);
});
}
if (info.trustProxy) {
app.set('trust proxy', ['loopback']);
//app.set('trust proxy', function (ip) { ... });
}
app.use('/', scrubTheDub);
app.use('/', handleApi);
return app;
};

290
master.js
View File

@ -1,237 +1,105 @@
'use strict';
// TODO if RAM is very low we should not fork at all,
// but use a different process altogether
console.log('pid:', process.pid);
console.log('title:', process.title);
console.log('arch:', process.arch);
console.log('platform:', process.platform);
console.log('\n\n\n[MASTER] Welcome to WALNUT!');
var PromiseA = require('bluebird');
var fs = PromiseA.promisifyAll(require('fs'));
var cluster = require('cluster');
var numForks = 0;
var numCores = Math.min(2, require('os').cpus().length);
var securePort = process.argv[2] || 443; // 443
var insecurePort = process.argv[3] || 80; // 80
var localPort = securePort;
var caddy;
var masterServer;
var rootMasterKey;
var redirects = require('./redirects.json');
var path = require('path');
var minWorkers = 2;
var numCores = Math.max(minWorkers, require('os').cpus().length);
var workers = [];
var caddypath = '/usr/local/bin/caddy';
var useCaddy = require('fs').existsSync(caddypath);
var conf = {
localPort: process.argv[2] || (useCaddy ? 4080 : 443) // system / local network
, insecurePort: process.argv[3] || (useCaddy ? 80 : 80) // meh
, externalPort: 443 // world accessible
// TODO externalInsecurePort?
, locked: false // TODO XXX
, ipcKey: null
, caddyfilepath: path.join(__dirname, 'Caddyfile')
, sitespath: path.join(__dirname, 'sites-enabled')
};
var state = {};
var caddy;
// force SSL upgrade server
var certPaths = [path.join(__dirname, 'certs', 'live')];
var promiseServer;
var masterApp;
var caddyConf = { localPort: 4080, locked: true };
//console.log('\n.');
if (useCaddy) {
conf.caddypath = caddypath;
}
function fork() {
if (numForks < numCores) {
numForks += 1;
cluster.fork();
if (workers.length < numCores) {
workers.push(cluster.fork());
}
}
// Note that this function will be called async, after promiseServer is returned
// it seems like a circular dependency, but it isn't... not exactly anyway
function promiseApps() {
if (masterApp) {
return PromiseA.resolve(masterApp);
}
masterApp = promiseServer.then(function (_masterServer) {
masterServer = _masterServer;
console.log("[MASTER] Listening on https://localhost:" + masterServer.address().port, '\n');
return require('./lib/unlock-device').create().then(function (result) {
result.promise.then(function (_rootMasterKey) {
var i;
caddyConf.locked = false;
if (caddy) {
caddy.update(caddyConf);
}
rootMasterKey = _rootMasterKey;
if (numCores <= 2) {
// we're on one core, stagger the remaning
fork();
return;
}
for (i = 0; i < numCores; i += 1) {
fork();
}
});
masterApp = result.app;
return result.app;
});
});
return masterApp;
}
// TODO have a fallback server than can download and apply an update?
require('./lib/insecure-server').create(securePort, insecurePort, redirects);
//console.log('\n.');
promiseServer = fs.existsAsync('/usr/local/bin/caddy').then(function () {
console.log("Caddy is not present");
// Caddy DOES NOT exist, use our node sni-server
return require('./lib/sni-server').create(certPaths, localPort, promiseApps);
}, function () {
console.log("Caddy is present (assumed running)");
// Caddy DOES exist, use our http server without sni
localPort = caddyConf.localPort;
caddy = require('./lib/spawn-caddy').create();
return caddy.spawn(caddyConf).then(function () {
console.log("caddy has spawned");
//return caddy.update(caddyConf).then(function () {
// console.log("caddy is updating");
setInterval(function () {
console.log('SIGUSR1 to caddy');
return caddy.update(caddyConf);
}, 60 * 1000);
return require('./lib/local-server').create(localPort, promiseApps);
//});
});
});
//console.log('\n.');
cluster.on('online', function (worker) {
var path = require('path');
// TODO XXX Should these be configurable? If so, where?
var certPaths = [path.join(__dirname, 'certs', 'live')];
var info;
console.log('[MASTER] Worker ' + worker.process.pid + ' is online');
fork();
if (masterServer) {
// NOTE: it's possible that this could survive idle for a while through keep-alive
// should default to connection: close
masterServer.close();
masterServer = null;
info = {
type: 'com.daplie.walnut.init'
, conf: {
protocol: useCaddy ? 'http' : 'https'
, externalPort: conf.externalPort
, localPort: conf.localPort
, insecurePort: conf.insecurePort
, trustProxy: useCaddy ? true : false
, certPaths: useCaddy ? null : certPaths
, ipcKey: null
}
};
worker.send(info);
setTimeout(function () {
// TODO use `id' to find user's uid / gid and set to file
// TODO set immediately?
if (!caddy) {
// TODO what about caddy
process.setgid(1000);
process.setuid(1000);
}
}, 1000);
function touchMaster(msg) {
if ('com.daplie.walnut.webserver.listening' !== msg.type) {
console.warn('[MASTER] received unexpected message from worker');
console.warn(msg);
return;
}
// calls init if init has not been called
state.caddy = caddy;
state.workers = workers;
require('./lib/master').touch(conf, state).then(function () {
info.type = 'com.daplie.walnut.webserver.onrequest';
info.conf.ipcKey = conf.ipcKey;
worker.send(info);
});
}
console.log("securePort", securePort);
worker.send({
type: 'init'
, securePort: localPort
, certPaths: caddy ? null : certPaths
});
worker.on('message', function (msg) {
console.log('message from worker');
console.log(msg);
});
worker.on('message', touchMaster);
});
cluster.on('exit', function (worker, code, signal) {
numForks -= 1;
console.log('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
workers = workers.map(function (w) {
if (worker !== w) {
return w;
}
return null;
}).filter(function (w) {
return w;
});
fork();
});
// TODO delegate to workers
function updateIps() {
console.log('[UPDATE IP]');
require('./lib/ddns-updater').update().then(function (results) {
results.forEach(function (result) {
if (result.error) {
console.error(result);
} else {
console.log('[SUCCESS]', result.service.hostname);
}
});
}).error(function (err) {
console.error('[UPDATE IP] ERROR');
console.error(err);
});
fork();
if (useCaddy) {
caddy = require('./lib/spawn-caddy').create(conf);
// relies on { localPort, locked }
caddy.spawn(conf);
}
// TODO check the IP every 5 minutes and update it every hour
setInterval(updateIps, 60 * 60 * 1000);
// we don't want this to load right away (extra procesing time)
setTimeout(updateIps, 1);
/*
worker.send({
insecurePort: insecurePort
});
*/
/*
var fs = require('fs');
var daplieReadFile = fs.readFileSync;
var time = 0;
fs.readFileSync = function (filename) {
var now = Date.now();
var data = daplieReadFile.apply(fs, arguments);
var t;
t = (Date.now() - now);
time += t;
console.log('loaded "' + filename + '" in ' + t + 'ms (total ' + time + 'ms)');
return data;
};
*/
//var config = require('./device.json');
// require('ssl-root-cas').inject();
/*
function phoneHome() {
var holepunch = require('./holepunch/beacon');
var ports;
ports = [
{ private: 65022
, public: 65022
, protocol: 'tcp'
, ttl: 0
, test: { service: 'ssh' }
, testable: false
}
, { private: 650443
, public: 650443
, protocol: 'tcp'
, ttl: 0
, test: { service: 'https' }
}
, { private: 65080
, public: 65080
, protocol: 'tcp'
, ttl: 0
, test: { service: 'http' }
}
];
// TODO return a middleware
holepunch.run(require('./redirects.json').reduce(function (all, redirect) {
if (!all[redirect.from.hostname]) {
all[redirect.from.hostname] = true;
all.push(redirect.from.hostname);
}
if (!all[redirect.to.hostname]) {
all[redirect.to.hostname] = true;
all.push(redirect.to.hostname);
}
return all;
}, []), ports).catch(function () {
console.error("Couldn't phone home. Oh well");
});
}
*/

259
worker.js
View File

@ -2,205 +2,72 @@
var cluster = require('cluster');
var id = cluster.worker.id.toString();
var path = require('path');
var vhostsdir = path.join(__dirname, 'vhosts');
console.log('[Worker #' + id + '] online!');
function init(info) {
var promiseServer;
var workerApp;
function promiseApps() {
var PromiseA = require('bluebird');
if (workerApp) {
return PromiseA.resolve(workerApp);
}
workerApp = promiseServer.then(function (secureServer) {
//secureServer = _secureServer;
console.log("#" + id + " Listening on https://localhost:" + secureServer.address().port, '\n');
var app = require('express')();
var apiHandler;
var staticHandlers = {};
app.use('/', function (req, res, next) {
if (!/^\/api/.test(req.url)) {
next();
return;
}
if (apiHandler) {
if (apiHandler.then) {
apiHandler.then(function (app) {
app(req, res, next);
});
return;
}
apiHandler(req, res, next);
return;
}
apiHandler = require('./lib/vhost-server').create(info.securePort, vhostsdir).create(secureServer, app).then(function (app) {
apiHandler = app;
app(req, res, next);
});
});
function scrubTheDub(req, res/*, next*/) {
// hack for bricked app-cache
if (/\.appcache\b/.test(req.url)) {
res.setHeader('Content-Type', 'text/cache-manifest');
res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*');
return;
}
// TODO port number for non-443
var escapeHtml = require('escape-html');
var newLocation = 'https://' + req.headers.host.replace(/^www\./, '') + req.url;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an old resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
// 301 redirects will not work for appcache
res.end(metaRedirect);
}
app.use('/', function (req, res, next) {
if (/^\/api/.test(req.url)) {
next();
return;
}
// TODO block absolute urls for mounted apps?
// i.e. referer daplie.com/connect requests daplie.com/scripts/blah -> daplie.com/connect/scripts ?
var host = req.headers.host;
var invalidHost = /(\.\.)|[\\:\/\s\|>\*<]/;
if (!host || 'string' !== typeof host) {
next();
return;
}
host = host.toLowerCase();
if (/^www\./.test(host)) {
scrubTheDub(req, res, next);
return;
}
function serveIt() {
// TODO redirect GET /favicon.ico to GET (req.headers.referer||'') + /favicon.ico
// TODO other common root things - robots.txt, app-icon, etc
staticHandlers[host].favicon(req, res, function (err) {
if (err) {
next(err);
return;
}
staticHandlers[host](req, res, next);
});
}
if (staticHandlers[host]) {
if (staticHandlers[host].then) {
staticHandlers[host].then(function () {
serveIt();
}, function (err) {
res.send({
error: {
message: err.message
, code: err.code
}
});
});
return;
}
serveIt();
return;
}
staticHandlers[host] = PromiseA.resolve().then(function () {
var fs = PromiseA.promisifyAll(require('fs'));
// host can be spoofed by the user, so lets be safe
// don't allow .. or / or whitespace
// RFC says domains must start with a-zA-Z0-9 and follow with normal characters
// HOWEVER, there are now Unicode character domains
// punycode?
//
if (invalidHost.test(host)) {
return PromiseA.reject({
message: "invalid Host header"
, code: "E_INVALID_HOST"
});
}
return fs.readdirAsync(path.join(__dirname, 'sites-enabled')).then(function (nodes) {
nodes.forEach(function (node) {
if ('function' === typeof staticHandlers[host] && !staticHandlers[host].then) {
return;
}
// ignore .gitkeep and folders without a .
if (0 === node.indexOf('.') || -1 === node.indexOf('.') || invalidHost.test(node)) {
return;
}
console.log('vhost static');
console.log(node);
staticHandlers[node] = require('serve-static')(path.join(__dirname, 'sites-enabled', node));
try {
// TODO look for favicon
staticHandlers[node].favicon = require('serve-favicon')(path.join(__dirname, 'sites-enabled', node, 'favicon.ico'));
} catch(e) {
staticHandlers[node].favicon = function (req, res, next) { next(); };
}
});
if (staticHandlers[host]) {
serveIt();
} else {
next();
}
return staticHandlers[host];
});
});
});
workerApp = app;
return app;
});
return workerApp;
}
if (info.certPaths) {
promiseServer = require('./lib/sni-server').create(info.certPaths, info.securePort, promiseApps);
} else {
promiseServer = require('./lib/local-server').create(info.securePort, promiseApps);
}
}
process.on('message', function (msg) {
if ('init' === msg.type) {
init(msg);
function waitForInit(message) {
if ('com.daplie.walnut.init' !== message.type) {
console.log('[Worker] 0 got unexpected message:');
console.log(message);
return;
}
console.log('[Worker] got unexpected message:');
var msg = message.conf;
process.removeListener('message', waitForInit);
require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) {
if (err) {
console.log('[ERROR] worker.js');
console.error(err.stack);
throw err;
}
console.log("#" + id + " Listening on " + msg.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
var PromiseA = require('bluebird');
return new PromiseA(function (resolve) {
function initWebServer(srvmsg) {
if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) {
console.log('[Worker] 1 got unexpected message:');
console.log(srvmsg);
return;
}
process.removeListener('message', initWebServer);
resolve(require('./lib/worker').create(webserver, srvmsg));
}
process.send({ type: 'com.daplie.walnut.webserver.listening' });
process.on('message', initWebServer);
});
});
}
// We have to wait to get the configuration from the master process
// before we can start our webserver
console.log('[Worker #' + id + '] online!');
process.on('message', waitForInit);
//
// Debugging
//
process.on('exit', function (code) {
// only sync code can run here
console.log('uptime:', process.uptime());
console.log(process.memoryUsage());
console.log('[exit] process.exit() has been called (or master has killed us).');
console.log(code);
});
process.on('beforeExit', function (msg) {
// async can be scheduled here
console.log('[beforeExit] Event Loop is empty. Process will end.');
console.log(msg);
});
process.on('unhandledRejection', function (err) {
// this should always throw
// (it means somewhere we're not using bluebird by accident)
console.error('[unhandledRejection]');
console.error(err.stack);
throw err;
});
process.on('rejectionHandled', function (msg) {
console.error('[rejectionHandled]');
console.error(msg);
});