coolaj86
/
walnut.js
1
1
フォーク 1
コード イシュー プルリクエスト リリース アクティビティ

Houston, this is big bird. We are multicore!

このコミットが含まれているのは:
AJ ONeal 2015-11-04 09:22:00 +00:00
親 045f3621d6
コミット 13dd677a27
8個のファイルの変更、457行の追加、306行の削除
統計情報を表示 Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む

26
bin/walnut
ファイルの表示

@ -12,33 +12,24 @@ function eagerLoad() {
var PromiseA = require('bluebird').Promise;
var promise = PromiseA.resolve();
[ 'passport'
, 'knex'
, 'bookshelf'
, 'express'
[ 'express'
, 'request'
, 'sqlite3'
, 'body-parser'
, 'express-session'
, 'urlrouter'
, 'express-lazy'
, 'connect-send-error'
, 'underscore.string'
, 'bookshelf'
, 'secret-utils'
, 'connect-cors'
, 'uuid'
, 'connect-recase'
, 'passport-local'
, 'passport-strategy'
, 'passport-http'
, 'passport-http-bearer'
, 'escape-string-regexp'
, 'connect-query'
, 'recase'
].forEach(function (name, i) {
].forEach(function (name/*, i*/) {
promise = promise.then(function () {
return new PromiseA(function (resolve, reject) {
return new PromiseA(function (resolve/*, reject*/) {
setTimeout(function () {
require(name);
resolve();
@ -48,17 +39,6 @@ function eagerLoad() {
});
[ function () {
return require('knex').initialize({
client: 'sqlite3'
, connection: {
filename : ':memory:'
}
});
}
, function (knex) {
require('bookshelf').initialize(knex);
}
, function () {
require('body-parser').json();
}
/*

17
init.public/index.html ノーマルファイル
ファイルの表示

@ -0,0 +1,17 @@
<html>
<head>
<title>Device Locked</title>
</head>
<body>
<h1>Device Locked</h1>
<p>This device is locked and can only be unlocked with the device's encryption key.</p>
<!-- TODO QR Code -->
<form method="POST" action="/api/unlock-device">
<label>Encryption Key</label>
<input type="text">
<button type="submit">Unlock Device</button>
</form>
<script src="./scripts/jquery.js">
<script src="./scripts/app.js">
</body>
</html>

116
lib/sni-server.js ノーマルファイル
ファイルの表示

@ -0,0 +1,116 @@
'use strict';
module.exports.create = function (certPaths, securePort, promiseApp) {
var https = require('https');
// there are a few things that must exist on every core anyway
var secureContexts = {};
function loadCerts(domainname, prevdomainname) {
var PromiseA = require('bluebird');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
if (secureContexts[domainname]) {
return PromiseA.resolve(secureContexts[domainname]);
}
return PromiseA.some(certPaths.map(function (pathname) {
return PromiseA.all([
fs.readFileAsync(path.join(pathname, domainname, 'privkey.pem'), 'ascii')
, fs.readFileAsync(path.join(pathname, domainname, 'fullchain.pem'), 'ascii')
]);
}), 1).then(function (some) {
var one = some[0];
secureContexts[domainname] = require('tls').createSecureContext({
key: one[0]
, cert: one[1]
// https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
// https://nodejs.org/api/tls.html
// removed :ECDH+AES256:DH+AES256 and added :!AES256 because AES-256 wastes CPU
, ciphers: 'ECDH+AESGCM:DH+AESGCM:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS:!AES256'
, honorCipherOrder: true
});
// guard against race condition on Promise.some
if (prevdomainname && !secureContexts[prevdomainname]) {
// TODO XXX make sure that letsencrypt www. domains handle the bare domains also (and vice versa)
secureContexts[prevdomainname] = secureContexts[domainname];
}
return secureContexts[domainname];
}, function (/*err*/) {
// AggregateError means both promises failed
// TODO check ENOENT
// test "is this server <<domainname>>?"
// try letsencrypt
// fail with www.example.com
if (/^www\./i.test(domainname)) {
return loadCerts(domainname.replace(/^www\./i, ''), domainname);
}
return (secureContexts['www.example.com'] || secureContexts['example.com']);
}).then(function (ctx) {
// TODO generate some self-signed certs?
if (!ctx) {
console.error("[loadCerts()] Could not load default HTTPS certificates!!!");
return PromiseA.reject({
message: "No default certificates for https"
, code: 'E_NO_DEFAULT_CERTS'
});
}
return ctx;
});
}
function createSecureServer() {
return loadCerts('www.example.com').then(function (secureOpts) {
//SNICallback is passed the domain name, see NodeJS docs on TLS
secureOpts.SNICallback = function (domainname, cb) {
if (/(^|\.)proxyable\./.test(domainname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
domainname = domainname.replace(/.*\.?proxyable\./, '');
}
loadCerts(domainname).then(function (context) {
cb(null, context);
}, function (err) {
console.error('[SNI Callback]');
console.error(err.stack);
cb(err);
});
};
return https.createServer(secureOpts);
});
}
return createSecureServer().then(function (secureServer) {
var PromiseA = require('bluebird');
return new PromiseA(function (resolve, reject) {
secureServer.on('error', reject);
secureServer.listen(securePort, function () {
resolve(secureServer);
});
// Get up and listening as absolutely quickly as possible
secureServer.on('request', function (req, res) {
if (/(^|\.)proxyable\./.test(req.headers.host)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, '');
}
promiseApp().then(function (app) {
app(req, res);
});
});
});
});
};

83
lib/unlock-device.js ノーマルファイル
ファイルの表示

@ -0,0 +1,83 @@
'use strict';
module.exports.create = function () {
var PromiseA = require('bluebird');
var express = require('connect');
var app = express();
var promise;
promise = new PromiseA(function (resolve) {
var path = require('path');
var serveStatic;
var serveInitStatic;
var jsonParser;
//var rootMasterKey;
app.use(function (req, res, next) {
console.log('yo yo yo soldya boy!', req.url);
res.setHeader('Connection', 'close');
next();
});
app.use('/api/unlock-device', function (req, res, next) {
console.log('[unlock-device]');
if (!jsonParser) {
jsonParser = require('body-parser').json({
strict: true // only objects and arrays
, inflate: true
, limit: 100 * 1024
, reviver: undefined
, type: 'json'
, verify: undefined
});
}
jsonParser(req, res, function (err) {
if (err) {
console.log('[unlock-device] err', err, err.stack);
next(err);
return;
}
console.log('[unlock-device] with root');
resolve("ROOT MASTER KEY");
//setRootMasterKey();
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Location', '/');
res.statusCode = 302;
res.end(JSON.stringify({ success: true }));
});
});
app.use('/api', function (req, res) {
console.log('[d] /api');
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.statusCode = 200;
res.end(JSON.stringify({
error: {
message: "This device is locked. It must be unlocked with its encryption key at /unlock-device"
, code: 'E_DEVICE_LOCKED'
, uri: '/unlock-device' }
}
));
});
// TODO break application cache?
// TODO serve public sites?
app.use('/', function (req, res, next) {
console.log('[pub] /');
if (!serveInitStatic) {
serveStatic = require('serve-static');
serveInitStatic = serveStatic(path.join(__dirname, '..', 'init.public'));
}
serveInitStatic(req, res, next);
});
});
return PromiseA.resolve({
app: app
, promise: promise
});
};

206
lib/vhost-sni-server.js
ファイルの表示

@ -1,36 +1,14 @@
'use strict';
module.exports.create = function (securePort, certsPath, vhostsdir) {
module.exports.create = function (securePort, vhostsdir) {
var PromiseA = require('bluebird').Promise;
var serveStatic;
var https = require('https');
var fs = require('fs');
var path = require('path');
var dummyCerts;
var serveFavicon;
var secureContexts = {};
var loopbackApp;
var loopbackToken = require('crypto').randomBytes(32).toString('hex');
function loadDummyCerts() {
if (dummyCerts) {
return dummyCerts;
}
dummyCerts = {
key: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.key.pem'))
, cert: fs.readFileSync(path.join(certsPath, 'server', 'dummy-server.crt.pem'))
, ca: fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) {
return /crt\.pem$/.test(node);
}).map(function (node) {
console.log('[log dummy ca]', node);
return fs.readFileSync(path.join(certsPath, 'ca', node));
})
};
return dummyCerts;
}
function handleAppScopedError(tag, domaininfo, req, res, fn) {
function next(err) {
if (!err) {
@ -78,15 +56,6 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
return next;
}
function createSecureContext(certs) {
// workaround for v0.12 / v1.2 backwards compat
try {
return require('tls').createSecureContext(certs);
} catch(e) {
return require('crypto').createCredentials(certs).context;
}
}
function createPromiseApps(secureServer) {
return new PromiseA(function (resolve) {
var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA);
@ -335,7 +304,7 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
console.error("[ERROR] initialization failed during create() for " + domaininfo.dirname);
console.error(e);
throw e;
return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname);
//return getDummyAppContext(e, "[ERROR] initialization failed during create() for " + domaininfo.dirname);
});
}
} catch(e) {
@ -355,7 +324,7 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
console.log('[log] [once] Loading all mounts for ' + domainApp.hostname);
domainApp._loaded = true;
app.use(vhost(domainApp.hostname, domainApp.apps));
app.use(vhost('www.' + domainApp.hostname, function (req, res, next) {
app.use(vhost('www.' + domainApp.hostname, function (req, res/*, next*/) {
if (/\.appcache\b/.test(req.url)) {
res.setHeader('Content-Type', 'text/cache-manifest');
res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*');
@ -453,172 +422,5 @@ module.exports.create = function (securePort, certsPath, vhostsdir) {
});
}
function loadCerts(domainname) {
// TODO make async
// WARNING: This must be SYNC until we KNOW we're not going to be running on v0.10
// Also, once we load Let's Encrypt, it's lights out for v0.10
var certsPath = path.join(vhostsdir, domainname, 'certs');
var secOpts;
try {
var nodes = fs.readdirSync(certsPath);
var keyNode = nodes.filter(function (node) { return 'privkey.pem' === node; })[0];
var crtNode = nodes.filter(function (node) { return 'fullchain.pem' === node; })[0];
if (keyNode && crtNode) {
keyNode = path.join(certsPath, keyNode);
crtNode = path.join(certsPath, crtNode);
} else {
nodes = fs.readdirSync(path.join(certsPath, 'server'));
keyNode = nodes.filter(function (node) { return /^privkey(\.key)?\.pem$/.test(node) || /\.key\.pem$/.test(node); })[0];
crtNode = nodes.filter(function (node) { return /^fullchain(\.crt)?\.pem$/.test(node) || /\.crt\.pem$/.test(node); })[0];
keyNode = path.join(certsPath, 'server', keyNode);
crtNode = path.join(certsPath, 'server', crtNode);
}
secOpts = {
key: fs.readFileSync(keyNode)
, cert: fs.readFileSync(crtNode)
};
// I misunderstood what the ca option was for
/*
if (fs.existsSync(path.join(certsPath, 'ca'))) {
secOpts.ca = fs.readdirSync(path.join(certsPath, 'ca')).filter(function (node) {
console.log('[log ca]', node);
return /crt\.pem$/.test(node);
}).map(function (node) {
return fs.readFileSync(path.join(certsPath, 'ca', node));
});
}
*/
} catch(err) {
// TODO Let's Encrypt / ACME HTTPS
console.error("[ERROR] Couldn't READ HTTPS certs from '" + certsPath + "':");
// this will be a simple file-read error
console.error(err.message);
return null;
}
try {
secureContexts[domainname] = createSecureContext(secOpts);
} catch(err) {
console.error("[ERROR] Certificates in '" + certsPath + "' could not be used:");
console.error(err);
return null;
}
if (!secureContexts[domainname]) {
console.error("[ERROR] Sanity check fail, no cert for '" + domainname + "'");
return null;
}
return secureContexts[domainname];
}
function createSecureServer() {
var localDummyCerts = loadDummyCerts();
var secureOpts = {
// fallback / default dummy certs
key: localDummyCerts.key
, cert: localDummyCerts.cert
//, ca: localDummyCerts.ca
// io.js defaults have disallowed insecure algorithms as of 2015-06-29
// https://iojs.org/api/tls.html
// previous version could use something like this
//, ciphers: "ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:AES128-GCM-SHA256:!RC4:HIGH:!MD5:!aNULL"
};
function addSniWorkaroundCallback() {
//SNICallback is passed the domain name, see NodeJS docs on TLS
secureOpts.SNICallback = function (domainname, cb) {
domainname = domainname.replace(/^www\./, '')
if (/(^|\.)proxyable\./.test(domainname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
domainname = domainname.replace(/.*\.?proxyable\./, '');
}
if (!secureContexts.dummy) {
console.log('[log] Loading dummy certs');
secureContexts.dummy = createSecureContext(localDummyCerts);
}
if (!secureContexts[domainname]) {
console.log('[log] Loading certs for', domainname);
// TODO keep trying to find the cert in case it's uploaded late?
secureContexts[domainname] = loadCerts(domainname) || secureContexts.dummy;
}
// workaround for v0.12 / v1.2 backwards compat bug
if ('function' === typeof cb) {
cb(null, secureContexts[domainname] || secureContexts.dummy);
} else {
return secureContexts[domainname] || secureContexts.dummy;
}
};
}
addSniWorkaroundCallback();
return https.createServer(secureOpts);
}
function runServer() {
return new PromiseA(function (resolve) {
var secureServer = createSecureServer();
var promiseApps;
function loadPromise() {
if (!promiseApps) {
promiseApps = createPromiseApps(secureServer);
}
return promiseApps;
}
secureServer.listen(securePort, function () {
resolve(secureServer);
console.log("Listening on https://localhost:" + secureServer.address().port, '\n');
loadPromise();
});
// Get up and listening as absolutely quickly as possible
secureServer.on('request', function (req, res) {
if (/(^|\.)proxyable\./.test(req.headers.host)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.proxyable.com => myapp.mydomain.com
req.headers.host = req.headers.host.replace(/.*\.?proxyable\./, '');
}
loadPromise().then(function (app) {
app(req, res);
});
});
return secureServer;
});
}
function updateIps() {
console.log('[UPDATE IP]');
require('./ddns-updater').update().then(function (results) {
results.forEach(function (result) {
if (result.error) {
console.error(result);
} else {
console.log('[SUCCESS]', result.service.hostname);
}
});
}).error(function (err) {
console.error('[UPDATE IP] ERROR');
console.error(err);
});
}
// TODO check the IP every 5 minutes and update it every hour
setInterval(updateIps, 60 * 60 * 1000);
updateIps();
return runServer();
return { create: createPromiseApps };
};

183
master.js ノーマルファイル
ファイルの表示

@ -0,0 +1,183 @@
'use strict';
console.log('\n\n\n[MASTER] Welcome to WALNUT!');
var PromiseA = require('bluebird');
var cluster = require('cluster');
var numCores = require('os').cpus().length;
var securePort = process.argv[2] || 443;
var insecurePort = process.argv[3] || 80;
var secureServer;
var rootMasterKey;
var redirects = require('./redirects.json');
var path = require('path');
// force SSL upgrade server
var certPaths = [path.join(__dirname, 'certs', 'live')];
var promiseServer;
var masterApp;
//console.log('\n.');
// Note that this function will be called async, after promiseServer is returned
// it seems like a circular dependency, but it isn't... not exactly anyway
function promiseApps() {
if (masterApp) {
return PromiseA.resolve(masterApp);
}
masterApp = promiseServer.then(function (_secureServer) {
secureServer = _secureServer;
console.log("[MASTER] Listening on https://localhost:" + secureServer.address().port, '\n');
return require('./lib/unlock-device').create().then(function (result) {
result.promise.then(function (_rootMasterKey) {
var i;
rootMasterKey = _rootMasterKey;
for (i = 0; i < numCores; i += 1) {
cluster.fork();
}
});
masterApp = result.app;
return result.app;
});
});
return masterApp;
}
// TODO have a fallback server than can download and apply an update?
require('./lib/insecure-server').create(securePort, insecurePort, redirects);
//console.log('\n.');
promiseServer = require('./lib/sni-server').create(certPaths, securePort, promiseApps);
//console.log('\n.');
cluster.on('online', function (worker) {
console.log('[MASTER] Worker ' + worker.process.pid + ' is online');
if (secureServer) {
// NOTE: it's possible that this could survive idle for a while through keep-alive
// should default to connection: close
secureServer.close();
secureServer = null;
setTimeout(function () {
// TODO use `id' to find user's uid / gid and set to file
// TODO set immediately?
process.setgid(1000);
process.setuid(1000);
}, 1000);
}
worker.send({
type: 'init'
, securePort: securePort
, certPaths: certPaths
});
worker.on('message', function (msg) {
console.log('message from worker');
console.log(msg);
});
});
cluster.on('exit', function (worker, code, signal) {
console.log('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
cluster.fork();
});
// TODO delegate to workers
function updateIps() {
console.log('[UPDATE IP]');
require('./lib/ddns-updater').update().then(function (results) {
results.forEach(function (result) {
if (result.error) {
console.error(result);
} else {
console.log('[SUCCESS]', result.service.hostname);
}
});
}).error(function (err) {
console.error('[UPDATE IP] ERROR');
console.error(err);
});
}
// TODO check the IP every 5 minutes and update it every hour
setInterval(updateIps, 60 * 60 * 1000);
// we don't want this to load right away (extra procesing time)
setTimeout(updateIps, 1);
/*
worker.send({
insecurePort: insecurePort
});
*/
/*
var fs = require('fs');
var daplieReadFile = fs.readFileSync;
var time = 0;
fs.readFileSync = function (filename) {
var now = Date.now();
var data = daplieReadFile.apply(fs, arguments);
var t;
t = (Date.now() - now);
time += t;
console.log('loaded "' + filename + '" in ' + t + 'ms (total ' + time + 'ms)');
return data;
};
*/
//var config = require('./device.json');
// require('ssl-root-cas').inject();
/*
function phoneHome() {
var holepunch = require('./holepunch/beacon');
var ports;
ports = [
{ private: 65022
, public: 65022
, protocol: 'tcp'
, ttl: 0
, test: { service: 'ssh' }
, testable: false
}
, { private: 650443
, public: 650443
, protocol: 'tcp'
, ttl: 0
, test: { service: 'https' }
}
, { private: 65080
, public: 65080
, protocol: 'tcp'
, ttl: 0
, test: { service: 'http' }
}
];
// TODO return a middleware
holepunch.run(require('./redirects.json').reduce(function (all, redirect) {
if (!all[redirect.from.hostname]) {
all[redirect.from.hostname] = true;
all.push(redirect.from.hostname);
}
if (!all[redirect.to.hostname]) {
all[redirect.to.hostname] = true;
all.push(redirect.to.hostname);
}
return all;
}, []), ports).catch(function () {
console.error("Couldn't phone home. Oh well");
});
}
*/

86
walnut.js
ファイルの表示

@ -1,85 +1,9 @@
'use strict';
console.log('\n\n\nWelcome to WALNUT!');
var cluster = require('cluster');
/*
var fs = require('fs');
var daplieReadFile = fs.readFileSync;
var time = 0;
fs.readFileSync = function (filename) {
var now = Date.now();
var data = daplieReadFile.apply(fs, arguments);
var t;
t = (Date.now() - now);
time += t;
console.log('loaded "' + filename + '" in ' + t + 'ms (total ' + time + 'ms)');
return data;
};
*/
var PromiseA = require('bluebird').Promise;
//var config = require('./device.json');
var securePort = process.argv[2] || 443;
var insecurePort = process.argv[3] || 80;
var redirects = require('./redirects.json');
var path = require('path');
// force SSL upgrade server
var certsPath = path.join(__dirname, 'certs');
// require('ssl-root-cas').inject();
var vhostsdir = path.join(__dirname, 'vhosts');
function phoneHome() {
var holepunch = require('./holepunch/beacon');
var ports;
ports = [
{ private: 65022
, public: 65022
, protocol: 'tcp'
, ttl: 0
, test: { service: 'ssh' }
, testable: false
}
, { private: 650443
, public: 650443
, protocol: 'tcp'
, ttl: 0
, test: { service: 'https' }
}
, { private: 65080
, public: 65080
, protocol: 'tcp'
, ttl: 0
, test: { service: 'http' }
}
];
// TODO return a middleware
holepunch.run(require('./redirects.json').reduce(function (all, redirect) {
if (!all[redirect.from.hostname]) {
all[redirect.from.hostname] = true;
all.push(redirect.from.hostname);
}
if (!all[redirect.to.hostname]) {
all[redirect.to.hostname] = true;
all.push(redirect.to.hostname);
}
return all;
}, []), ports).catch(function () {
console.error("Couldn't phone home. Oh well");
});
if (cluster.isMaster) {
require('./master');
} else {
require('./worker');
}
PromiseA.all([
require('./lib/insecure-server').create(securePort, insecurePort, redirects)
, require('./lib/vhost-sni-server.js').create(securePort, certsPath, vhostsdir)
]).then(function () {
// TODO use `id' to find user's uid / gid and set to file
process.setgid(1000);
process.setuid(1000);
});//.then(phoneHome);

46
worker.js ノーマルファイル
ファイルの表示

@ -0,0 +1,46 @@
'use strict';
var cluster = require('cluster');
var id = cluster.worker.id.toString();
var path = require('path');
var vhostsdir = path.join(__dirname, 'vhosts');
console.log('[Worker #' + id + '] online!');
function init(info) {
var promiseServer;
var workerApp;
function promiseApps() {
var PromiseA = require('bluebird');
if (workerApp) {
return PromiseA.resolve(workerApp);
}
workerApp = promiseServer.then(function (secureServer) {
//secureServer = _secureServer;
console.log("#" + id + " Listening on https://localhost:" + secureServer.address().port, '\n');
return require('./lib/vhost-sni-server').create(info.securePort, vhostsdir).create(secureServer).then(function (app) {
workerApp = app;
return app;
});
});
return workerApp;
}
promiseServer = require('./lib/sni-server').create(info.certPaths, info.securePort, promiseApps);
}
process.on('message', function (msg) {
if ('init' === msg.type) {
init(msg);
return;
}
console.log('[Worker] got unexpected message:');
console.log(msg);
});