coolaj86
/
walnut.js
1
1
Fork 1
Code Issues Pull Requests Releases Activity

refactoring to use fs config

This commit is contained in:
AJ ONeal 2016-04-09 19:14:00 -04:00
parent 28db03ae23
commit d792404d67
17 changed files with 1708 additions and 738 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/walnut.js
View File

@ -62,4 +62,7 @@ function eagerLoad() {
});
}
setTimeout(eagerLoad, 100);
// this isn't relevant to do in the master process, duh
if (false) {
setTimeout(eagerLoad, 100);
}

59
boot/local-server.js Normal file
View File

@ -0,0 +1,59 @@
'use strict';
// Note the odd use of callbacks (instead of promises) here
// It's to avoid loading bluebird yet (see sni-server.js for explanation)
module.exports.create = function (lex, certPaths, port, conf, serverCallback) {
function initServer(err, server) {
var app;
var promiseApp;
if (err) {
serverCallback(err);
return;
}
server.on('error', serverCallback);
server.listen(port, function () {
// is it even theoritically possible for
// a request to come in before this callback has fired?
// I'm assuming this event must fire before any request event
promiseApp = serverCallback(null, server);
});
/*
server.listen(port, '::::', function () {
// is it even theoritically possible for
// a request to come in before this callback has fired?
// I'm assuming this event must fire before any request event
promiseApp = serverCallback(null, server);
});
*/
// Get up and listening as absolutely quickly as possible
function onRequest(req, res) {
// this is a hot piece of code, so we cache the result
if (app) {
app(req, res);
return;
}
promiseApp.then(function (_app) {
console.log('[Server]', req.method, req.host || req.headers['x-forwarded-host'] || req.headers.host, req.url);
app = _app;
app(req, res);
});
}
if (lex) {
var LEX = require('letsencrypt-express');
server.on('request', LEX.createAcmeResponder(lex, onRequest));
} else {
server.on('request', onRequest);
}
}
if (certPaths) {
require('../lib/sni-server').create(lex, certPaths, initServer);
} else {
initServer(null, require('http').createServer());
}
};

77
boot/master.js
View File

@ -25,28 +25,44 @@ var workers = [];
var state = { firstRun: true };
// TODO Should these be configurable? If so, where?
// TODO communicate config with environment vars?
var walnut = tryConf(
path.join('..', '..', 'config.walnut')
, { externalPort: 443
, externalInsecurePort: 80
, certspath: path.join(__dirname, '..', '..', 'certs', 'live')
}
);
var caddy = tryConf(
path.join('..', '..', 'config.caddy.json')
, { conf: null // __dirname + '/Caddyfile'
path.join('..', '..', 'config.caddy')
, { conf: path.join(__dirname, '..', '..', 'Caddyfile')
, bin: null // '/usr/local/bin/caddy'
, sitespath: null // path.join(__dirname, 'sites-enabled')
, locked: false // true
}
);
var useCaddy = require('fs').existsSync(caddy.bin);
var letsencrypt = tryConf(
path.join('..', '..', 'config.letsencrypt')
, { configDir: path.join(__dirname, '..', '..', 'letsencrypt')
, email: null
, agreeTos: false
}
);
var useCaddy = caddy.bin && require('fs').existsSync(caddy.bin);
var info = {
type: 'walnut.init'
, conf: {
protocol: useCaddy ? 'http' : 'https'
, externalPort: 443
, externalPortInsecure: 80 // TODO externalInsecurePort
, localPort: process.argv[2] || (useCaddy ? 4080 : 443) // system / local network
, insecurePort: process.argv[3] || (useCaddy ? 80 : 80) // meh
, externalPort: walnut.externalPort
, externalPortInsecure: walnut.externalInsecurePort // TODO externalInsecurePort
, localPort: walnut.localPort || (useCaddy ? 4080 : 443) // system / local network
, insecurePort: walnut.insecurePort || (useCaddy ? 80 : 80) // meh
, certPaths: useCaddy ? null : [
path.join(__dirname, '..', '..', 'certs', 'live')
, path.join(__dirname, '..', '..', 'letsencrypt', 'live')
walnut.certspath
, path.join(letsencrypt.configDir, 'live')
]
, trustProxy: useCaddy ? true : false
, lexConf: letsencrypt
, varpath: path.join(__dirname, '..', '..', 'var')
}
};
@ -67,6 +83,7 @@ cluster.on('online', function (worker) {
// relies on { localPort, locked }
caddy.spawn(caddy);
}
// TODO dyndns in master?
}
function touchMaster(msg) {
@ -76,47 +93,11 @@ cluster.on('online', function (worker) {
return;
}
// calls init if init has not been called
state.caddy = caddy;
state.workers = workers;
require('../lib/master').touch(info.conf, state).then(function (results) {
//var memstore = results.memstore;
var sqlstore = results.sqlstore;
info.type = 'walnut.webserver.onrequest';
// TODO let this load after server is listening
info.conf['org.oauth3.consumer'] = results['org.oauth3.consumer'];
info.conf['org.oauth3.provider'] = results['org.oauth3.provider'];
info.conf.keys = results.keys;
//info.conf.memstoreSock = config.memstoreSock;
//info.conf.sqlite3Sock = config.sqlite3Sock;
// TODO get this from db config instead
//info.conf.privkey = config.privkey;
//info.conf.pubkey = config.pubkey;
info.conf.redirects = [
{ "ip": false, "id": "*", "value": false } // default no-www
, { "ip": false, "id": "daplie.domains", "value": null }
, { "ip": false, "id": "*.daplie.domains", "value": false }
, { "ip": false, "id": "no.daplie.domains", "value": false }
, { "ip": false, "id": "*.no.daplie.domains", "value": false }
, { "ip": false, "id": "ns2.daplie.domains", "value": false }
, { "ip": true, "id": "maybe.daplie.domains", "value": null }
, { "ip": true, "id": "*.maybe.daplie.domains", "value": null }
, { "ip": true, "id": "www.daplie.domains", "value": null }
, { "ip": true, "id": "yes.daplie.domains", "value": true }
, { "ip": true, "id": "*.yes.daplie.domains", "value": true }
, { "ip": true, "id": "ns1.daplie.domains", "value": false }
];
// TODO use sqlite3 or autogenerate ?
info.conf.privkey = require('fs').readFileSync(__dirname + '/../../' + '/nsx.redirect-www.org.key.pem', 'ascii');
info.conf.pubkey = require('fs').readFileSync(__dirname + '/../../' + '/nsx.redirect-www.org.key.pem.pub', 'ascii');
// keys
// letsencrypt
// com.example.provider
// com.example.consumer
worker.send(info);
// calls init if init has not been called
require('../lib/master').touch(info.conf, state).then(function (newConf) {
worker.send({ type: 'walnut.webserver.onrequest', conf: newConf });
});
}

150
boot/worker.js
View File

@ -4,25 +4,93 @@ module.exports.create = function (opts) {
var id = '0';
var promiseApp;
function createAndBindInsecure(lex, message, cb) {
function createAndBindInsecure(lex, conf, getOrCreateHttpApp) {
// TODO conditional if 80 is being served by caddy
require('../lib/insecure-server').create(lex, message.conf.externalPort, message.conf.insecurePort, message, function (err, webserver) {
console.info("#" + id + " Listening on http://" + webserver.address().address + ":" + webserver.address().port, '\n');
// we are returning the promise result to the caller
return cb(null, webserver, null, message);
var appPromise = null;
var app = null;
var http = require('http');
var insecureServer = http.createServer();
function onRequest(req, res) {
if (app) {
app(req, res);
return;
}
if (!appPromise) {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.end('{ "error": { "code": "E_SANITY_FAIL", "message": "should have an express app, but didn\'t" } }');
return;
}
appPromise.then(function (_app) {
appPromise = null;
app = _app;
app(req, res);
});
}
insecureServer.listen(conf.insecurePort, function () {
console.info("#" + id + " Listening on http://"
+ insecureServer.address().address + ":" + insecureServer.address().port, '\n');
appPromise = getOrCreateHttpApp(null, insecureServer);
if (!appPromise) {
throw new Error('appPromise returned nothing');
}
});
insecureServer.on('request', onRequest);
}
function walkLe(domainname) {
var PromiseA = require('bluebird');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
var parts = domainname.split('.'); //.replace(/^www\./, '').split('.');
var configname = parts.join('.') + '.json';
var configpath = path.join(__dirname, '..', '..', 'config', configname);
if (parts.length < 2) {
return PromiseA.resolve(null);
}
// TODO configpath a la varpath
return fs.readFileAsync(configpath, 'utf8').then(function (text) {
var data = JSON.parse(text);
data.name = configname;
return data;
}, function (/*err*/) {
parts.shift();
return walkLe(parts.join('.'));
});
}
function createLe(conf) {
function createLe(lexConf, conf) {
var LEX = require('letsencrypt-express');
var lex = LEX.create({
configDir: conf.letsencrypt.configDir // i.e. __dirname + '/letsencrypt.config'
configDir: lexConf.configDir // i.e. __dirname + '/letsencrypt.config'
, approveRegistration: function (hostname, cb) {
cb(null, {
domains: [hostname] // TODO handle www and bare on the same cert
, email: conf.letsencrypt.email
, agreeTos: conf.letsencrypt.agreeTos
// TODO cache/report unauthorized
if (!hostname) {
cb(new Error("[lex.approveRegistration] undefined hostname"), null);
return;
}
walkLe(hostname).then(function (leAuth) {
// TODO should still check dns for hostname (and mx for email)
if (leAuth && leAuth.email && leAuth.agreeTos) {
cb(null, {
domains: [hostname] // TODO handle www and bare on the same cert
, email: leAuth.email
, agreeTos: leAuth.agreeTos
});
}
else {
// TODO report unauthorized
cb(new Error("Valid LetsEncrypt config with email and agreeTos not found for '" + hostname + "'"), null);
}
});
/*
letsencrypt.getConfig({ domains: [domain] }, function (err, config) {
@ -42,81 +110,92 @@ module.exports.create = function (opts) {
*/
}
});
//var letsencrypt = lex.letsencrypt;
conf.letsencrypt = lex.letsencrypt;
conf.lex = lex;
conf.walkLe = walkLe;
return lex;
}
function createAndBindServers(message, cb) {
function createAndBindServers(conf, getOrCreateHttpApp) {
var lex;
if (message.conf.letsencrypt) {
lex = createLe(message.conf);
if (conf.lexConf) {
lex = createLe(conf.lexConf, conf);
}
// NOTE that message.conf[x] will be overwritten when the next message comes in
require('../lib/local-server').create(lex, message.conf.certPaths, message.conf.localPort, message, function (err, webserver) {
require('./local-server').create(lex, conf.certPaths, conf.localPort, conf, function (err, webserver) {
if (err) {
console.error('[ERROR] worker.js');
console.error(err.stack);
throw err;
}
console.info("#" + id + " Listening on " + message.conf.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
console.info("#" + id + " Listening on " + conf.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n');
// we don't need time to pass, just to be able to return
process.nextTick(function () {
createAndBindInsecure(lex, message, cb);
createAndBindInsecure(lex, conf, getOrCreateHttpApp);
});
// we are returning the promise result to the caller
return cb(null, null, webserver, message);
return getOrCreateHttpApp(null, null, webserver, conf);
});
}
//
// Worker Mode
//
function waitForConfig(message) {
if ('walnut.init' !== message.type) {
function waitForConfig(realMessage) {
if ('walnut.init' !== realMessage.type) {
console.warn('[Worker] 0 got unexpected message:');
console.warn(message);
console.warn(realMessage);
return;
}
var conf = realMessage.conf;
process.removeListener('message', waitForConfig);
// NOTE: this callback must return a promise for an express app
createAndBindServers(message, function (err, insecserver, webserver, oldMessage) {
// TODO deep merge new message into old message
Object.keys(message.conf).forEach(function (key) {
oldMessage.conf[key] = message.conf[key];
});
function getExpressApp(err, insecserver, webserver/*, newMessage*/) {
var PromiseA = require('bluebird');
if (promiseApp) {
return promiseApp;
}
promiseApp = new PromiseA(function (resolve) {
function initWebServer(srvmsg) {
function initHttpApp(srvmsg) {
if ('walnut.webserver.onrequest' !== srvmsg.type) {
console.warn('[Worker] 1 got unexpected message:');
console.warn('[Worker] [onrequest] unexpected message:');
console.warn(srvmsg);
return;
}
process.removeListener('message', initWebServer);
process.removeListener('message', initHttpApp);
resolve(require('../lib/worker').create(webserver, srvmsg.conf));
if (srvmsg.conf) {
Object.keys(srvmsg.conf).forEach(function (key) {
conf[key] = srvmsg.conf[key];
});
}
resolve(require('../lib/worker').create(webserver, conf));
}
process.send({ type: 'walnut.webserver.listening' });
process.on('message', initWebServer);
process.on('message', initHttpApp);
}).then(function (app) {
console.info('[Worker Ready]');
return app;
});
return promiseApp;
});
}
createAndBindServers(conf, getExpressApp);
}
//
@ -124,11 +203,13 @@ module.exports.create = function (opts) {
//
if (opts) {
// NOTE: this callback must return a promise for an express app
createAndBindServers(opts, function (err, insecserver, webserver/*, message*/) {
createAndBindServers(opts, function (err, insecserver, webserver/*, conf*/) {
var PromiseA = require('bluebird');
if (promiseApp) {
return promiseApp;
}
promiseApp = new PromiseA(function (resolve) {
opts.getConfig(function (srvmsg) {
resolve(require('../lib/worker').create(webserver, srvmsg));
@ -137,6 +218,7 @@ module.exports.create = function (opts) {
console.info('[Standalone Ready]');
return app;
});
return promiseApp;
});
} else {

20
install.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
sudo mkdir -p /srv/walnut/{certs,core,letsencrypt,lib}
sudo mkdir -p /srv/walnut/{certs,core,letsencrypt,lib,config}
sudo mkdir -p /srv/walnut/packages/{api,pages,services}
sudo chown -R $(whoami):$(whoami) /srv/walnut
@ -9,11 +9,23 @@ git clone https://github.com/Daplie/walnut.git /srv/walnut/core
pushd /srv/walnut/core
npm install
sudo rsync -av /srv/walnut/core/etc/init/walnut.conf /etc/init/walnut.conf
rsync -av /srv/walnut/core/etc/letsencrypt/ /srv/walnut/certs/
popd
sudo rsync -a /srv/walnut/core/etc/init/walnut.conf /etc/init/walnut.conf
rsync -a /srv/walnut/core/etc/letsencrypt/ /srv/walnut/certs/
mv /srv/walnut/core/node_modules /srv/walnut
echo -n "Enter an email address to use for LetsEncrypt and press [ENTER]: "
read LE_EMAIL
node -e "
'use strict';
require('fs').writeFileSync('/srv/walnut/config.letsencrypt.json', JSON.stringify({
configDir: '/srv/walnut/letsencrypt'
, email: '$LE_EMAIL'
, agreeTos: true
}, null, ' '));
"
sudo service walnut stop
sudo service walnut start

250
lib/apis.js Normal file
View File

@ -0,0 +1,250 @@
'use strict';
module.exports.create = function (xconfx, apiFactories, apiDeps) {
var PromiseA = apiDeps.Promise;
var express = require('express');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
var localCache = { apis: {}, pkgs: {} };
// TODO xconfx.apispath
xconfx.apispath = path.join(__dirname, '..', '..', 'packages', 'apis');
function notConfigured(req, res) {
res.send({ error: { message: "api '" + req.apiId + "' not configured for domain '" + req.experienceId + "'" } });
}
function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
function handlePromise(p) {
return p.then(function (api) {
packagedApi._api = api;
return api;
});
}
if (!packagedApi._promise_api) {
packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
}
return handlePromise(packagedApi._promise_api);
}
function getApi(conf, pkgConf, pkgDeps, packagedApi) {
var PromiseA = pkgDeps.Promise;
var path = require('path');
var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
// TODO needs some version stuff (which would also allow hot-loading of updates)
// TODO version could be tied to sha256sum
return new PromiseA(function (resolve, reject) {
var myApp;
var ursa;
var promise;
// TODO dynamic requires are a no-no
// can we statically generate a require-er? on each install?
// module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
// requirer[pkgpath]()
myApp = pkgDeps.express();
myApp.disable('x-powered-by');
if (pkgDeps.app.get('trust proxy')) {
myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
}
if (!pkgConf.pubkey) {
/*
return ursa.createPrivateKey(pem, password, encoding);
var pem = myKey.toPrivatePem();
return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
});
*/
ursa = require('ursa');
pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
}
try {
packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
packagedApi._apiname = packagedApi._apipkg.name;
if (packagedApi._apipkg.walnut) {
pkgpath += '/' + packagedApi._apipkg.walnut;
}
promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
} catch(e) {
reject(e);
return;
}
promise.then(function () {
// TODO give pub/priv pair for app and all public keys
// packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
packagedApi._api = require('express-lazy')();
packagedApi._api_app = myApp;
//require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
pkgDeps.getOauth3Controllers =
packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
// DEBUG
//
/*
packagedApi._api.use('/', function (req, res, next) {
console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
next();
});
//*/
// TODO fix backwards compat
// /api/com.example.foo (no change)
packagedApi._api.use('/', packagedApi._api_app);
// /api/com.example.foo => /api
packagedApi._api.use('/', function (req, res, next) {
var priorUrl = req.url;
req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
// console.log('api mangle 3:', req.url);
packagedApi._api_app(req, res, function (err) {
req.url = priorUrl;
next(err);
});
});
// /api/com.example.foo => /
packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
// console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
// console.log(packagedApi._api_app.toString());
packagedApi._api_app(req, res, next);
});
resolve(packagedApi._api);
}, reject);
});
}
// Read packages/apis/sub.sld.tld (forward dns) to find list of apis as tld.sld.sub (reverse dns)
// TODO packages/allowed_apis/sub.sld.tld (?)
// TODO auto-register org.oauth3.consumer for primaryDomain (and all sites?)
function loadApiHandler() {
return function handler(req, res, next) {
var name = req.experienceId;
var apiId = req.apiId;
var packagepath = path.join(xconfx.apispath, name);
return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
return text.trim().split(/\n/);
}, function () {
return [];
}).then(function (apis) {
return function (req, res, next) {
var apipath;
if (!apis.some(function (api) {
if (api === apiId) {
return true;
}
})) {
if (req.experienceId === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === apiId) {
// fallthrough
} else {
return null;
}
}
apipath = path.join(xconfx.apispath, apiId);
if (!localCache.pkgs[apiId]) {
return fs.readFileAsync(path.join(apipath, 'package.json'), 'utf8').then(function (text) {
var pkg = JSON.parse(text);
var deps = {};
var myApp;
if (pkg.walnut) {
apipath = path.join(apipath, pkg.walnut);
}
Object.keys(apiDeps).forEach(function (key) {
deps[key] = apiDeps[key];
});
Object.keys(apiFactories).forEach(function (key) {
deps[key] = apiFactories[key];
});
// TODO pull db stuff from package.json somehow and pass allowed data models as deps
//
// how can we tell which of these would be correct?
// deps.memstore = apiFactories.memstoreFactory.create(apiId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + apiId);
// let's go with this one for now and the api can choose to scope or not to scope
deps.memstore = apiFactories.memstoreFactory.create(apiId);
console.log('DEBUG apipath', apipath);
myApp = express();
//
// TODO handle /accounts/:accountId
//
return PromiseA.resolve(require(apipath).create({}/*pkgConf*/, deps/*pkgDeps*/, myApp/*myApp*/)).then(function (handler) {
localCache.pkgs[apiId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
localCache.pkgs[apiId].handler(req, res, next);
});
});
}
else {
localCache.pkgs[apiId].handler(req, res, next);
// TODO expire require cache
/*
if (Date.now() - localCache.pkgs[apiId].createdAt < (5 * 60 * 1000)) {
return;
}
*/
}
};
}, function (/*err*/) {
return null;
}).then(function (handler) {
// keep object reference intact
// DO NOT cache non-existant api
if (handler) {
localCache.apis[name].handler = handler;
} else {
handler = notConfigured;
}
handler(req, res, next);
});
};
}
return function (req, res, next) {
var experienceId = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
var apiId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
Object.defineProperty(req, 'experienceId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: experienceId
});
Object.defineProperty(req, 'apiId', {
enumerable: true
, configurable: false
, writable: false
, value: apiId
});
if (!localCache.apis[experienceId]) {
localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
}
localCache.apis[experienceId].handler(req, res, next);
if (Date.now() - localCache.apis[experienceId].createdAt > (5 * 60 * 1000)) {
localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
}
};
};

306
lib/bootstrap.js vendored Normal file
View File

@ -0,0 +1,306 @@
'use strict';
//
// IMPORTANT !!!
//
// None of this is authenticated or encrypted
//
module.exports.create = function (app, xconfx, models) {
var PromiseA = require('bluebird');
var path = require('path');
var fs = PromiseA.promisifyAll(require('fs'));
var dns = PromiseA.promisifyAll(require('dns'));
function isInitialized() {
// TODO read from file only, not db
return models.ComDaplieWalnutConfig.get('config').then(function (conf) {
if (!conf || !conf.primaryDomain || !conf.primaryEmail) {
console.log('DEBUG incomplete conf', conf);
return false;
}
xconfx.primaryDomain = xconfx.primaryDomain || conf.primaryDomain;
var configname = conf.primaryDomain + '.json';
var configpath = path.join(__dirname, '..', '..', 'config', configname);
return fs.readFileAsync(configpath, 'utf8').then(function (text) {
return JSON.parse(text);
}, function (/*err*/) {
console.log('DEBUG not exists leconf', configpath);
return false;
}).then(function (data) {
if (!data || !data.email || !data.agreeTos) {
console.log('DEBUG incomplete leconf', data);
return false;
}
return true;
});
});
}
function initialize() {
var express = require('express');
var getIpAddresses = require('./ip-checker').getExternalAddresses;
var resolve;
function errorIfNotApi(req, res, next) {
// if it's not an ip address
if (/[a-z]+/.test(req.headers.host)) {
if (!/^api\./.test(req.headers.host)) {
console.log('req.headers.host');
console.log(req.headers.host);
res.send({ error: { message: "no api. subdomain prefix" } });
return;
}
}
next();
}
function errorIfApi(req, res, next) {
if (!/^api\./.test(req.headers.host)) {
next();
return;
}
// has api. hostname prefix
// doesn't have /api url prefix
if (!/^\/api\//.test(req.url)) {
res.send({ error: { message: "missing /api/ url prefix" } });
return;
}
res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
}
function getConfig(req, res) {
getIpAddresses().then(function (inets) {
var results = {
hostname: require('os').hostname()
, inets: inets.addresses.map(function (a) {
a.time = undefined;
return a;
})
};
//res.send({ inets: require('os').networkInterfaces() });
res.send(results);
});
}
function verifyIps(inets, hostname) {
var map = {};
var arr = [];
inets.forEach(function (addr) {
if (!map[addr.family]) {
map[addr.family] = true;
if (4 === addr.family) {
arr.push(dns.resolve4Async(hostname).then(function (arr) {
return arr;
}, function (/*err*/) {
return [];
}));
}
if (6 === addr.family) {
arr.push(dns.resolve6Async(hostname).then(function (arr) {
return arr;
}, function (/*err*/) {
return [];
}));
}
}
});
return PromiseA.all(arr).then(function (fams) {
console.log('DEBUG hostname', hostname);
var ips = [];
fams.forEach(function (addrs) {
console.log('DEBUG ipv46');
console.log(addrs);
addrs.forEach(function (addr) {
inets.forEach(function (a) {
if (a.address === addr) {
a.time = undefined;
ips.push(a);
}
});
});
console.log('');
});
return ips;
});
}
function setConfig(req, res) {
var config = req.body;
var results = {};
return PromiseA.resolve().then(function () {
if (!config.agreeTos && !config.tls) {
return PromiseA.reject(new Error("To enable encryption you must agree to the LetsEncrypt terms of service"));
}
if (!config.domain) {
return PromiseA.reject(new Error("You must specify a valid domain name"));
}
config.domain = config.domain.replace(/^www\./, '');
return getIpAddresses().then(function (inet) {
if (!inet.addresses.length) {
return PromiseA.reject(new Error("no ip addresses"));
}
results.inets = inet.addresses.map(function (a) {
a.time = undefined;
return a;
});
results.resolutions = [];
return PromiseA.all([
// for static content
verifyIps(inet.addresses, config.domain).then(function (ips) {
results.resolutions.push({ hostname: config.domain, ips: ips });
})
// for redirects
, verifyIps(inet.addresses, 'www.' + config.domain).then(function (ips) {
results.resolutions.push({ hostname: 'www.' + config.domain, ips: ips });
})
// for api
, verifyIps(inet.addresses, 'api.' + config.domain).then(function (ips) {
results.resolutions.push({ hostname: 'api.' + config.domain, ips: ips });
})
// for protected assets
, verifyIps(inet.addresses, 'assets.' + config.domain).then(function (ips) {
results.resolutions.push({ hostname: 'assets.' + config.domain, ips: ips });
})
// for the cloud management
, verifyIps(inet.addresses, 'cloud.' + config.domain).then(function (ips) {
results.resolutions.push({ hostname: 'cloud.' + config.domain, ips: ips });
})
, verifyIps(inet.addresses, 'api.cloud.' + config.domain).then(function (ips) {
results.resolutions.push({ hostname: 'api.cloud.' + config.domain, ips: ips });
})
]).then(function () {
if (!results.resolutions[0].ips.length) {
results.error = { message: "bare domain could not be resolved to this device" };
}
else if (!results.resolutions[2].ips.length) {
results.error = { message: "api subdomain could not be resolved to this device" };
}
/*
else if (!results.resolutions[1].ips.length) {
results.error = { message: "" }
}
else if (!results.resolutions[3].ips.length) {
results.error = { message: "" }
}
else if (!results.resolutions[4].ips.length || !results.resolutions[4].ips.length) {
results.error = { message: "cloud and api.cloud subdomains should be set up" };
}
*/
});
});
}).then(function () {
if (results.error) {
return;
}
var configname = config.domain + '.json';
var configpath = path.join(__dirname, '..', '..', 'config', configname);
var leAuth = {
agreeTos: true
, email: config.email // TODO check email
, domain: config.domain
, createdAt: Date.now()
};
return dns.resolveMxAsync(config.email.replace(/.*@/, '')).then(function (/*addrs*/) {
// TODO allow private key to be uploaded
return fs.writeFileAsync(configpath, JSON.stringify(leAuth, null, ' '), 'utf8').then(function () {
return models.ComDaplieWalnutConfig.upsert('config', {
letsencrypt: leAuth
, primaryDomain: config.domain
, primaryEmail: config.email
});
});
}, function () {
return PromiseA.reject(new Error("invalid email address (MX record lookup failed)"));
});
}).then(function () {
if (!results.error && results.inets && resolve) {
resolve();
resolve = null;
}
res.send(results);
}, function (err) {
console.error('Error lib/bootstrap.js');
console.error(err.stack || err);
res.send({ error: { message: err.message || err.toString() } });
});
}
var CORS = require('connect-cors');
var cors = CORS({ credentials: true, headers: [
'X-Requested-With'
, 'X-HTTP-Method-Override'
, 'Content-Type'
, 'Accept'
, 'Authorization'
], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
app.use('/', function (req, res, next) {
return isInitialized().then(function (initialized) {
if (!initialized) {
next();
return;
}
resolve(true);
// force page refresh
// TODO goto top of routes?
res.statusCode = 302;
res.setHeader('Location', req.url);
res.end();
});
});
app.use('/api', errorIfNotApi);
// NOTE Allows CORS access to API with ?access_token=
// TODO Access-Control-Max-Age: 600
// TODO How can we help apps handle this? token?
// TODO allow apps to configure trustedDomains, auth, etc
app.use('/api', cors);
app.get('/api/com.daplie.walnut.init', getConfig);
app.post('/api/com.daplie.walnut.init', setConfig);
app.use('/', errorIfApi);
app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'com.daplie.walnut.init')));
return new PromiseA(function (_resolve) {
resolve = _resolve;
});
}
return isInitialized().then(function (initialized) {
if (initialized) {
return true;
}
return initialize();
}, function (err) {
console.error('FATAL ERROR:');
console.error(err.stack || err);
app.use('/', function (req, res) {
res.send({
error: {
message: "Unrecoverable Error Requires manual server update: " + (err.message || err.toString())
}
});
});
});
};

113
lib/insecure-server.js
View File

@ -1,113 +0,0 @@
'use strict';
module.exports.create = function (lex, securePort, insecurePort, info, serverCallback) {
var PromiseA = require('bluebird').Promise;
var appPromise;
//var app;
var http = require('http');
var redirectives;
function useAppInsecurely(req, res) {
if (!appPromise) {
return false;
}
appPromise.then(function (app) {
req._WALNUT_SECURITY_EXCEPTION = true;
app(req, res);
});
return true;
}
function redirectHttps(req, res) {
if (req.headers.host && /^\/.well-known\/acme-challenge/.test(req.url) && useAppInsecurely(req, res)) {
return true;
}
// TODO
// XXX NOTE: info.conf.redirects may or may not be loaded at first
// the object will be modified when the config is loaded
if (!redirectives && info.redirects || info.conf.redirects) {
redirectives = require('./hostname-redirects').compile(info.redirects || info.conf.redirects);
}
if (require('./no-www').scrubTheDub(req, res, redirectives)) {
return true;
}
// Let it do this once they visit the https site
// res.setHeader('Strict-Transport-Security', 'max-age=10886400; includeSubDomains; preload');
var host = req.headers.host || '';
var url = req.url;
// TODO
// allow exceptions for the case of arduino and whatnot that cannot handle https?
// http://evothings.com/is-it-possible-to-secure-micro-controllers-used-within-iot/
// needs ECDSA?
var escapeHtml = require('escape-html');
var newLocation = 'https://'
+ host.replace(/:\d+/, ':' + securePort) + url
;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an insecure resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
// DO NOT HTTP REDIRECT
/*
res.setHeader('Location', newLocation);
res.statusCode = 302;
*/
// BAD NEWS BEARS
//
// When people are experimenting with the API and posting tutorials
// they'll use cURL and they'll forget to prefix with https://
// If we allow that, then many users will be sending private tokens
// and such with POSTs in clear text and, worse, it will work!
// To minimize this, we give browser users a mostly optimal experience,
// but people experimenting with the API get a message letting them know
// that they're doing it wrong and thus forces them to ensure they encrypt.
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.end(metaRedirect);
}
// TODO localhost-only server shutdown mechanism
// that closes all sockets, waits for them to finish,
// and then hands control over completely to respawned server
//
// Redirect HTTP to HTTPS
//
// This simply redirects from the current insecure location to the encrypted location
//
var insecureServer;
insecureServer = http.createServer();
insecureServer.listen(insecurePort, function () {
console.log("\nListening on http://localhost:" + insecureServer.address().port);
console.log("(handling any explicit redirects and redirecting all other traffic to https)\n");
if (serverCallback) {
appPromise = serverCallback(null, insecureServer);
}
});
if (lex) {
var LEX = require('letsencrypt-express');
insecureServer.on('request', LEX.createAcmeResponder(lex, redirectHttps));
} else {
insecureServer.on('request', redirectHttps);
}
return PromiseA.resolve(insecureServer);
};

138
lib/ip-checker.js Normal file
View File

@ -0,0 +1,138 @@
"use strict";
var PromiseA = require('bluebird').Promise;
var ifaces = require('os').networkInterfaces();
var dns = PromiseA.promisifyAll(require('dns'));
var https = require('https');
function getExternalAddresses() {
var iftypes = {};
var ipv4check = 'api.ipify.org';
var ipv6check = 'myexternalip.com';
Object.keys(ifaces).forEach(function (ifname) {
ifaces[ifname].forEach(function (iface) {
// local addresses
if (iface.internal) {
return;
}
// auto address space
if (/^(fe80:|169\.)/.test(iface.address)) {
return;
}
/*
if (/^(fe80:|10\.|192\.168|172\.1[6-9]|172\.2[0-9]|172\.3[0-1])/.test(iface.address)) {
return;
}
*/
iftypes[iface.family] = true;
});
});
console.log(iftypes);
var now = Date.now();
return PromiseA.all([
dns.lookupAsync(ipv4check, { family: 4/*, all: true*/ }).then(function (ans) {
iftypes.IPv4 = { address: ans[0], family: ans[1], time: Date.now() - now };
}).error(function () {
//console.log('no ipv4', Date.now() - now);
iftypes.IPv4 = false;
})
// curl -6 https://myexternalip.com/raw
, dns.lookupAsync(ipv6check, { family: 6/*, all: true*/ }).then(function (ans) {
iftypes.IPv6 = { address: ans[0], family: ans[1], time: Date.now() - now };
}).error(function (err) {
console.error('Error ip-checker.js');
console.error(err.stack || err);
//console.log('no ipv6', Date.now() - now);
iftypes.IPv6 = false;
})
]).then(function () {
var requests = [];
if (iftypes.IPv4) {
requests.push(new PromiseA(function (resolve) {
var req = https.request({
method: 'GET'
, hostname: iftypes.IPv4.address
, port: 443
, headers: {
Host: ipv4check
}
, path: '/'
//, family: 4
// TODO , localAddress: <<external_ipv4>>
}, function (res) {
var result = '';
res.on('error', function (/*err*/) {
resolve(null);
});
res.on('data', function (chunk) {
result += chunk.toString('utf8');
});
res.on('end', function () {
resolve({ address: result, family: 4/*, wan: result === iftypes.IPv4.localAddress*/, time: iftypes.IPv4.time });
});
});
req.on('error', function () {
resolve(null);
});
req.end();
}));
}
if (iftypes.IPv6) {
requests.push(new PromiseA(function (resolve) {
var req = https.request({
method: 'GET'
, hostname: iftypes.IPv6.address
, port: 443
, headers: {
Host: ipv6check
}
, path: '/raw'
//, family: 6
// TODO , localAddress: <<external_ipv6>>
}, function (res) {
var result = '';
res.on('error', function (/*err*/) {
resolve(null);
});
res.on('data', function (chunk) {
result += chunk.toString('utf8').trim();
});
res.on('end', function () {
resolve({ address: result, family: 6/*, wan: result === iftypes.IPv6.localAaddress*/, time: iftypes.IPv4.time });
});
});
req.on('error', function () {
resolve(null);
});
req.end();
}));
}
return PromiseA.all(requests).then(function (ips) {
ips = ips.filter(function (ip) {
return ip;
});
return {
addresses: ips
, time: Date.now() - now
};
});
});
}
exports.getExternalAddresses = getExternalAddresses;

15
lib/local-server.js
View File

@ -2,7 +2,7 @@
// Note the odd use of callbacks (instead of promises) here
// It's to avoid loading bluebird yet (see sni-server.js for explanation)
module.exports.create = function (lex, certPaths, port, info, serverCallback) {
module.exports.create = function (certPaths, port, info, serverCallback) {
function initServer(err, server) {
var app;
var promiseApp;
@ -29,7 +29,7 @@ module.exports.create = function (lex, certPaths, port, info, serverCallback) {
*/
// Get up and listening as absolutely quickly as possible
function onRequest(req, res) {
server.on('request', function (req, res) {
// this is a hot piece of code, so we cache the result
if (app) {
app(req, res);
@ -41,18 +41,11 @@ module.exports.create = function (lex, certPaths, port, info, serverCallback) {
app = _app;
app(req, res);
});
}
if (lex) {
var LEX = require('letsencrypt-express');
server.on('request', LEX.createAcmeResponder(lex, onRequest));
} else {
server.on('request', onRequest);
}
});
}
if (certPaths) {
require('./sni-server').create(lex, certPaths, initServer);
require('./sni-server').create(certPaths, initServer);
} else {
initServer(null, require('http').createServer());
}

297
lib/main.js Normal file
View File

@ -0,0 +1,297 @@
'use strict';
module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
var PromiseA = require('bluebird');
var path = require('path');
var fs = PromiseA.promisifyAll(require('fs'));
// NOTE: each process has its own cache
var localCache = { le: {}, statics: {} };
var express = require('express');
var apiApp;
var setupDomain = xconfx.setupDomain = ('cloud.' + xconfx.primaryDomain);
var setupApp;
function redirectHttpsHelper(req, res) {
var host = req.hostname || req.headers.host || '';
var url = req.url;
// TODO
// allow exceptions for the case of arduino and whatnot that cannot handle https?
// http://evothings.com/is-it-possible-to-secure-micro-controllers-used-within-iot/
// needs ECDSA?
var escapeHtml = require('escape-html');
var newLocation = 'https://'
+ host.replace(/:\d+/, ':' + xconfx.externalPort) + url
;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an insecure resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
// DO NOT HTTP REDIRECT
/*
res.setHeader('Location', newLocation);
res.statusCode = 302;
*/
// BAD NEWS BEARS
//
// When people are experimenting with the API and posting tutorials
// they'll use cURL and they'll forget to prefix with https://
// If we allow that, then many users will be sending private tokens
// and such with POSTs in clear text and, worse, it will work!
// To minimize this, we give browser users a mostly optimal experience,
// but people experimenting with the API get a message letting them know
// that they're doing it wrong and thus forces them to ensure they encrypt.
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.end(metaRedirect);
}
function redirectHttps(req, res) {
if (localCache.le[req.hostname]) {
if (localCache.le[req.hostname].conf) {
redirectHttpsHelper(req, res);
return;
}
else {
// TODO needs IPC to expire cache
redirectSetup(req.hostname, req, res);
return;
/*
if (Date.now() - localCache.le[req.hostname].createdAt < (5 * 60 * 1000)) {
// TODO link to dbconf.primaryDomain
res.send({ error: { message: "Security Error: Encryption for '" + req.hostname + "' has not been configured."
+ " Please use the management interface to set up ACME / Let's Encrypt (or another solution)." } });
return;
}
*/
}
}
return xconfx.walkLe(req.hostname).then(function (leAuth) {
if (!leAuth) {
redirectSetup(req.hostname, req, res);
return;
}
localCache.le[req.hostname] = { conf: leAuth, createdAt: Date.now() };
redirectHttps(req, res);
});
}
function disallowSymLinks(req, res) {
res.end(
"Symbolic Links are not supported on all platforms and are therefore disallowed."
+ " Instead, simply create a file of the same name as the link with a single line of text"
+ " which should be the relative or absolute path to the target directory."
);
}
function disallowNonFiles(req, res) {
res.end(
"Pipes, Blocks, Sockets, FIFOs, and other such nonsense are not permitted."
+ " Instead please create a directory from which to read or create a file "
+ " with a single line of text which should be the target directory to read from."
);
}
function securityError(req, res) {
res.end("Security Error: Link points outside of packages/pages");
}
function notConfigured(req, res, next) {
if (setupDomain !== req.hostname) {
redirectSetup(req.hostname, req, res);
return;
}
if (!setupApp) {
setupApp = express.static(path.join(xconfx.staticpath, 'com.daplie.walnut'));
}
setupApp(req, res, function () {
if ('/' === req.url) {
res.end('Sanity Fail: Configurator not found');
return;
}
next();
});
}
function loadHandler(name) {
return function handler(req, res, next) {
var packagepath = path.join(xconfx.staticpath, name);
return fs.lstatAsync(packagepath).then(function (stat) {
if (stat.isSymbolicLink()) {
return disallowSymLinks;
}
if (stat.isDirectory()) {
return express.static(packagepath);
}
if (!stat.isFile()) {
return disallowNonFiles;
}
return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
// TODO allow cascading
text = text.trim().split(/\n/)[0];
// TODO rerun the above, disallowing link-style (or count or memoize to prevent infinite loop)
// TODO make safe
packagepath = path.resolve(xconfx.staticpath, text);
if (0 !== packagepath.indexOf(xconfx.staticpath)) {
return securityError;
}
return express.static(packagepath);
});
}, function (/*err*/) {
return notConfigured;
}).then(function (handler) {
// keep object reference intact
localCache.statics[name].handler = handler;
handler(req, res, next);
});
};
}
function staticHelper(appId, opts) {
// TODO inter-process cache expirey
// TODO add to xconfx.staticpath
xconfx.staticpath = path.join(__dirname, '..', '..', 'packages', 'pages');
return fs.readdirAsync(xconfx.staticpath).then(function (nodes) {
if (opts && opts.clear) {
localCache.statics = {};
}
// longest to shortest
function shortToLong(a, b) {
return b.length - a.length;
}
nodes.sort(shortToLong);
nodes.forEach(function (name) {
if (!localCache.statics[name]) {
localCache.statics[name] = { handler: loadHandler(name), createdAt: Date.now() };
}
});
// Secure Matching
// apple.com#blah# apple.com#blah#
// apple.com.us# apple.com#foo#
// apple.com# apple.com#foo#
nodes.some(function (name) {
if (0 === (name + '#').indexOf(appId + '#')) {
if (appId !== name) {
localCache.statics[appId] = localCache.statics[name];
}
return true;
}
});
if (!localCache.statics[appId]) {
localCache.statics[appId] = { handler: notConfigured, createdAt: Date.now() };
}
localCache.staticsKeys = Object.keys(localCache.statics).sort(shortToLong);
return localCache.statics[appId];
});
}
function redirectSetup(reason, req, res/*, next*/) {
var url = 'https://cloud.' + xconfx.primaryDomain;
if (443 !== xconfx.externalPort) {
url += ':' + xconfx.externalPort;
}
url += '#referrer=' + reason;
res.statusCode = 302;
res.setHeader('Location', url);
res.end();
}
function serveStatic(req, res, next) {
// If we get this far we can be pretty confident that
// the domain was already set up because it's encrypted
var appId = req.hostname + req.url.replace(/\/+/g, '#').replace(/#$/, '');
var appIdParts = appId.split('#');
var appIdPart;
if (!req.secure) {
// did not come from https
if (/\.(appcache|manifest)\b/.test(req.url)) {
require('./unbrick-appcache').unbrick(req, res);
return;
}
return redirectHttps(req, res);
}
// TODO configuration for allowing www
if (/^www\./.test(req.hostname)) {
// NOTE: acme responder and appcache unbricker must come before scrubTheDub
if (/\.(appcache|manifest)\b/.test(req.url)) {
require('./unbrick-appcache').unbrick(req, res);
return;
}
require('./no-www').scrubTheDub(req, res);
return;
}
/*
if (!redirectives && config.redirects) {
redirectives = require('./hostname-redirects').compile(config.redirects);
}
*/
// TODO assets.example.com/sub/assets/com.example.xyz/
if (/^api\./.test(req.hostname) && /\/api(\/|$)/.test(req.url)) {
// supports api.example.com/sub/app/api/com.example.xyz/
if (!apiApp) {
apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
}
apiApp(req, res, next);
return;
}
while (appIdParts.length) {
// TODO needs IPC to expire cache
appIdPart = appIdParts.join('#');
if (localCache.statics[appIdPart]) {
break;
}
// TODO test via staticsKeys
appIdParts.pop();
}
if (!appIdPart || !localCache.statics[appIdPart]) {
return staticHelper(appId).then(function () {
localCache.statics[appId].handler(req, res, next);
});
}
localCache.statics[appIdPart].handler(req, res, next);
if (Date.now() - localCache.statics[appIdPart].createdAt > (5 * 60 * 1000)) {
staticHelper(appId, { clear: true });
}
}
app.use('/', serveStatic);
return PromiseA.resolve();
};

26
lib/master.js
View File

@ -2,18 +2,17 @@
var cluster = require('cluster');
var PromiseA = require('bluebird');
// TODO
// var rootMasterKey;
function init(conf, state) {
var newConf = {};
if (!conf.ipcKey) {
conf.ipcKey = require('crypto').randomBytes(16).toString('base64');
conf.ipcKey = newConf.ipcKey = require('crypto').randomBytes(16).toString('base64');
}
if (!conf.sqlite3Sock) {
conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
conf.sqlite3Sock = newConf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
}
if (!conf.memstoreSock) {
conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
conf.memstoreSock = newConf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
}
try {
@ -49,15 +48,14 @@ function init(conf, state) {
verbose: null
, sock: conf.sqlite3Sock
, ipcKey: conf.ipcKey
})
})/*.then(function () {
var sqlite3 = require('sqlite3-cluster/client');
return sqliet3.createClientFactory(...);
})*/
]).then(function (args) {
state.memstore = args[0];
state.sqlstore = args[1];
return {
conf: conf
, memstore: args[0]
, sqlstore: args[1]
};
//state.sqlstore = args[1];
return newConf;
});
return promise;
@ -69,10 +67,10 @@ function touch(conf, state) {
}
// TODO if no xyz worker, start on xyz worker (unlock, for example)
return state.initialize.then(function () {
return state.initialize.then(function (newConf) {
// TODO conf.locked = true|false;
conf.initialized = true;
return conf;
return newConf;
});
}

215
lib/package-server-apis.js Normal file
View File

@ -0,0 +1,215 @@
'use strict';
var escapeStringRegexp = require('escape-string-regexp');
//var apiHandlers = {};
function getApi(conf, pkgConf, pkgDeps, packagedApi) {
var PromiseA = pkgDeps.Promise;
var path = require('path');
var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
// TODO needs some version stuff (which would also allow hot-loading of updates)
// TODO version could be tied to sha256sum
return new PromiseA(function (resolve, reject) {
var myApp;
var ursa;
var promise;
// TODO dynamic requires are a no-no
// can we statically generate a require-er? on each install?
// module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
// requirer[pkgpath]()
myApp = pkgDeps.express();
myApp.disable('x-powered-by');
if (pkgDeps.app.get('trust proxy')) {
myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
}
if (!pkgConf.pubkey) {
/*
return ursa.createPrivateKey(pem, password, encoding);
var pem = myKey.toPrivatePem();
return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
});
*/
ursa = require('ursa');
pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
}
try {
packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
packagedApi._apiname = packagedApi._apipkg.name;
if (packagedApi._apipkg.walnut) {
pkgpath += '/' + packagedApi._apipkg.walnut;
}
promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
} catch(e) {
reject(e);
return;
}
promise.then(function () {
// TODO give pub/priv pair for app and all public keys
// packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
packagedApi._api = require('express-lazy')();
packagedApi._api_app = myApp;
//require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
pkgDeps.getOauth3Controllers =
packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
// DEBUG
//
/*
packagedApi._api.use('/', function (req, res, next) {
console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
next();
});
//*/
// TODO fix backwards compat
// /api/com.example.foo (no change)
packagedApi._api.use('/', packagedApi._api_app);
// /api/com.example.foo => /api
packagedApi._api.use('/', function (req, res, next) {
var priorUrl = req.url;
req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
// console.log('api mangle 3:', req.url);
packagedApi._api_app(req, res, function (err) {
req.url = priorUrl;
next(err);
});
});
// /api/com.example.foo => /
packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
// console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
// console.log(packagedApi._api_app.toString());
packagedApi._api_app(req, res, next);
});
resolve(packagedApi._api);
}, reject);
});
}
function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
function handlePromise(p) {
return p.then(function (api) {
packagedApi._api = api;
return api;
});
}
if (!packagedApi._promise_api) {
packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
}
return handlePromise(packagedApi._promise_api);
}
function runApi(opts, router, req, res, next) {
var path = require('path');
var pkgConf = opts.config;
var pkgDeps = opts.deps;
//var Services = opts.Services;
var packagedApi;
var pathname;
// TODO compile packagesMap
// TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name)
router.packagedApis.some(function (_packagedApi) {
// console.log('[DEBUG _packagedApi.id]', _packagedApi.id);
pathname = router.pathname;
if ('/' === pathname) {
pathname = '';
}
// TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
if (!_packagedApi._api_re) {
_packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)');
//console.log('[api re 2]', _packagedApi._api_re);
}
if (_packagedApi._api_re.test(req.url)) {
packagedApi = _packagedApi;
return true;
}
});
if (!packagedApi) {
console.log("[ODD] no api for '" + req.url + "'");
next();
return;
}
// Reaching this point means that there are APIs for this pathname
// it is important to identify this host + pathname (example.com/foo) as the app
Object.defineProperty(req, 'experienceId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '')
});
Object.defineProperty(req, 'escapedExperienceId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: req.experienceId.replace(/\//g, ':')
});
// packageId should mean hash(api.id + host + path) - also called "api"
Object.defineProperty(req, 'packageId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: packagedApi.domain.id
});
Object.defineProperty(req, 'appConfig', {
enumerable: true
, configurable: false
, writable: false
, value: {} // TODO just the app-scoped config
});
Object.defineProperty(req, 'appDeps', {
enumerable: true
, configurable: false
, writable: false
, value: {} // TODO app-scoped deps
// i.e. when we need to use things such as stripe id
// without exposing them to the app
});
//
// TODO user authentication should go right about here
//
//
// TODO freeze objects for passing them into app
//
if (packagedApi._api) {
packagedApi._api(req, res, next);
return;
}
// console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id);
loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) {
api(req, res, next);
}, function (err) {
console.error('[App Promise Error]');
next(err);
});
}
module.exports.runApi = runApi;

87
lib/package-server-static.js Normal file
View File

@ -0,0 +1,87 @@
'use strict';
var staticHandlers = {};
function loadPages(pkgConf, packagedPage, req, res, next) {
var PromiseA = require('bluebird');
var fs = require('fs');
var path = require('path');
var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || ''));
// TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc)
function handlePromise(p) {
p.then(function (app) {
app(req, res, next);
packagedPage._page = app;
}, function (err) {
console.error('[App Promise Error]');
next(err);
});
}
if (staticHandlers[pkgpath]) {
packagedPage._page = staticHandlers[pkgpath];
packagedPage._page(req, res, next);
return;
}
if (!packagedPage._promise_page) {
packagedPage._promise_page = new PromiseA(function (resolve, reject) {
fs.exists(pkgpath, function (exists) {
var staticServer;
if (!exists) {
reject(new Error("package '" + pkgpath + "' is registered but does not exist"));
return;
}
//console.log('[static mount]', pkgpath);
// https://github.com/expressjs/serve-static/issues/54
// https://github.com/pillarjs/send/issues/91
// https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx
staticServer = require('serve-static')(pkgpath, { dotfiles: undefined });
resolve(staticServer);
});
});
}
handlePromise(packagedPage._promise_page);
}
function layerItUp(pkgConf, router, req, res, next) {
var nexti = -1;
// Layers exist so that static apps can use them like a virtual filesystem
// i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html
// or perhaps some dynamic content (like application cache)
function nextify(err) {
var packagedPage;
nexti += 1;
if (err) {
next(err);
return;
}
// shortest to longest
//route = packages.pop();
// longest to shortest
packagedPage = router.packagedPages[nexti];
if (!packagedPage) {
next();
return;
}
if (packagedPage._page) {
packagedPage._page(req, res, nextify);
return;
}
// could attach to req.{ pkgConf, pkgDeps, Services}
loadPages(pkgConf, packagedPage, req, res, next);
}
nextify();
}
module.exports.layerItUp = layerItUp;

296
lib/package-server.js
View File

@ -1,8 +1,8 @@
'use strict';
var escapeStringRegexp = require('escape-string-regexp');
var staticHandlers = {};
//var apiHandlers = {};
var runApi = require('./package-server-apis').runApi;
var layerItUp = require('./package-server-static').layerItUp;
function compileVhosts(vhostsMap) {
var results = {
@ -62,297 +62,6 @@ function compileVhosts(vhostsMap) {
return results;
}
function loadPages(pkgConf, packagedPage, req, res, next) {
var PromiseA = require('bluebird');
var fs = require('fs');
var path = require('path');
var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || ''));
// TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc)
function handlePromise(p) {
p.then(function (app) {
app(req, res, next);
packagedPage._page = app;
}, function (err) {
console.error('[App Promise Error]');
next(err);
});
}
if (staticHandlers[pkgpath]) {
packagedPage._page = staticHandlers[pkgpath];
packagedPage._page(req, res, next);
return;
}
if (!packagedPage._promise_page) {
packagedPage._promise_page = new PromiseA(function (resolve, reject) {
fs.exists(pkgpath, function (exists) {
var staticServer;
if (!exists) {
reject(new Error("package '" + pkgpath + "' is registered but does not exist"));
return;
}
//console.log('[static mount]', pkgpath);
// https://github.com/expressjs/serve-static/issues/54
// https://github.com/pillarjs/send/issues/91
// https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx
staticServer = require('serve-static')(pkgpath, { dotfiles: undefined });
resolve(staticServer);
});
});
}
handlePromise(packagedPage._promise_page);
}
function getApi(conf, pkgConf, pkgDeps, packagedApi) {
var PromiseA = pkgDeps.Promise;
var path = require('path');
var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
// TODO needs some version stuff (which would also allow hot-loading of updates)
// TODO version could be tied to sha256sum
return new PromiseA(function (resolve, reject) {
var myApp;
var ursa;
var promise;
// TODO dynamic requires are a no-no
// can we statically generate a require-er? on each install?
// module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
// requirer[pkgpath]()
myApp = pkgDeps.express();
myApp.disable('x-powered-by');
if (pkgDeps.app.get('trust proxy')) {
myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
}
if (!pkgConf.pubkey) {
/*
return ursa.createPrivateKey(pem, password, encoding);
var pem = myKey.toPrivatePem();
return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
});
*/
ursa = require('ursa');
pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
}
try {
packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
packagedApi._apiname = packagedApi._apipkg.name;
if (packagedApi._apipkg.walnut) {
pkgpath += '/' + packagedApi._apipkg.walnut;
}
promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
} catch(e) {
reject(e);
return;
}
promise.then(function () {
// TODO give pub/priv pair for app and all public keys
// packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
packagedApi._api = require('express-lazy')();
packagedApi._api_app = myApp;
//require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
pkgDeps.getOauth3Controllers =
packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
// DEBUG
//
/*
packagedApi._api.use('/', function (req, res, next) {
console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
next();
});
//*/
// TODO fix backwards compat
// /api/com.example.foo (no change)
packagedApi._api.use('/', packagedApi._api_app);
// /api/com.example.foo => /api
packagedApi._api.use('/', function (req, res, next) {
var priorUrl = req.url;
req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
// console.log('api mangle 3:', req.url);
packagedApi._api_app(req, res, function (err) {
req.url = priorUrl;
next(err);
});
});
// /api/com.example.foo => /
packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
// console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
// console.log(packagedApi._api_app.toString());
packagedApi._api_app(req, res, next);
});
resolve(packagedApi._api);
}, reject);
});
}
function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
function handlePromise(p) {
return p.then(function (api) {
packagedApi._api = api;
return api;
});
}
if (!packagedApi._promise_api) {
packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
}
return handlePromise(packagedApi._promise_api);
}
function layerItUp(pkgConf, router, req, res, next) {
var nexti = -1;
// Layers exist so that static apps can use them like a virtual filesystem
// i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html
// or perhaps some dynamic content (like application cache)
function nextify(err) {
var packagedPage;
nexti += 1;
if (err) {
next(err);
return;
}
// shortest to longest
//route = packages.pop();
// longest to shortest
packagedPage = router.packagedPages[nexti];
if (!packagedPage) {
next();
return;
}
if (packagedPage._page) {
packagedPage._page(req, res, nextify);
return;
}
// could attach to req.{ pkgConf, pkgDeps, Services}
loadPages(pkgConf, packagedPage, req, res, next);
}
nextify();
}
function runApi(opts, router, req, res, next) {
var path = require('path');
var pkgConf = opts.config;
var pkgDeps = opts.deps;
//var Services = opts.Services;
var packagedApi;
var pathname;
// TODO compile packagesMap
// TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name)
router.packagedApis.some(function (_packagedApi) {
// console.log('[DEBUG _packagedApi.id]', _packagedApi.id);
pathname = router.pathname;
if ('/' === pathname) {
pathname = '';
}
// TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc)
if (!_packagedApi._api_re) {
_packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)');
//console.log('[api re 2]', _packagedApi._api_re);
}
if (_packagedApi._api_re.test(req.url)) {
packagedApi = _packagedApi;
return true;
}
});
if (!packagedApi) {
console.log("[ODD] no api for '" + req.url + "'");
next();
return;
}
// Reaching this point means that there are APIs for this pathname
// it is important to identify this host + pathname (example.com/foo) as the app
Object.defineProperty(req, 'experienceId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '')
});
Object.defineProperty(req, 'escapedExperienceId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: req.experienceId.replace(/\//g, ':')
});
// packageId should mean hash(api.id + host + path) - also called "api"
Object.defineProperty(req, 'packageId', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: packagedApi.domain.id
});
Object.defineProperty(req, 'appConfig', {
enumerable: true
, configurable: false
, writable: false
, value: {} // TODO just the app-scoped config
});
Object.defineProperty(req, 'appDeps', {
enumerable: true
, configurable: false
, writable: false
, value: {} // TODO app-scoped deps
// i.e. when we need to use things such as stripe id
// without exposing them to the app
});
//
// TODO user authentication should go right about here
//
//
// TODO freeze objects for passing them into app
//
if (packagedApi._api) {
packagedApi._api(req, res, next);
return;
}
// console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id);
loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) {
api(req, res, next);
}, function (err) {
console.error('[App Promise Error]');
next(err);
});
}
function mapToApp(opts, req, res, next) {
// opts = { config, deps, services }
var vhost;
@ -450,6 +159,5 @@ function mapToApp(opts, req, res, next) {
return runApi(opts, router, req, res, next);
}
module.exports.runApi = runApi;
module.exports.compileVhosts = compileVhosts;
module.exports.mapToApp = mapToApp;

379
lib/worker.js
View File

@ -1,137 +1,50 @@
'use strict';
module.exports.create = function (webserver, conf, state) {
module.exports.create = function (webserver, xconfx, state) {
console.log('DEBUG create worker');
if (!state) {
state = {};
}
var PromiseA = state.Promise || require('bluebird');
var path = require('path');
//var vhostsdir = path.join(__dirname, 'vhosts');
var express = require('express-lazy');
var app = express();
var memstore;
var sqlstores = {};
var models = {};
var systemFactory = require('sqlite3-cluster/client').createClientFactory({
dirname: path.join(__dirname, '..', '..', 'var') // TODO conf
, prefix: 'com.example.'
dirname: xconfx.varpath
, prefix: 'com.daplie.walnut.'
//, dbname: 'config'
, suffix: ''
, ext: '.sqlite3'
, sock: conf.sqlite3Sock
, ipcKey: conf.ipcKey
, sock: xconfx.sqlite3Sock
, ipcKey: xconfx.ipcKey
});
/*
var clientFactory = require('sqlite3-cluster/client').createClientFactory({
algorithm: 'aes'
, bits: 128
, mode: 'cbc'
, dirname: path.join(__dirname, '..', '..', 'var') // TODO conf
, prefix: 'com.example.'
, dirname: xconfx.varpath // TODO conf
, prefix: 'com.daplie.walnut.'
//, dbname: 'cluster'
, suffix: ''
, ext: '.sqlcipher'
, sock: conf.sqlite3Sock
, ipcKey: conf.ipcKey
, sock: xconfx.sqlite3Sock
, ipcKey: xconfx.ipcKey
});
var cstore = require('cluster-store');
var redirectives;
app.disable('x-powered-by');
if (conf.trustProxy) {
console.info('[Trust Proxy]');
app.set('trust proxy', ['loopback']);
//app.set('trust proxy', function (ip) { console.log('[ip]', ip); return true; });
} else {
console.info('[DO NOT trust proxy]');
// TODO make sure the gzip module loads if there isn't a proxy gzip-ing for us
// app.use(compression())
}
/*
function unlockDevice(conf, state) {
return require('./lib/unlock-device').create().then(function (result) {
result.promise.then(function (_rootMasterKey) {
process.send({
type: 'walnut.keys.root'
conf: {
rootMasterKey: _rootMasterkey
}
});
conf.locked = false;
if (state.caddy) {
state.caddy.update(conf);
}
conf.rootMasterKey = _rootMasterKey;
});
return result.app;
});
}
*/
// TODO handle insecure to actual redirect
// blog.coolaj86.com -> coolaj86.com/blog
// hmm... that won't really matter with hsts
// I guess I just needs letsencrypt
function scrubTheDub(req, res, next) {
var host = req.hostname;
if (!host || 'string' !== typeof host) {
next();
return;
}
// TODO test if this is even necessary
host = host.toLowerCase();
// TODO this should be hot loadable / changeable
if (!redirectives && conf.redirects) {
redirectives = require('./hostname-redirects').compile(conf.redirects);
}
if (!/^www\./.test(host) && !redirectives) {
next();
return;
}
// TODO misnomer, handles all exact redirects
if (!require('./no-www').scrubTheDub(req, res, redirectives)) {
next();
return;
}
}
function caddyBugfix(req, res, next) {
// workaround for Caddy
// https://github.com/mholt/caddy/issues/341
if (app.get('trust proxy')) {
if (req.headers['x-forwarded-proto']) {
req.headers['x-forwarded-proto'] = (req.headers['x-forwarded-proto'] || '').split(/,\s+/g)[0] || undefined;
}
if (req.headers['x-forwarded-host']) {
req.headers['x-forwarded-host'] = (req.headers['x-forwarded-host'] || '').split(/,\s+/g)[0] || undefined;
}
}
next();
}
// TODO misnomer, this can handle nowww, yeswww, and exact hostname redirects
app.use('/', scrubTheDub);
app.use('/', caddyBugfix);
var cstore = require('cluster-store');
return PromiseA.all([
// TODO security on memstore
// TODO memstoreFactory.create
cstore.create({
sock: conf.memstoreSock
, connect: conf.memstoreSock
sock: xconfx.memstoreSock
, connect: xconfx.memstoreSock
// TODO implement
, key: conf.ipcKey
, key: xconfx.ipcKey
}).then(function (_memstore) {
memstore = _memstore;
memstore = PromiseA.promisifyAll(_memstore);
return memstore;
})
// TODO mark a device as lost, stolen, missing in DNS records
@ -140,101 +53,127 @@ module.exports.create = function (webserver, conf, state) {
init: true
, dbname: 'config'
})
, clientFactory.create({
init: true
, key: '00000000000000000000000000000000'
// TODO only complain if the values are different
//, algo: 'aes'
, dbname: 'auth'
})
, clientFactory.create({
init: false
, dbname: 'system'
})
]).then(function (args) {
memstore = args[0];
sqlstores.config = args[1];
sqlstores.auth = args[2];
sqlstores.system = args[3];
sqlstores.create = clientFactory.create;
return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) {
models.Config = tables;
return models.Config.Config.get().then(function (vhostsMap) {
// TODO the core needs to be replacable in one shot
// rm -rf /tmp/walnut/; tar xvf -C /tmp/walnut/; mv /srv/walnut /srv/walnut.{{version}}; mv /tmp/walnut /srv/
// this means that any packages must be outside, perhaps /srv/walnut/{boot,core,packages}
var pkgConf = {
pagespath: path.join(__dirname, '..', '..', 'packages', 'pages') + path.sep
, apipath: path.join(__dirname, '..', '..', 'packages', 'apis') + path.sep
, servicespath: path.join(__dirname, '..', '..', 'packages', 'services')
, vhostsMap: vhostsMap
, vhostPatterns: null
, server: webserver
, externalPort: conf.externalPort
, privkey: conf.privkey
, pubkey: conf.pubkey
, redirects: conf.redirects
, apiPrefix: '/api'
, 'org.oauth3.consumer': conf['org.oauth3.consumer']
, 'org.oauth3.provider': conf['org.oauth3.provider']
, keys: conf.keys
};
var pkgDeps = {
memstore: memstore
, sqlstores: sqlstores
, clientSqlFactory: clientFactory
, systemSqlFactory: systemFactory
//, handlePromise: require('./lib/common').promisableRequest;
//, handleRejection: require('./lib/common').rejectableRequest;
//, localPort: conf.localPort
, Promise: PromiseA
, express: express
, app: app
//, oauthmodels: require('oauthcommon/example-oauthmodels').create(conf)
};
var Services = require('./services-loader').create(pkgConf, {
memstore: memstore
, sqlstores: sqlstores
, clientSqlFactory: clientFactory
, systemSqlFactory: systemFactory
, Promise: PromiseA
});
var recase = require('connect-recase')({
// TODO allow explicit and or default flag
explicit: false
, default: 'snake'
, prefixes: ['/api']
// TODO allow exclude
//, exclusions: [config.oauthPrefix]
, exceptions: {}
//, cancelParam: 'camel'
});
var wrap = require('masterquest-sqlite3');
var dir = [
{ tablename: 'com_daplie_walnut_config'
, idname: 'id'
, unique: [ 'id' ]
, indices: [ 'createdAt', 'updatedAt' ]
}
, { tablename: 'com_daplie_walnut_redirects'
, idname: 'id' // blog.example.com:sample.net/blog
, unique: [ 'id' ]
, indices: [ 'createdAt', 'updatedAt' ]
}
];
function handlePackages(req, res, next) {
// TODO move to caddy parser?
if (/(^|\.)proxyable\./.test(req.hostname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.example.proxyable.com => myapp.mydomain.com
req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
}
require('./package-server').mapToApp({
config: pkgConf
, deps: pkgDeps
, services: Services
, conf: conf
}, req, res, next);
function scopeMemstore(expId) {
var scope = expId + '|';
return {
getAsync: function (id) {
return memstore.getAsync(scope + id);
}
, setAsync: function (id, data) {
return memstore.setAsync(scope + id, data);
}
, touchAsync: function (id, data) {
return memstore.touchAsync(scope + id, data);
}
, destroyAsync: function (id) {
return memstore.destroyAsync(scope + id);
}
// TODO recase
// helpers
, allAsync: function () {
return memstore.allASync().then(function (db) {
return Object.keys(db).filter(function (key) {
return 0 === key.indexOf(scope);
}).map(function (key) {
return db[key];
});
});
}
, lengthAsync: function () {
return memstore.allASync().then(function (db) {
return Object.keys(db).filter(function (key) {
return 0 === key.indexOf(scope);
}).length;
});
}
, clearAsync: function () {
return memstore.allASync().then(function (db) {
return Object.keys(db).filter(function (key) {
return 0 === key.indexOf(scope);
}).map(function (key) {
return memstore.destroyAsync(key);
});
}).then(function () {
return null;
});
}
};
}
//
// Generic Template API
//
app
.use('/api', require('body-parser').json({
return wrap.wrap(sqlstores.config, dir).then(function (models) {
return models.ComDaplieWalnutConfig.find(null, { limit: 100 }).then(function (results) {
return models.ComDaplieWalnutConfig.find(null, { limit: 10000 }).then(function (redirects) {
var express = require('express-lazy');
var app = express();
var recase = require('connect-recase')({
// TODO allow explicit and or default flag
explicit: false
, default: 'snake'
, prefixes: ['/api']
// TODO allow exclude
//, exclusions: [config.oauthPrefix]
, exceptions: {}
//, cancelParam: 'camel'
});
var bootstrapApp;
var mainApp;
var apiDeps = {
models: models
// TODO don't let packages use this directly
, Promise: PromiseA
};
var apiFactories = {
memstoreFactory: { create: scopeMemstore }
, systemSqlFactory: systemFactory
};
function log(req, res, next) {
console.log('[worker/log]', req.method, req.headers.host, req.url);
next();
}
function setupMain() {
mainApp = express();
require('./main').create(mainApp, xconfx, apiFactories, apiDeps).then(function () {
// TODO process.send({});
});
}
if (!bootstrapApp) {
bootstrapApp = express();
require('./bootstrap').create(bootstrapApp, xconfx, models).then(function () {
// TODO process.send({});
setupMain();
});
}
process.on('message', function (data) {
if ('com.daplie.walnut.bootstrap' === data.type) {
setupMain();
}
});
app.disable('x-powered-by');
app.use('/', log);
app.use('/api', require('body-parser').json({
strict: true // only objects and arrays
, inflate: true
// limited to due performance issues with JSON.parse and JSON.stringify
@ -244,38 +183,40 @@ module.exports.create = function (webserver, conf, state) {
, reviver: undefined
, type: 'json'
, verify: undefined
}))
// DO NOT allow urlencoded at any point, it is expressly forbidden
//.use(require('body-parser').urlencoded({
// extended: true
//, inflate: true
//, limit: 100 * 1024
//, type: 'urlencoded'
//, verify: undefined
//}))
.use(require('connect-send-error').error())
;
}));
app.use('/api', recase);
app.use('/api', recase);
app.use('/', function (req, res) {
if (!req.secure) {
// did not come from https
if (/\.(appcache|manifest)\b/.test(req.url)) {
require('./unbrick-appcache').unbrick(req, res);
return;
}
}
app.use('/', handlePackages);
app.use('/', function (err, req, res, next) {
console.error('[Error Handler]');
console.error(err.stack);
if (req.xhr) {
res.send({ error: { message: "kinda unknownish error" } });
} else {
res.send('<html><head><title>ERROR</title></head><body>Error</body></html>');
}
if (xconfx.lex && /\.well-known\/acme-challenge\//.test(req.url)) {
var LEX = require('letsencrypt-express');
xconfx.lex.debug = true;
xconfx.acmeResponder = xconfx.acmeResponder || LEX.createAcmeResponder(xconfx.lex/*, next*/);
xconfx.acmeResponder(req, res);
return;
}
// sadly express uses arity checking
// so the fourth parameter must exist
if (false) {
next();
}
// TODO check https://letsencrypt.status.io to see if https certification is not available
if (mainApp) {
mainApp(req, res);
return;
}
else {
bootstrapApp(req, res);
return;
}
});
return app;
});
return app;
});
});
});

13
setup-dev-deps.sh Normal file
View File

@ -0,0 +1,13 @@
#!/bin/bash
pushd node_modules/authentication-microservice/ || git clone git@github.com:coolaj86/node-authentication-microservice node_modules/authentication-microservice
git pull
popd
pushd node_modules/oauthclient-microservice/ || git clone git@github.com:OAuth3/node-oauth3clients.git node_modules/oauthclient-microservice
git pull
popd
pushd node_modules/oauthcommon/ || git clone git@github.com:coolaj86/node-oauthcommon.git node_modules/oauthcommon
git pull
popd