reworking vhost routes

This commit is contained in:
AJ ONeal 2015-11-12 11:14:59 +00:00
parent dd8f8b426f
commit 1bb21f5a07
9 changed files with 501 additions and 32 deletions

View File

@ -3,6 +3,7 @@
var cluster = require('cluster');
var PromiseA = require('bluebird');
var memstore;
var sqlstore;
// TODO
// var rootMasterKey;
@ -26,24 +27,24 @@ function init(conf/*, state*/) {
if (!conf.ipcKey) {
conf.ipcKey = require('crypto').randomBytes(16).toString('base64');
}
if (!conf.sqlite3Sock) {
conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
}
if (!conf.memstoreSock) {
conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock';
}
var memstoreOpts = {
sock: conf.memstoreSock || '/tmp/memstore.sock'
// If left 'null' or 'undefined' this defaults to a similar memstore
// with no special logic for 'cookie' or 'expires'
, store: cluster.isMaster && null //new require('express-session/session/memory')()
// a good default to use for instances where you might want
// to cluster or to run standalone, but with the same API
, serve: cluster.isMaster
, connect: cluster.isWorker
//, standalone: (1 === numCores) // overrides serve and connect
// TODO implement
, key: conf.ipcKey
};
try {
require('fs').unlinkSync(memstoreOpts.sock);
require('fs').unlinkSync(conf.memstoreSock);
} catch(e) {
if ('ENOENT' !== e.code) {
console.error(e.stack);
console.error(JSON.stringify(e));
}
// ignore
}
try {
require('fs').unlinkSync(conf.sqlite3Sock);
} catch(e) {
if ('ENOENT' !== e.code) {
console.error(e.stack);
@ -53,8 +54,35 @@ function init(conf/*, state*/) {
}
var cstore = require('cluster-store');
var memstorePromise = cstore.create(memstoreOpts).then(function (_memstore) {
var sqlite3 = require('sqlite3-cluster/server');
var promise = PromiseA.all([
cstore.create({
sock: conf.memstoreSock
, serve: cluster.isMaster && conf.memstoreSock
, store: cluster.isMaster && null //new require('express-session/session/memory')()
// TODO implement
, key: conf.ipcKey
}).then(function (_memstore) {
memstore = _memstore;
return memstore;
})
, sqlite3.createServer({
verbose: null
, sock: conf.sqlite3Sock
, ipcKey: conf.ipcKey
}).then(function (_sqlstore) {
sqlstore = _sqlstore;
return sqlstore;
})
]).then(function (/*args*/) {
return conf;
/*
{
conf: conf
, memstore: memstore // args[0]
, sqlstore: sqlstore // args[1]
};
*/
});
// TODO check the IP every 5 minutes and update it every hour
@ -62,7 +90,7 @@ function init(conf/*, state*/) {
// we don't want this to load right away (extra procesing time)
setTimeout(updateIps, 1);
return memstorePromise;
return promise;
}
function touch(conf, state) {
@ -88,6 +116,7 @@ function touch(conf, state) {
//var config = require('./device.json');
// require('ssl-root-cas').inject();
// TODO try SNI loopback.example.com as result of api.ipify.com with loopback token
/*
function phoneHome() {

174
lib/schemes-config.js Normal file
View File

@ -0,0 +1,174 @@
'use strict';
function deserialize(results) {
var config = { apis: {}, apps: {}, domains: {} };
results.apis.forEach(function (api) {
config.apis[api.id] = api;
api.domains = [];
api.domainIds = [];
api.domainsMap = {};
});
results.apps.forEach(function (app) {
config.apps[app.id] = app;
app.domains = [];
app.domainIds = [];
app.domainsMap = {};
});
results.domains.forEach(function (domain) {
config.domains[domain.id] = domain;
// as it currently stands each of these will only have one
/*
domain.apis = [];
domain.apiIds = [];
domain.apisMap = {};
domain.apps = [];
domain.appIds = [];
domain.appsMap = {};
*/
domain.api = null;
domain.apiId = null;
domain.app = null;
domain.appId = null;
domain.appsMap = null;
});
results.apisDomains.forEach(function (ad) {
var api = config.apis[ad.apiId];
var domain = config.domains[ad.domainId];
if (api && !api.domainsMap[domain.id]) {
api.domainIds.push(domain.id);
api.domainsMap[domain.id] = domain;
api.domains.push(domain);
}
if (domain) {
if (domain.api) {
console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'");
}
domain.apiId = api.id;
domain.api = api;
}
});
results.appsDomains.forEach(function (ad) {
var app = config.apps[ad.appId];
var domain = config.domains[ad.domainId];
if (app && !app.domainsMap[domain.id]) {
app.domainIds.push(domain.id);
app.domainsMap[domain.id] = domain;
app.domains.push(domain);
}
if (domain) {
if (domain.app) {
console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'");
}
domain.appId = app.id;
domain.app = app;
}
});
return config;
}
module.exports.deserialize = deserialize;
module.exports.create = function (db) {
console.log('[DB -1]');
var wrap = require('dbwrap');
var dir = [
//
// Collections
//
{ tablename: 'apis'
, idname: 'id' // io.lds.auth, com.daplie.radio
, unique: ['id']
// name // LDS Account, Radio
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name']
}
, { tablename: 'apps'
, idname: 'id' // io.lds.auth, com.daplie.radio
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name']
}
, { tablename: 'domains'
, idname: 'id' // api.coolaj86.com#radio
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name', 'token', 'accountId']
}
//
// Joins
//
, { tablename: 'apis_domains'
, idname: 'id' // hash(api_id + domain_id)
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'apiId', 'domainId']
// TODO auto-form relations
, hasMany: ['apis', 'domains']
}
, { tablename: 'apps_domains'
, idname: 'id' // hash(domain_id + app_id)
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'appId', 'domainId']
// TODO auto-form relations
, hasMany: ['apps', 'domains']
}
/*
, { tablename: 'accounts_apis'
, idname: 'id' // hash(account_id + api_id)
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'apiId']
// TODO auto-form relations
, hasMany: ['accounts', 'apis']
}
, { tablename: 'accounts_domains'
, idname: 'id' // hash(account_id + domain_id)
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'domainId']
// TODO auto-form relations
, hasMany: ['accounts', 'domains']
}
, { tablename: 'accounts_apps'
, idname: 'id' // hash(account_id + static_id)
, unique: ['id']
, indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'staticId']
// TODO auto-form relations
, hasMany: ['accounts', 'apps']
}
*/
];
return wrap.wrap(db, dir).then(function (models) {
models.Config = {
get: function () {
var PromiseA = require('bluebird');
return PromiseA.all([
models.Apis.find(null, { limit: 10000 })
, models.Apps.find(null, { limit: 10000 })
, models.Domains.find(null, { limit: 10000 })
, models.ApisDomains.find(null, { limit: 10000 })
, models.AppsDomains.find(null, { limit: 10000 })
]).then(function (args) {
var results = {
apis: args[0]
, apps: args[1]
, domains: args[2]
, apisDomains: args[3]
, appsDomains: args[4]
};
// create fixture with which to test
// console.log(JSON.stringify(results));
var config = deserialize(results);
return config;
});
}
};
return models;
});
};

View File

@ -3,7 +3,7 @@
module.exports.getDomainInfo = function (apppath) {
var parts = apppath.split(/[#%]+/);
var hostname = parts.shift();
var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, '');
var pathname = parts.join('/').replace(/\/+/g, '/').replace(/\/$/g, '').replace(/^\//g, '');
return {
hostname: hostname

View File

@ -1,10 +1,40 @@
'use strict';
module.exports.create = function (webserver, info) {
module.exports.create = function (webserver, info, state) {
if (!state) {
state = {};
}
var PromiseA = state.Promise || require('bluebird');
var path = require('path');
var vhostsdir = path.join(__dirname, 'vhosts');
var app = require('express')();
var apiHandler;
var memstore;
var sqlstores = {};
var models = {};
var systemFactory = require('sqlite3-cluster/client').createClientFactory({
dirname: path.join(__dirname, '..', 'var') // TODO info.conf
, prefix: 'com.daplie.'
//, dbname: 'config'
, suffix: ''
, ext: '.sqlite3'
, sock: info.conf.sqlite3Sock
, ipcKey: info.conf.ipcKey
});
var clientFactory = require('sqlite3-cluster/client').createClientFactory({
algorithm: 'aes'
, bits: 128
, mode: 'cbc'
, dirname: path.join(__dirname, '..', 'var') // TODO info.conf
, prefix: 'com.daplie.'
//, dbname: 'cluster'
, suffix: ''
, ext: '.sqlcipher'
, sock: info.conf.sqlite3Sock
, ipcKey: info.conf.ipcKey
});
var cstore = require('cluster-store');
/*
function unlockDevice(conf, state) {
@ -58,6 +88,11 @@ module.exports.create = function (webserver, info) {
res.end(metaRedirect);
}
// TODO handle insecure to actual redirect
// blog.coolaj86.com -> coolaj86.com/blog
// hmm... that won't really matter with hsts
// I guess I just needs letsencrypt
function scrubTheDub(req, res, next) {
var host = req.hostname;
@ -115,5 +150,55 @@ module.exports.create = function (webserver, info) {
app.use('/', scrubTheDub);
app.use('/', handleApi);
return PromiseA.all([
cstore.create({
sock: info.conf.memstoreSock
, connect: info.conf.memstoreSock
// TODO implement
, key: info.conf.ipcKey
}).then(function (_memstore) {
memstore = _memstore;
return memstore;
})
// TODO mark a device as lost, stolen, missing in DNS records
// (and in turn allow other devices to lock it, turn on location reporting, etc)
, systemFactory.create({
init: true
, dbname: 'config'
})
, clientFactory.create({
init: true
, key: '00000000000000000000000000000000'
// TODO only complain if the values are different
//, algo: 'aes'
, dbname: 'auth'
})
, clientFactory.create({
init: false
, dbname: 'system'
})
]).then(function (args) {
memstore = args[0];
sqlstores.config = args[1];
sqlstores.auth = args[2];
sqlstores.system = args[3];
sqlstores.create = clientFactory.create;
return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) {
models.Config = tables;
models.Config.Config.get().then(function (circ) {
/*
// todo getDomainInfo
var utils = require('./utils');
results.domains.forEach(function (domain) {
utils.getDomainInfo(domain.id);
});
*/
console.log(circ);
return app;
});
});
});
};

View File

@ -12,7 +12,7 @@ console.log('\n\n\n[MASTER] Welcome to WALNUT!');
var cluster = require('cluster');
var path = require('path');
var minWorkers = 2;
var numCores = Math.max(minWorkers, require('os').cpus().length);
var numCores = 1; // Math.max(minWorkers, require('os').cpus().length);
var workers = [];
var caddypath = '/usr/local/bin/caddy';
var useCaddy = require('fs').existsSync(caddypath);
@ -75,6 +75,8 @@ cluster.on('online', function (worker) {
require('./lib/master').touch(conf, state).then(function () {
info.type = 'com.daplie.walnut.webserver.onrequest';
info.conf.ipcKey = conf.ipcKey;
info.conf.memstoreSock = conf.memstoreSock;
info.conf.sqlite3Sock = conf.sqlite3Sock;
worker.send(info);
});
}
@ -93,7 +95,8 @@ cluster.on('exit', function (worker, code, signal) {
return w;
});
fork();
console.log('WARNING: worker spawning turned off for debugging ');
//fork();
});
fork();

View File

@ -88,7 +88,7 @@
"ms": "^0.7.0",
"negotiator": "^0.5.1",
"node-pre-gyp": "^0.6.4",
"node-uuid": "1.x",
"node-uuid": "^1.4.4",
"nodemailer": "^1.4.0",
"nodemailer-mailgun-transport": "1.x",
"oauth": "0.9.x",

160
tests/schemes-config.js Normal file
View File

@ -0,0 +1,160 @@
'use strict';
// var results = {"apis":[{"id":"oauth3-api","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"apps":[{"id":"oauth3-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"hellabit-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsio-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsconnect-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"domains":[{"id":"oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com#connect","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null}],"apisDomains":[{"id":"oauth3-api_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"apiId":"oauth3-api","domainId":"oauth3.org","json":null}],"appsDomains":[{"id":"oauth3-app_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"oauth3-app","domainId":"oauth3.org","json":null},{"id":"hellabit-app_hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"hellabit-app","domainId":"hellabit.com","json":null},{"id":"ldsio-app_lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsio-app","domainId":"lds.io","json":null},{"id":"ldsconnect-app_ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsconnect-app","domainId":"ldsconnect.org","json":null}]};
var results = {
"apis":[
{"id":"oauth3-api"}
]
, "apps":[
{"id":"oauth3-app"}
, {"id":"hellabit-app"}
, {"id":"ldsio-app"}
, {"id":"ldsconnect-app"}
]
, "domains":[
{"id":"oauth3.org"}
, {"id":"lds.io"}
, {"id":"ldsconnect.org"}
, {"id":"hellabit.com#####"}
, {"id":"hellabit.com"}
, {"id":"hellabit.com###"}
, {"id":"hellabit.com#connect###"}
, {"id":"hellabit.com#connect"}
, {"id":"hellabit.com#connect#too"}
]
, "apisDomains":[
{"id":"oauth3-api_oauth3.org","apiId":"oauth3-api","domainId":"oauth3.org"}
]
,"appsDomains":[
{"id":"oauth3-app_oauth3.org","appId":"oauth3-app","domainId":"oauth3.org"}
, {"id":"hellabit-app_hellabit.com","appId":"hellabit-app","domainId":"hellabit.com"}
, {"id":"hellabit-app_hellabit.com###","appId":"hellabit-app","domainId":"hellabit.com#connect###"}
, {"id":"ldsio-app_lds.io","appId":"ldsio-app","domainId":"lds.io"}
, {"id":"ldsconnect-app_ldsconnect.org","appId":"ldsconnect-app","domainId":"ldsconnect.org"}
]
};
var deserialize = require('../lib/schemes-config').deserialize;
var getDomainInfo = require('../lib/utils').getDomainInfo;
var config = deserialize(results);
var req = { host: 'hellabit.com', url: '/connect' };
var vhosts = [];
var vhostsMap = {};
function sortApps(a, b) {
// hlen isn't important in this current use of the sorter,
// but is important for an alternate version
var hlen = b.hostname.length - a.hostname.length;
var plen = b.pathname.length - a.pathname.length;
// A directory could be named example.com, example.com# example.com##
// to indicate order of preference (for API addons, for example)
var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
if (!hlen) {
if (!plen) {
return dlen;
}
return plen;
}
return hlen;
}
Object.keys(config.domains).forEach(function (domainname) {
var domain = config.domains[domainname];
var info = getDomainInfo(domainname);
domain.hostname = info.hostname;
domain.pathname = '/' + (info.pathname || '');
domain.dirname = info.dirname;
vhosts.push(domain);
});
vhosts.sort(sortApps);
vhosts.forEach(function (domain) {
console.log(domain.hostname, domain.pathname, domain.dirname);
if (!vhostsMap[domain.hostname]) {
vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] };
}
if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) {
vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] };
vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]);
}
vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain);
});
if (!vhostsMap[req.host]) {
console.log("there's no app for this hostname");
return;
}
//console.log("load an app", vhosts[req.host]);
//console.log(vhosts[req.host]);
function getApp(route) {
var PromiseA = require('bluebird');
return new PromiseA(function (resolve, reject) {
console.log(route);
// route.hostname
});
}
function api(req, res, next) {
var apps;
vhostsMap[req.host].pathnames.some(function (route) {
// /connect /
if (req.url.match(route.pathname) && route.pathname.match(req.url)) {
apps = route.apps;
return true;
}
});
//console.log(apps);
function nextify(err) {
var route;
if (err) {
next(err);
return;
}
// shortest to longest
//route = apps.pop();
// longest to shortest
route = apps.shift();
if (!route) {
next();
return;
}
if (route.route) {
route.route(req, res, nextify);
return;
}
getApp(route).then(function (route) {
route.route = route;
try {
route.route(req, res, nextify);
} catch(e) {
console.error('[App Load Error]');
console.error(e.stack);
nextify(new Error("couldn't load app"));
}
});
}
nextify();
}
api(req);

View File

@ -2,6 +2,23 @@
var cluster = require('cluster');
var crypto;
var stacks = {};
Math.random = function () {
var err = new Error("Math.random() was used");
if (!stacks[err.stack.toString()]) {
stacks[err.stack.toString()] = true;
console.warn(err.stack);
}
if (!crypto) {
crypto = require('crypto');
}
return parseFloat(('0.' + (parseInt(crypto.randomBytes(8).toString('hex'), 16))).replace(/(^0)|(0$)/g, ''));
};
if (cluster.isMaster) {
require('./master');
} else {

View File

@ -5,8 +5,8 @@ var id = cluster.worker.id.toString();
function waitForInit(message) {
if ('com.daplie.walnut.init' !== message.type) {
console.log('[Worker] 0 got unexpected message:');
console.log(message);
console.warn('[Worker] 0 got unexpected message:');
console.warn(message);
return;
}
@ -15,7 +15,7 @@ function waitForInit(message) {
require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) {
if (err) {
console.log('[ERROR] worker.js');
console.error('[ERROR] worker.js');
console.error(err.stack);
throw err;
}
@ -26,8 +26,8 @@ function waitForInit(message) {
return new PromiseA(function (resolve) {
function initWebServer(srvmsg) {
if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) {
console.log('[Worker] 1 got unexpected message:');
console.log(srvmsg);
console.warn('[Worker] 1 got unexpected message:');
console.warn(srvmsg);
return;
}
@ -63,9 +63,10 @@ process.on('beforeExit', function (msg) {
process.on('unhandledRejection', function (err) {
// this should always throw
// (it means somewhere we're not using bluebird by accident)
console.error('[unhandledRejection]');
console.error('[caught] [unhandledRejection]');
console.error(Object.keys(err));
console.error(err);
console.error(err.stack);
throw err;
});
process.on('rejectionHandled', function (msg) {
console.error('[rejectionHandled]');