diff --git a/lib/master.js b/lib/master.js index 9e7d1a7..91b28f0 100644 --- a/lib/master.js +++ b/lib/master.js @@ -3,6 +3,7 @@ var cluster = require('cluster'); var PromiseA = require('bluebird'); var memstore; +var sqlstore; // TODO // var rootMasterKey; @@ -26,24 +27,24 @@ function init(conf/*, state*/) { if (!conf.ipcKey) { conf.ipcKey = require('crypto').randomBytes(16).toString('base64'); } + if (!conf.sqlite3Sock) { + conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; + } + if (!conf.memstoreSock) { + conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; + } - var memstoreOpts = { - sock: conf.memstoreSock || '/tmp/memstore.sock' - - // If left 'null' or 'undefined' this defaults to a similar memstore - // with no special logic for 'cookie' or 'expires' - , store: cluster.isMaster && null //new require('express-session/session/memory')() - - // a good default to use for instances where you might want - // to cluster or to run standalone, but with the same API - , serve: cluster.isMaster - , connect: cluster.isWorker - //, standalone: (1 === numCores) // overrides serve and connect - // TODO implement - , key: conf.ipcKey - }; try { - require('fs').unlinkSync(memstoreOpts.sock); + require('fs').unlinkSync(conf.memstoreSock); + } catch(e) { + if ('ENOENT' !== e.code) { + console.error(e.stack); + console.error(JSON.stringify(e)); + } + // ignore + } + try { + require('fs').unlinkSync(conf.sqlite3Sock); } catch(e) { if ('ENOENT' !== e.code) { console.error(e.stack); @@ -53,8 +54,35 @@ function init(conf/*, state*/) { } var cstore = require('cluster-store'); - var memstorePromise = cstore.create(memstoreOpts).then(function (_memstore) { - memstore = _memstore; + var sqlite3 = require('sqlite3-cluster/server'); + var promise = PromiseA.all([ + cstore.create({ + sock: conf.memstoreSock + , serve: cluster.isMaster && conf.memstoreSock + , store: cluster.isMaster && null //new require('express-session/session/memory')() + // TODO implement + , key: conf.ipcKey + }).then(function (_memstore) { + memstore = _memstore; + return memstore; + }) + , sqlite3.createServer({ + verbose: null + , sock: conf.sqlite3Sock + , ipcKey: conf.ipcKey + }).then(function (_sqlstore) { + sqlstore = _sqlstore; + return sqlstore; + }) + ]).then(function (/*args*/) { + return conf; + /* + { + conf: conf + , memstore: memstore // args[0] + , sqlstore: sqlstore // args[1] + }; + */ }); // TODO check the IP every 5 minutes and update it every hour @@ -62,7 +90,7 @@ function init(conf/*, state*/) { // we don't want this to load right away (extra procesing time) setTimeout(updateIps, 1); - return memstorePromise; + return promise; } function touch(conf, state) { @@ -88,6 +116,7 @@ function touch(conf, state) { //var config = require('./device.json'); // require('ssl-root-cas').inject(); + // TODO try SNI loopback.example.com as result of api.ipify.com with loopback token /* function phoneHome() { diff --git a/lib/schemes-config.js b/lib/schemes-config.js new file mode 100644 index 0000000..d0e25b4 --- /dev/null +++ b/lib/schemes-config.js @@ -0,0 +1,174 @@ +'use strict'; + +function deserialize(results) { + var config = { apis: {}, apps: {}, domains: {} }; + results.apis.forEach(function (api) { + config.apis[api.id] = api; + api.domains = []; + api.domainIds = []; + api.domainsMap = {}; + }); + results.apps.forEach(function (app) { + config.apps[app.id] = app; + app.domains = []; + app.domainIds = []; + app.domainsMap = {}; + }); + + results.domains.forEach(function (domain) { + config.domains[domain.id] = domain; + // as it currently stands each of these will only have one + /* + domain.apis = []; + domain.apiIds = []; + domain.apisMap = {}; + domain.apps = []; + domain.appIds = []; + domain.appsMap = {}; + */ + domain.api = null; + domain.apiId = null; + domain.app = null; + domain.appId = null; + domain.appsMap = null; + }); + + results.apisDomains.forEach(function (ad) { + var api = config.apis[ad.apiId]; + var domain = config.domains[ad.domainId]; + if (api && !api.domainsMap[domain.id]) { + api.domainIds.push(domain.id); + api.domainsMap[domain.id] = domain; + api.domains.push(domain); + } + if (domain) { + if (domain.api) { + console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'"); + } + domain.apiId = api.id; + domain.api = api; + } + }); + + results.appsDomains.forEach(function (ad) { + var app = config.apps[ad.appId]; + var domain = config.domains[ad.domainId]; + if (app && !app.domainsMap[domain.id]) { + app.domainIds.push(domain.id); + app.domainsMap[domain.id] = domain; + app.domains.push(domain); + } + if (domain) { + if (domain.app) { + console.error("[SANITY FAIL] single domain has multiple frontends in db: '" + domain.id + "'"); + } + domain.appId = app.id; + domain.app = app; + } + }); + + return config; +} + +module.exports.deserialize = deserialize; +module.exports.create = function (db) { + console.log('[DB -1]'); + var wrap = require('dbwrap'); + + var dir = [ + // + // Collections + // + { tablename: 'apis' + , idname: 'id' // io.lds.auth, com.daplie.radio + , unique: ['id'] + // name // LDS Account, Radio + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name'] + } + , { tablename: 'apps' + , idname: 'id' // io.lds.auth, com.daplie.radio + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name'] + } + , { tablename: 'domains' + , idname: 'id' // api.coolaj86.com#radio + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'revokedAt', 'name', 'token', 'accountId'] + } + + // + // Joins + // + , { tablename: 'apis_domains' + , idname: 'id' // hash(api_id + domain_id) + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'apiId', 'domainId'] + // TODO auto-form relations + , hasMany: ['apis', 'domains'] + } + , { tablename: 'apps_domains' + , idname: 'id' // hash(domain_id + app_id) + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'appId', 'domainId'] + // TODO auto-form relations + , hasMany: ['apps', 'domains'] + } + +/* + , { tablename: 'accounts_apis' + , idname: 'id' // hash(account_id + api_id) + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'apiId'] + // TODO auto-form relations + , hasMany: ['accounts', 'apis'] + } + , { tablename: 'accounts_domains' + , idname: 'id' // hash(account_id + domain_id) + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'domainId'] + // TODO auto-form relations + , hasMany: ['accounts', 'domains'] + } + , { tablename: 'accounts_apps' + , idname: 'id' // hash(account_id + static_id) + , unique: ['id'] + , indices: ['createdAt', 'updatedAt', 'deletedAt', 'accountId', 'staticId'] + // TODO auto-form relations + , hasMany: ['accounts', 'apps'] + } +*/ + ]; + + return wrap.wrap(db, dir).then(function (models) { + models.Config = { + get: function () { + var PromiseA = require('bluebird'); + + return PromiseA.all([ + models.Apis.find(null, { limit: 10000 }) + , models.Apps.find(null, { limit: 10000 }) + , models.Domains.find(null, { limit: 10000 }) + , models.ApisDomains.find(null, { limit: 10000 }) + , models.AppsDomains.find(null, { limit: 10000 }) + ]).then(function (args) { + var results = { + apis: args[0] + , apps: args[1] + , domains: args[2] + , apisDomains: args[3] + , appsDomains: args[4] + }; + + // create fixture with which to test + // console.log(JSON.stringify(results)); + + var config = deserialize(results); + + return config; + }); + } + }; + + return models; + }); +}; diff --git a/lib/utils.js b/lib/utils.js index 33a844a..479a86f 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -3,7 +3,7 @@ module.exports.getDomainInfo = function (apppath) { var parts = apppath.split(/[#%]+/); var hostname = parts.shift(); - var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, ''); + var pathname = parts.join('/').replace(/\/+/g, '/').replace(/\/$/g, '').replace(/^\//g, ''); return { hostname: hostname diff --git a/lib/worker.js b/lib/worker.js index 0f01d60..cb9c21a 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -1,10 +1,40 @@ 'use strict'; -module.exports.create = function (webserver, info) { +module.exports.create = function (webserver, info, state) { + if (!state) { + state = {}; + } + + var PromiseA = state.Promise || require('bluebird'); var path = require('path'); var vhostsdir = path.join(__dirname, 'vhosts'); var app = require('express')(); var apiHandler; + var memstore; + var sqlstores = {}; + var models = {}; + var systemFactory = require('sqlite3-cluster/client').createClientFactory({ + dirname: path.join(__dirname, '..', 'var') // TODO info.conf + , prefix: 'com.daplie.' + //, dbname: 'config' + , suffix: '' + , ext: '.sqlite3' + , sock: info.conf.sqlite3Sock + , ipcKey: info.conf.ipcKey + }); + var clientFactory = require('sqlite3-cluster/client').createClientFactory({ + algorithm: 'aes' + , bits: 128 + , mode: 'cbc' + , dirname: path.join(__dirname, '..', 'var') // TODO info.conf + , prefix: 'com.daplie.' + //, dbname: 'cluster' + , suffix: '' + , ext: '.sqlcipher' + , sock: info.conf.sqlite3Sock + , ipcKey: info.conf.ipcKey + }); + var cstore = require('cluster-store'); /* function unlockDevice(conf, state) { @@ -58,6 +88,11 @@ module.exports.create = function (webserver, info) { res.end(metaRedirect); } + // TODO handle insecure to actual redirect + // blog.coolaj86.com -> coolaj86.com/blog + // hmm... that won't really matter with hsts + // I guess I just needs letsencrypt + function scrubTheDub(req, res, next) { var host = req.hostname; @@ -115,5 +150,55 @@ module.exports.create = function (webserver, info) { app.use('/', scrubTheDub); app.use('/', handleApi); - return app; + return PromiseA.all([ + cstore.create({ + sock: info.conf.memstoreSock + , connect: info.conf.memstoreSock + // TODO implement + , key: info.conf.ipcKey + }).then(function (_memstore) { + memstore = _memstore; + return memstore; + }) + // TODO mark a device as lost, stolen, missing in DNS records + // (and in turn allow other devices to lock it, turn on location reporting, etc) + , systemFactory.create({ + init: true + , dbname: 'config' + }) + , clientFactory.create({ + init: true + , key: '00000000000000000000000000000000' + // TODO only complain if the values are different + //, algo: 'aes' + , dbname: 'auth' + }) + , clientFactory.create({ + init: false + , dbname: 'system' + }) + ]).then(function (args) { + memstore = args[0]; + sqlstores.config = args[1]; + sqlstores.auth = args[2]; + sqlstores.system = args[3]; + sqlstores.create = clientFactory.create; + + return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) { + models.Config = tables; + models.Config.Config.get().then(function (circ) { + + /* + // todo getDomainInfo + var utils = require('./utils'); + results.domains.forEach(function (domain) { + utils.getDomainInfo(domain.id); + }); + */ + console.log(circ); + + return app; + }); + }); + }); }; diff --git a/master.js b/master.js index 7bc7296..0664fda 100644 --- a/master.js +++ b/master.js @@ -12,7 +12,7 @@ console.log('\n\n\n[MASTER] Welcome to WALNUT!'); var cluster = require('cluster'); var path = require('path'); var minWorkers = 2; -var numCores = Math.max(minWorkers, require('os').cpus().length); +var numCores = 1; // Math.max(minWorkers, require('os').cpus().length); var workers = []; var caddypath = '/usr/local/bin/caddy'; var useCaddy = require('fs').existsSync(caddypath); @@ -75,6 +75,8 @@ cluster.on('online', function (worker) { require('./lib/master').touch(conf, state).then(function () { info.type = 'com.daplie.walnut.webserver.onrequest'; info.conf.ipcKey = conf.ipcKey; + info.conf.memstoreSock = conf.memstoreSock; + info.conf.sqlite3Sock = conf.sqlite3Sock; worker.send(info); }); } @@ -93,7 +95,8 @@ cluster.on('exit', function (worker, code, signal) { return w; }); - fork(); + console.log('WARNING: worker spawning turned off for debugging '); + //fork(); }); fork(); diff --git a/package.json b/package.json index b57cfe3..67673a2 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "ms": "^0.7.0", "negotiator": "^0.5.1", "node-pre-gyp": "^0.6.4", - "node-uuid": "1.x", + "node-uuid": "^1.4.4", "nodemailer": "^1.4.0", "nodemailer-mailgun-transport": "1.x", "oauth": "0.9.x", diff --git a/tests/schemes-config.js b/tests/schemes-config.js new file mode 100644 index 0000000..aa023a6 --- /dev/null +++ b/tests/schemes-config.js @@ -0,0 +1,160 @@ +'use strict'; + +// var results = {"apis":[{"id":"oauth3-api","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"apps":[{"id":"oauth3-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"hellabit-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsio-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsconnect-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"domains":[{"id":"oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com#connect","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null}],"apisDomains":[{"id":"oauth3-api_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"apiId":"oauth3-api","domainId":"oauth3.org","json":null}],"appsDomains":[{"id":"oauth3-app_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"oauth3-app","domainId":"oauth3.org","json":null},{"id":"hellabit-app_hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"hellabit-app","domainId":"hellabit.com","json":null},{"id":"ldsio-app_lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsio-app","domainId":"lds.io","json":null},{"id":"ldsconnect-app_ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsconnect-app","domainId":"ldsconnect.org","json":null}]}; +var results = { + "apis":[ + {"id":"oauth3-api"} + ] +, "apps":[ + {"id":"oauth3-app"} + , {"id":"hellabit-app"} + , {"id":"ldsio-app"} + , {"id":"ldsconnect-app"} + ] +, "domains":[ + {"id":"oauth3.org"} + , {"id":"lds.io"} + , {"id":"ldsconnect.org"} + , {"id":"hellabit.com#####"} + , {"id":"hellabit.com"} + , {"id":"hellabit.com###"} + , {"id":"hellabit.com#connect###"} + , {"id":"hellabit.com#connect"} + , {"id":"hellabit.com#connect#too"} + ] +, "apisDomains":[ + {"id":"oauth3-api_oauth3.org","apiId":"oauth3-api","domainId":"oauth3.org"} + ] +,"appsDomains":[ + {"id":"oauth3-app_oauth3.org","appId":"oauth3-app","domainId":"oauth3.org"} + , {"id":"hellabit-app_hellabit.com","appId":"hellabit-app","domainId":"hellabit.com"} + , {"id":"hellabit-app_hellabit.com###","appId":"hellabit-app","domainId":"hellabit.com#connect###"} + , {"id":"ldsio-app_lds.io","appId":"ldsio-app","domainId":"lds.io"} + , {"id":"ldsconnect-app_ldsconnect.org","appId":"ldsconnect-app","domainId":"ldsconnect.org"} + ] +}; + +var deserialize = require('../lib/schemes-config').deserialize; +var getDomainInfo = require('../lib/utils').getDomainInfo; +var config = deserialize(results); +var req = { host: 'hellabit.com', url: '/connect' }; +var vhosts = []; +var vhostsMap = {}; + +function sortApps(a, b) { + // hlen isn't important in this current use of the sorter, + // but is important for an alternate version + var hlen = b.hostname.length - a.hostname.length; + var plen = b.pathname.length - a.pathname.length; + + // A directory could be named example.com, example.com# example.com## + // to indicate order of preference (for API addons, for example) + var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length); + + if (!hlen) { + if (!plen) { + return dlen; + } + return plen; + } + return hlen; +} + +Object.keys(config.domains).forEach(function (domainname) { + var domain = config.domains[domainname]; + var info = getDomainInfo(domainname); + + domain.hostname = info.hostname; + domain.pathname = '/' + (info.pathname || ''); + domain.dirname = info.dirname; + + vhosts.push(domain); +}); + +vhosts.sort(sortApps); + +vhosts.forEach(function (domain) { + console.log(domain.hostname, domain.pathname, domain.dirname); + + if (!vhostsMap[domain.hostname]) { + vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] }; + } + + if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) { + vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] }; + vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]); + } + + vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain); +}); + +if (!vhostsMap[req.host]) { + console.log("there's no app for this hostname"); + return; +} + +//console.log("load an app", vhosts[req.host]); + +//console.log(vhosts[req.host]); + + +function getApp(route) { + var PromiseA = require('bluebird'); + + return new PromiseA(function (resolve, reject) { + console.log(route); + // route.hostname + }); +} + +function api(req, res, next) { + var apps; + + vhostsMap[req.host].pathnames.some(function (route) { + // /connect / + if (req.url.match(route.pathname) && route.pathname.match(req.url)) { + apps = route.apps; + return true; + } + }); + + //console.log(apps); + + function nextify(err) { + var route; + + if (err) { + next(err); + return; + } + + // shortest to longest + //route = apps.pop(); + // longest to shortest + route = apps.shift(); + if (!route) { + next(); + return; + } + + if (route.route) { + route.route(req, res, nextify); + return; + } + + getApp(route).then(function (route) { + route.route = route; + try { + route.route(req, res, nextify); + } catch(e) { + console.error('[App Load Error]'); + console.error(e.stack); + nextify(new Error("couldn't load app")); + } + }); + } + + nextify(); +} + +api(req); diff --git a/walnut.js b/walnut.js index cf10a5d..fafacff 100644 --- a/walnut.js +++ b/walnut.js @@ -2,6 +2,23 @@ var cluster = require('cluster'); +var crypto; +var stacks = {}; +Math.random = function () { + var err = new Error("Math.random() was used"); + + if (!stacks[err.stack.toString()]) { + stacks[err.stack.toString()] = true; + console.warn(err.stack); + } + + if (!crypto) { + crypto = require('crypto'); + } + + return parseFloat(('0.' + (parseInt(crypto.randomBytes(8).toString('hex'), 16))).replace(/(^0)|(0$)/g, '')); +}; + if (cluster.isMaster) { require('./master'); } else { diff --git a/worker.js b/worker.js index 5cf8355..888ad01 100644 --- a/worker.js +++ b/worker.js @@ -5,8 +5,8 @@ var id = cluster.worker.id.toString(); function waitForInit(message) { if ('com.daplie.walnut.init' !== message.type) { - console.log('[Worker] 0 got unexpected message:'); - console.log(message); + console.warn('[Worker] 0 got unexpected message:'); + console.warn(message); return; } @@ -15,7 +15,7 @@ function waitForInit(message) { require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) { if (err) { - console.log('[ERROR] worker.js'); + console.error('[ERROR] worker.js'); console.error(err.stack); throw err; } @@ -26,8 +26,8 @@ function waitForInit(message) { return new PromiseA(function (resolve) { function initWebServer(srvmsg) { if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) { - console.log('[Worker] 1 got unexpected message:'); - console.log(srvmsg); + console.warn('[Worker] 1 got unexpected message:'); + console.warn(srvmsg); return; } @@ -63,9 +63,10 @@ process.on('beforeExit', function (msg) { process.on('unhandledRejection', function (err) { // this should always throw // (it means somewhere we're not using bluebird by accident) - console.error('[unhandledRejection]'); + console.error('[caught] [unhandledRejection]'); + console.error(Object.keys(err)); + console.error(err); console.error(err.stack); - throw err; }); process.on('rejectionHandled', function (msg) { console.error('[rejectionHandled]');