From 65645a760208c1daac9126eab583cda513fc7e38 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sat, 14 Nov 2015 04:25:12 +0000 Subject: [PATCH] load apps from DB --- .gitignore | 2 + lib/api-server.js | 133 ++++++++++++++ lib/insecure-server.js | 6 +- lib/local-server.js | 11 +- lib/no-www.js | 27 +++ lib/schemes-config.js | 61 ++++++- lib/spawn-caddy.js | 6 +- lib/unbrick-appcache.js | 10 ++ lib/worker.js | 165 ++++++++++-------- {backends => packages/apis}/.gitkeep | 0 packages/apps/.gitkeep | 0 .../com.daplie.cloud.lockscreen}/index.html | 0 tests/schemes-config.js | 137 ++------------- worker.js | 4 +- 14 files changed, 360 insertions(+), 202 deletions(-) create mode 100644 lib/api-server.js create mode 100644 lib/no-www.js create mode 100644 lib/unbrick-appcache.js rename {backends => packages/apis}/.gitkeep (100%) create mode 100644 packages/apps/.gitkeep rename {init.public => packages/apps/com.daplie.cloud.lockscreen}/index.html (100%) diff --git a/.gitignore b/.gitignore index 5087394..a4181dd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ dyndns-token.js vhosts certs .*.sw* +packages +var # Logs logs diff --git a/lib/api-server.js b/lib/api-server.js new file mode 100644 index 0000000..3dfca8a --- /dev/null +++ b/lib/api-server.js @@ -0,0 +1,133 @@ +'use strict'; + +// TODO handle static app urls? +// NOTE rejecting non-api urls should happen before this +module.exports.create = function (conf, deps, app) { + var escapeStringRegexp = require('escape-string-regexp'); + var vhostsMap = conf.vhostsMap; + if (!app) { + app = deps.app; + } + + function getApi(route) { + // TODO don't modify route, modify some other variable instead + + var PromiseA = require('bluebird'); + var path = require('path'); + console.log(route); + // TODO needs some version stuff (which would also allow hot-loading of updates) + // TODO version could be tied to sha256sum + var pkgpath = path.join(conf.apipath, (route.api.package || route.api.id), (route.api.version || '')); + + console.log('pkgpath'); + console.log(pkgpath); + + return new PromiseA(function (resolve, reject) { + try { + route.route = require(pkgpath).create(conf, deps.app, app); + } catch(e) { + reject(e); + return; + } + + resolve(route.route); + }); + } + + function api(req, res, next) { + var apps; + + console.log('hostname', req.hostname); + console.log('headers', req.headers); + + if (!vhostsMap[req.hostname]) { + // TODO keep track of match-only vhosts, such as '*.example.com', + // separate from exact matches + next(new Error("this domain is not registered")); + return; + } + + vhostsMap[req.hostname].pathnames.some(function (route) { + var pathname = route.pathname; + if ('/' === pathname) { + pathname = '/api'; + } + if (-1 === pathname.indexOf('/api')) { + pathname = '/api' + pathname; + } + + if (!route.re) { + route.re = new RegExp(escapeStringRegexp(pathname) + '(#|\\/|\\?|$)'); + } + // re.test("/api") + // re.test("/api?") + // re.test("/api/") + // re.test("/api/foo") + // re.test("/apifoo") // false + if (route.re.test(req.url)) { + apps = route.apps; + return true; + } + }); + + if (!apps) { + console.log('No apps to try for this hostname'); + console.log(vhostsMap[req.hostname]); + next(); + return; + } + + //console.log(apps); + + function nextify(err) { + var route; + + if (err) { + next(err); + return; + } + + // shortest to longest + //route = apps.pop(); + // longest to shortest + route = apps.shift(); + if (!route) { + next(); + return; + } + + if (route.route) { + route.route(req, res, nextify); + return; + } + + if (route._errored) { + nextify(new Error("couldn't load api")); + return; + } + + if (!route.api) { + nextify(new Error("no api available for this route")); + return; + } + + getApi(route).then(function (route) { + try { + route(req, res, nextify); + route.route = route; + } catch(e) { + route._errored = true; + console.error('[App Load Error]'); + console.error(e.stack); + nextify(new Error("couldn't load api")); + } + }); + } + + nextify(); + } + + return { + api: api + }; +}; diff --git a/lib/insecure-server.js b/lib/insecure-server.js index 93a6f40..8638e25 100644 --- a/lib/insecure-server.js +++ b/lib/insecure-server.js @@ -12,8 +12,12 @@ module.exports.create = function (securePort, insecurePort, redirects) { var host = req.headers.host || ''; var url = req.url; + if (require('./unbrick-appcache').unbrick(req, res)) { + return; + } + // because I have domains for which I don't want to pay for SSL certs - insecureRedirects = redirects.sort(function (a, b) { + insecureRedirects = (redirects||[]).sort(function (a, b) { var hlen = b.from.hostname.length - a.from.hostname.length; var plen; if (!hlen) { diff --git a/lib/local-server.js b/lib/local-server.js index f0801e4..c96157c 100644 --- a/lib/local-server.js +++ b/lib/local-server.js @@ -13,12 +13,20 @@ module.exports.create = function (certPaths, port, serverCallback) { } server.on('error', serverCallback); - server.listen(port, function () { + server.listen(port, '0.0.0.0', function () { // is it even theoritically possible for // a request to come in before this callback has fired? // I'm assuming this event must fire before any request event promiseApp = serverCallback(null, server); }); + /* + server.listen(port, '::::', function () { + // is it even theoritically possible for + // a request to come in before this callback has fired? + // I'm assuming this event must fire before any request event + promiseApp = serverCallback(null, server); + }); + */ // Get up and listening as absolutely quickly as possible server.on('request', function (req, res) { @@ -29,6 +37,7 @@ module.exports.create = function (certPaths, port, serverCallback) { } promiseApp.then(function (_app) { + console.log('[Server]', req.method, req.host || req.headers['x-forwarded-host'] || req.headers.host, req.url); app = _app; app(req, res); }); diff --git a/lib/no-www.js b/lib/no-www.js new file mode 100644 index 0000000..8237483 --- /dev/null +++ b/lib/no-www.js @@ -0,0 +1,27 @@ +module.exports.scrubTheDub = function (req, res) { + // hack for bricked app-cache + // Also 301 redirects will not work for appcache (must issue html) + if (require('./unbrick-appcache').unbrick(req, res)) { + return; + } + + // TODO port number for non-443 + var escapeHtml = require('escape-html'); + var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url; + var safeLocation = escapeHtml(newLocation); + + var metaRedirect = '' + + '\n' + + '\n' + + ' \n' + + ' \n' + + '\n' + + '\n' + + '

You requested an old resource. Please use this instead: \n' + + ' ' + safeLocation + '

\n' + + '\n' + + '\n' + ; + + res.end(metaRedirect); +}; diff --git a/lib/schemes-config.js b/lib/schemes-config.js index d0e25b4..aa3772b 100644 --- a/lib/schemes-config.js +++ b/lib/schemes-config.js @@ -1,5 +1,7 @@ 'use strict'; +var getDomainInfo = require('../lib/utils').getDomainInfo; + function deserialize(results) { var config = { apis: {}, apps: {}, domains: {} }; results.apis.forEach(function (api) { @@ -70,7 +72,62 @@ function deserialize(results) { return config; } +function getVhostsMap(config) { + var vhosts = []; + var vhostsMap = {}; + + function sortApps(a, b) { + // hlen isn't important in this current use of the sorter, + // but is important for an alternate version + var hlen = b.hostname.length - a.hostname.length; + var plen = b.pathname.length - a.pathname.length; + + // A directory could be named example.com, example.com# example.com## + // to indicate order of preference (for API addons, for example) + var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length); + + if (!hlen) { + if (!plen) { + return dlen; + } + return plen; + } + return hlen; + } + + Object.keys(config.domains).forEach(function (domainname) { + var domain = config.domains[domainname]; + var info = getDomainInfo(domainname); + + domain.hostname = info.hostname; + domain.pathname = '/' + (info.pathname || ''); + domain.dirname = info.dirname; + + vhosts.push(domain); + }); + + vhosts.sort(sortApps); + + vhosts.forEach(function (domain) { + console.log(domain.hostname, domain.pathname, domain.dirname); + + if (!vhostsMap[domain.hostname]) { + vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] }; + } + + if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) { + vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] }; + vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]); + } + + vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain); + }); + + return vhostsMap; +} + module.exports.deserialize = deserialize; +module.exports.getVhostsMap = getVhostsMap; module.exports.create = function (db) { console.log('[DB -1]'); var wrap = require('dbwrap'); @@ -162,9 +219,7 @@ module.exports.create = function (db) { // create fixture with which to test // console.log(JSON.stringify(results)); - var config = deserialize(results); - - return config; + return getVhostsMap(deserialize(results)); }); } }; diff --git a/lib/spawn-caddy.js b/lib/spawn-caddy.js index b9cb96a..a983a2f 100644 --- a/lib/spawn-caddy.js +++ b/lib/spawn-caddy.js @@ -20,8 +20,12 @@ function tplCaddyfile(conf) { } content += - " proxy /api http://localhost:" + conf.localPort.toString() + "\n" + " proxy /api http://localhost:" + conf.localPort.toString() + " {\n" + + " proxy_header Host {host}\n" + + " proxy_header X-Forwarded-Host {host}\n" + + " proxy_header X-Forwarded-Proto {scheme}\n" // # TODO internal + + " }\n" + "}"; contents.push(content); diff --git a/lib/unbrick-appcache.js b/lib/unbrick-appcache.js new file mode 100644 index 0000000..6afa03e --- /dev/null +++ b/lib/unbrick-appcache.js @@ -0,0 +1,10 @@ +module.exports.unbrick = function (req, res) { + // hack for bricked app-cache + if (/\.(appcache|manifest)\b/.test(req.url)) { + res.setHeader('Content-Type', 'text/cache-manifest'); + res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*'); + return true; + } + + return false; +}; diff --git a/lib/worker.js b/lib/worker.js index cb9c21a..17968a5 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -7,8 +7,9 @@ module.exports.create = function (webserver, info, state) { var PromiseA = state.Promise || require('bluebird'); var path = require('path'); - var vhostsdir = path.join(__dirname, 'vhosts'); - var app = require('express')(); + //var vhostsdir = path.join(__dirname, 'vhosts'); + var express = require('express-lazy'); + var app = express(); var apiHandler; var memstore; var sqlstores = {}; @@ -58,89 +59,29 @@ module.exports.create = function (webserver, info, state) { } */ - function scrubTheDubHelper(req, res/*, next*/) { - // hack for bricked app-cache - if (/\.appcache\b/.test(req.url)) { - res.setHeader('Content-Type', 'text/cache-manifest'); - res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*'); - return; - } - - // TODO port number for non-443 - var escapeHtml = require('escape-html'); - var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url; - var safeLocation = escapeHtml(newLocation); - - var metaRedirect = '' - + '\n' - + '\n' - + ' \n' - + ' \n' - + '\n' - + '\n' - + '

You requested an old resource. Please use this instead: \n' - + ' ' + safeLocation + '

\n' - + '\n' - + '\n' - ; - - // 301 redirects will not work for appcache - res.end(metaRedirect); - } - // TODO handle insecure to actual redirect // blog.coolaj86.com -> coolaj86.com/blog // hmm... that won't really matter with hsts // I guess I just needs letsencrypt function scrubTheDub(req, res, next) { + console.log('[no-www]', req.method, req.url); var host = req.hostname; if (!host || 'string' !== typeof host) { next(); return; } + + // TODO test if this is even necessary host = host.toLowerCase(); - if (/^www\./.test(host)) { - scrubTheDubHelper(req, res, next); - return; - } - } - - function handleApi(req, res, next) { - if (!/^\/api/.test(req.url)) { + if (!/^www\./.test(host)) { next(); return; } - // TODO move to caddy parser? - if (/(^|\.)proxyable\./.test(req.hostname)) { - // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com - // proxyable.myapp.mydomain.com => myapp.mydomain.com - // TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com - req.hostname = req.hostname.replace(/.*\.?proxyable\./, ''); - } - - if (apiHandler) { - if (apiHandler.then) { - apiHandler.then(function (app) { - app(req, res, next); - }); - return; - } - - apiHandler(req, res, next); - return; - } - - apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app).then(function (app) { - // X-Forwarded-For - // X-Forwarded-Proto - console.log('api server', req.hostname, req.secure, req.ip); - apiHandler = app; - app(req, res, next); - }); + require('./no-www').scrubTheDub(req, res); } if (info.trustProxy) { @@ -148,7 +89,6 @@ module.exports.create = function (webserver, info, state) { //app.set('trust proxy', function (ip) { ... }); } app.use('/', scrubTheDub); - app.use('/', handleApi); return PromiseA.all([ cstore.create({ @@ -186,7 +126,7 @@ module.exports.create = function (webserver, info, state) { return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) { models.Config = tables; - models.Config.Config.get().then(function (circ) { + return models.Config.Config.get().then(function (vhostsMap) { /* // todo getDomainInfo @@ -195,7 +135,92 @@ module.exports.create = function (webserver, info, state) { utils.getDomainInfo(domain.id); }); */ - console.log(circ); + + function handleApi(req, res, next) { + console.log('[API]', req.method, req.url); + var myApp; + + if (!/^\/api/.test(req.url)) { + next(); + return; + } + + // TODO move to caddy parser? + if (/(^|\.)proxyable\./.test(req.hostname)) { + // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com + // proxyable.myapp.mydomain.com => myapp.mydomain.com + // TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com + req.hostname = req.hostname.replace(/.*\.?proxyable\./, ''); + } + + if (apiHandler) { + if (apiHandler.then) { + apiHandler.then(function (app) { + app(req, res, next); + }); + return; + } + + apiHandler(req, res, next); + return; + } + + // apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app) + myApp = express(); + apiHandler = require('./api-server').create( + { apppath: '../packages/apps/' + , apipath: '../packages/apis/' + , vhostsMap: vhostsMap + , server: webserver + , externalPort: info.externalPort + } + , { app: myApp + , memstore: memstore + , sqlstores: sqlstores + , clientSqlFactory: clientFactory + , systemSqlFactory: systemFactory + //, handlePromise: require('./lib/common').promisableRequest; + //, handleRejection: require('./lib/common').rejectableRequest; + //, localPort: info.localPort + } + ).api; + + // TODO + // X-Forwarded-For + // X-Forwarded-Proto + console.log('api server', req.hostname, req.secure, req.ip); + + apiHandler(req, res, next); + } + + // TODO recase + + // + // Generic Template API + // + app + .use(require('body-parser').json({ + strict: true // only objects and arrays + , inflate: true + // limited to due performance issues with JSON.parse and JSON.stringify + // http://josh.zeigler.us/technology/web-development/how-big-is-too-big-for-json/ + //, limit: 128 * 1024 + , limit: 1.5 * 1024 * 1024 + , reviver: undefined + , type: 'json' + , verify: undefined + })) + // DO NOT allow urlencoded at any point, it is expressly forbidden + //.use(require('body-parser').urlencoded({ + // extended: true + //, inflate: true + //, limit: 100 * 1024 + //, type: 'urlencoded' + //, verify: undefined + //})) + .use(require('connect-send-error').error()) + ; + app.use('/', handleApi); return app; }); diff --git a/backends/.gitkeep b/packages/apis/.gitkeep similarity index 100% rename from backends/.gitkeep rename to packages/apis/.gitkeep diff --git a/packages/apps/.gitkeep b/packages/apps/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/init.public/index.html b/packages/apps/com.daplie.cloud.lockscreen/index.html similarity index 100% rename from init.public/index.html rename to packages/apps/com.daplie.cloud.lockscreen/index.html diff --git a/tests/schemes-config.js b/tests/schemes-config.js index aa023a6..358e34b 100644 --- a/tests/schemes-config.js +++ b/tests/schemes-config.js @@ -1,9 +1,13 @@ 'use strict'; +var deserialize = require('../lib/schemes-config').deserialize; +var getVhostsMap = require('../lib/schemes-config').getVhostsMap; +var getDomainInfo = require('../lib/utils').getDomainInfo; + // var results = {"apis":[{"id":"oauth3-api","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"apps":[{"id":"oauth3-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"hellabit-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsio-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsconnect-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"domains":[{"id":"oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com#connect","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null}],"apisDomains":[{"id":"oauth3-api_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"apiId":"oauth3-api","domainId":"oauth3.org","json":null}],"appsDomains":[{"id":"oauth3-app_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"oauth3-app","domainId":"oauth3.org","json":null},{"id":"hellabit-app_hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"hellabit-app","domainId":"hellabit.com","json":null},{"id":"ldsio-app_lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsio-app","domainId":"lds.io","json":null},{"id":"ldsconnect-app_ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsconnect-app","domainId":"ldsconnect.org","json":null}]}; var results = { "apis":[ - {"id":"oauth3-api"} + {"id":"org.oauth3"} ] , "apps":[ {"id":"oauth3-app"} @@ -23,7 +27,8 @@ var results = { , {"id":"hellabit.com#connect#too"} ] , "apisDomains":[ - {"id":"oauth3-api_oauth3.org","apiId":"oauth3-api","domainId":"oauth3.org"} + {"id":"org.oauth3_oauth3.org","apiId":"org.oauth3","domainId":"oauth3.org"} + , {"id":"org.oauth3_hellabit.com#connect###","apiId":"org.oauth3","domainId":"hellabit.com#connect###"} ] ,"appsDomains":[ {"id":"oauth3-app_oauth3.org","appId":"oauth3-app","domainId":"oauth3.org"} @@ -34,127 +39,9 @@ var results = { ] }; -var deserialize = require('../lib/schemes-config').deserialize; -var getDomainInfo = require('../lib/utils').getDomainInfo; -var config = deserialize(results); var req = { host: 'hellabit.com', url: '/connect' }; -var vhosts = []; -var vhostsMap = {}; - -function sortApps(a, b) { - // hlen isn't important in this current use of the sorter, - // but is important for an alternate version - var hlen = b.hostname.length - a.hostname.length; - var plen = b.pathname.length - a.pathname.length; - - // A directory could be named example.com, example.com# example.com## - // to indicate order of preference (for API addons, for example) - var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length); - - if (!hlen) { - if (!plen) { - return dlen; - } - return plen; - } - return hlen; -} - -Object.keys(config.domains).forEach(function (domainname) { - var domain = config.domains[domainname]; - var info = getDomainInfo(domainname); - - domain.hostname = info.hostname; - domain.pathname = '/' + (info.pathname || ''); - domain.dirname = info.dirname; - - vhosts.push(domain); -}); - -vhosts.sort(sortApps); - -vhosts.forEach(function (domain) { - console.log(domain.hostname, domain.pathname, domain.dirname); - - if (!vhostsMap[domain.hostname]) { - vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] }; - } - - if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) { - vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] }; - vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]); - } - - vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain); -}); - -if (!vhostsMap[req.host]) { - console.log("there's no app for this hostname"); - return; -} - -//console.log("load an app", vhosts[req.host]); - -//console.log(vhosts[req.host]); - - -function getApp(route) { - var PromiseA = require('bluebird'); - - return new PromiseA(function (resolve, reject) { - console.log(route); - // route.hostname - }); -} - -function api(req, res, next) { - var apps; - - vhostsMap[req.host].pathnames.some(function (route) { - // /connect / - if (req.url.match(route.pathname) && route.pathname.match(req.url)) { - apps = route.apps; - return true; - } - }); - - //console.log(apps); - - function nextify(err) { - var route; - - if (err) { - next(err); - return; - } - - // shortest to longest - //route = apps.pop(); - // longest to shortest - route = apps.shift(); - if (!route) { - next(); - return; - } - - if (route.route) { - route.route(req, res, nextify); - return; - } - - getApp(route).then(function (route) { - route.route = route; - try { - route.route(req, res, nextify); - } catch(e) { - console.error('[App Load Error]'); - console.error(e.stack); - nextify(new Error("couldn't load app")); - } - }); - } - - nextify(); -} - -api(req); +module.exports.create({ + apppath: '../packages/apps/' +, apipath: '../packages/apis/' +, vhostsMap: vhostsMap +}).api(req); diff --git a/worker.js b/worker.js index 888ad01..bdc9ec8 100644 --- a/worker.js +++ b/worker.js @@ -12,7 +12,6 @@ function waitForInit(message) { var msg = message.conf; process.removeListener('message', waitForInit); - require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) { if (err) { console.error('[ERROR] worker.js'); @@ -38,6 +37,9 @@ function waitForInit(message) { process.on('message', initWebServer); }); }); + + // TODO conditional if 80 is being served by caddy + require('./lib/insecure-server').create(msg.externalPort, msg.insecurePort); } // We have to wait to get the configuration from the master process