coolaj86
/
walnut.js
1
1
Fork 1
Code Issues Pull Requests Releases Activity

load apps from DB

This commit is contained in:
AJ ONeal 2015-11-14 04:25:12 +00:00
parent 1bb21f5a07
commit 65645a7602
14 changed files with 360 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -6,6 +6,8 @@ dyndns-token.js
vhosts
certs
.*.sw*
packages
var
# Logs
logs

133
lib/api-server.js Normal file
View File

@ -0,0 +1,133 @@
'use strict';
// TODO handle static app urls?
// NOTE rejecting non-api urls should happen before this
module.exports.create = function (conf, deps, app) {
var escapeStringRegexp = require('escape-string-regexp');
var vhostsMap = conf.vhostsMap;
if (!app) {
app = deps.app;
}
function getApi(route) {
// TODO don't modify route, modify some other variable instead
var PromiseA = require('bluebird');
var path = require('path');
console.log(route);
// TODO needs some version stuff (which would also allow hot-loading of updates)
// TODO version could be tied to sha256sum
var pkgpath = path.join(conf.apipath, (route.api.package || route.api.id), (route.api.version || ''));
console.log('pkgpath');
console.log(pkgpath);
return new PromiseA(function (resolve, reject) {
try {
route.route = require(pkgpath).create(conf, deps.app, app);
} catch(e) {
reject(e);
return;
}
resolve(route.route);
});
}
function api(req, res, next) {
var apps;
console.log('hostname', req.hostname);
console.log('headers', req.headers);
if (!vhostsMap[req.hostname]) {
// TODO keep track of match-only vhosts, such as '*.example.com',
// separate from exact matches
next(new Error("this domain is not registered"));
return;
}
vhostsMap[req.hostname].pathnames.some(function (route) {
var pathname = route.pathname;
if ('/' === pathname) {
pathname = '/api';
}
if (-1 === pathname.indexOf('/api')) {
pathname = '/api' + pathname;
}
if (!route.re) {
route.re = new RegExp(escapeStringRegexp(pathname) + '(#|\\/|\\?|$)');
}
// re.test("/api")
// re.test("/api?")
// re.test("/api/")
// re.test("/api/foo")
// re.test("/apifoo") // false
if (route.re.test(req.url)) {
apps = route.apps;
return true;
}
});
if (!apps) {
console.log('No apps to try for this hostname');
console.log(vhostsMap[req.hostname]);
next();
return;
}
//console.log(apps);
function nextify(err) {
var route;
if (err) {
next(err);
return;
}
// shortest to longest
//route = apps.pop();
// longest to shortest
route = apps.shift();
if (!route) {
next();
return;
}
if (route.route) {
route.route(req, res, nextify);
return;
}
if (route._errored) {
nextify(new Error("couldn't load api"));
return;
}
if (!route.api) {
nextify(new Error("no api available for this route"));
return;
}
getApi(route).then(function (route) {
try {
route(req, res, nextify);
route.route = route;
} catch(e) {
route._errored = true;
console.error('[App Load Error]');
console.error(e.stack);
nextify(new Error("couldn't load api"));
}
});
}
nextify();
}
return {
api: api
};
};

6
lib/insecure-server.js
View File

@ -12,8 +12,12 @@ module.exports.create = function (securePort, insecurePort, redirects) {
var host = req.headers.host || '';
var url = req.url;
if (require('./unbrick-appcache').unbrick(req, res)) {
return;
}
// because I have domains for which I don't want to pay for SSL certs
insecureRedirects = redirects.sort(function (a, b) {
insecureRedirects = (redirects||[]).sort(function (a, b) {
var hlen = b.from.hostname.length - a.from.hostname.length;
var plen;
if (!hlen) {

11
lib/local-server.js
View File

@ -13,12 +13,20 @@ module.exports.create = function (certPaths, port, serverCallback) {
}
server.on('error', serverCallback);
server.listen(port, function () {
server.listen(port, '0.0.0.0', function () {
// is it even theoritically possible for
// a request to come in before this callback has fired?
// I'm assuming this event must fire before any request event
promiseApp = serverCallback(null, server);
});
/*
server.listen(port, '::::', function () {
// is it even theoritically possible for
// a request to come in before this callback has fired?
// I'm assuming this event must fire before any request event
promiseApp = serverCallback(null, server);
});
*/
// Get up and listening as absolutely quickly as possible
server.on('request', function (req, res) {
@ -29,6 +37,7 @@ module.exports.create = function (certPaths, port, serverCallback) {
}
promiseApp.then(function (_app) {
console.log('[Server]', req.method, req.host || req.headers['x-forwarded-host'] || req.headers.host, req.url);
app = _app;
app(req, res);
});

27
lib/no-www.js Normal file
View File

@ -0,0 +1,27 @@
module.exports.scrubTheDub = function (req, res) {
// hack for bricked app-cache
// Also 301 redirects will not work for appcache (must issue html)
if (require('./unbrick-appcache').unbrick(req, res)) {
return;
}
// TODO port number for non-443
var escapeHtml = require('escape-html');
var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an old resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
res.end(metaRedirect);
};

61
lib/schemes-config.js
View File

@ -1,5 +1,7 @@
'use strict';
var getDomainInfo = require('../lib/utils').getDomainInfo;
function deserialize(results) {
var config = { apis: {}, apps: {}, domains: {} };
results.apis.forEach(function (api) {
@ -70,7 +72,62 @@ function deserialize(results) {
return config;
}
function getVhostsMap(config) {
var vhosts = [];
var vhostsMap = {};
function sortApps(a, b) {
// hlen isn't important in this current use of the sorter,
// but is important for an alternate version
var hlen = b.hostname.length - a.hostname.length;
var plen = b.pathname.length - a.pathname.length;
// A directory could be named example.com, example.com# example.com##
// to indicate order of preference (for API addons, for example)
var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
if (!hlen) {
if (!plen) {
return dlen;
}
return plen;
}
return hlen;
}
Object.keys(config.domains).forEach(function (domainname) {
var domain = config.domains[domainname];
var info = getDomainInfo(domainname);
domain.hostname = info.hostname;
domain.pathname = '/' + (info.pathname || '');
domain.dirname = info.dirname;
vhosts.push(domain);
});
vhosts.sort(sortApps);
vhosts.forEach(function (domain) {
console.log(domain.hostname, domain.pathname, domain.dirname);
if (!vhostsMap[domain.hostname]) {
vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] };
}
if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) {
vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] };
vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]);
}
vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain);
});
return vhostsMap;
}
module.exports.deserialize = deserialize;
module.exports.getVhostsMap = getVhostsMap;
module.exports.create = function (db) {
console.log('[DB -1]');
var wrap = require('dbwrap');
@ -162,9 +219,7 @@ module.exports.create = function (db) {
// create fixture with which to test
// console.log(JSON.stringify(results));
var config = deserialize(results);
return config;
return getVhostsMap(deserialize(results));
});
}
};

6
lib/spawn-caddy.js
View File

@ -20,8 +20,12 @@ function tplCaddyfile(conf) {
}
content +=
" proxy /api http://localhost:" + conf.localPort.toString() + "\n"
" proxy /api http://localhost:" + conf.localPort.toString() + " {\n"
+ " proxy_header Host {host}\n"
+ " proxy_header X-Forwarded-Host {host}\n"
+ " proxy_header X-Forwarded-Proto {scheme}\n"
// # TODO internal
+ " }\n"
+ "}";
contents.push(content);

10
lib/unbrick-appcache.js Normal file
View File

@ -0,0 +1,10 @@
module.exports.unbrick = function (req, res) {
// hack for bricked app-cache
if (/\.(appcache|manifest)\b/.test(req.url)) {
res.setHeader('Content-Type', 'text/cache-manifest');
res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*');
return true;
}
return false;
};

165
lib/worker.js
View File

@ -7,8 +7,9 @@ module.exports.create = function (webserver, info, state) {
var PromiseA = state.Promise || require('bluebird');
var path = require('path');
var vhostsdir = path.join(__dirname, 'vhosts');
var app = require('express')();
//var vhostsdir = path.join(__dirname, 'vhosts');
var express = require('express-lazy');
var app = express();
var apiHandler;
var memstore;
var sqlstores = {};
@ -58,89 +59,29 @@ module.exports.create = function (webserver, info, state) {
}
*/
function scrubTheDubHelper(req, res/*, next*/) {
// hack for bricked app-cache
if (/\.appcache\b/.test(req.url)) {
res.setHeader('Content-Type', 'text/cache-manifest');
res.end('CACHE MANIFEST\n\n# v0__DELETE__CACHE__MANIFEST__\n\nNETWORK:\n*');
return;
}
// TODO port number for non-443
var escapeHtml = require('escape-html');
var newLocation = 'https://' + req.hostname.replace(/^www\./, '') + req.url;
var safeLocation = escapeHtml(newLocation);
var metaRedirect = ''
+ '<html>\n'
+ '<head>\n'
+ ' <style>* { background-color: white; color: white; text-decoration: none; }</style>\n'
+ ' <META http-equiv="refresh" content="0;URL=' + safeLocation + '">\n'
+ '</head>\n'
+ '<body style="display: none;">\n'
+ ' <p>You requested an old resource. Please use this instead: \n'
+ ' <a href="' + safeLocation + '">' + safeLocation + '</a></p>\n'
+ '</body>\n'
+ '</html>\n'
;
// 301 redirects will not work for appcache
res.end(metaRedirect);
}
// TODO handle insecure to actual redirect
// blog.coolaj86.com -> coolaj86.com/blog
// hmm... that won't really matter with hsts
// I guess I just needs letsencrypt
function scrubTheDub(req, res, next) {
console.log('[no-www]', req.method, req.url);
var host = req.hostname;
if (!host || 'string' !== typeof host) {
next();
return;
}
// TODO test if this is even necessary
host = host.toLowerCase();
if (/^www\./.test(host)) {
scrubTheDubHelper(req, res, next);
return;
}
}
function handleApi(req, res, next) {
if (!/^\/api/.test(req.url)) {
if (!/^www\./.test(host)) {
next();
return;
}
// TODO move to caddy parser?
if (/(^|\.)proxyable\./.test(req.hostname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com
req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
}
if (apiHandler) {
if (apiHandler.then) {
apiHandler.then(function (app) {
app(req, res, next);
});
return;
}
apiHandler(req, res, next);
return;
}
apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app).then(function (app) {
// X-Forwarded-For
// X-Forwarded-Proto
console.log('api server', req.hostname, req.secure, req.ip);
apiHandler = app;
app(req, res, next);
});
require('./no-www').scrubTheDub(req, res);
}
if (info.trustProxy) {
@ -148,7 +89,6 @@ module.exports.create = function (webserver, info, state) {
//app.set('trust proxy', function (ip) { ... });
}
app.use('/', scrubTheDub);
app.use('/', handleApi);
return PromiseA.all([
cstore.create({
@ -186,7 +126,7 @@ module.exports.create = function (webserver, info, state) {
return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) {
models.Config = tables;
models.Config.Config.get().then(function (circ) {
return models.Config.Config.get().then(function (vhostsMap) {
/*
// todo getDomainInfo
@ -195,7 +135,92 @@ module.exports.create = function (webserver, info, state) {
utils.getDomainInfo(domain.id);
});
*/
console.log(circ);
function handleApi(req, res, next) {
console.log('[API]', req.method, req.url);
var myApp;
if (!/^\/api/.test(req.url)) {
next();
return;
}
// TODO move to caddy parser?
if (/(^|\.)proxyable\./.test(req.hostname)) {
// device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com
// proxyable.myapp.mydomain.com => myapp.mydomain.com
// TODO myapp.mydomain.com.daplieproxyable.com => myapp.mydomain.com
req.hostname = req.hostname.replace(/.*\.?proxyable\./, '');
}
if (apiHandler) {
if (apiHandler.then) {
apiHandler.then(function (app) {
app(req, res, next);
});
return;
}
apiHandler(req, res, next);
return;
}
// apiHandler = require('./vhost-server').create(info.localPort, vhostsdir).create(webserver, app)
myApp = express();
apiHandler = require('./api-server').create(
{ apppath: '../packages/apps/'
, apipath: '../packages/apis/'
, vhostsMap: vhostsMap
, server: webserver
, externalPort: info.externalPort
}
, { app: myApp
, memstore: memstore
, sqlstores: sqlstores
, clientSqlFactory: clientFactory
, systemSqlFactory: systemFactory
//, handlePromise: require('./lib/common').promisableRequest;
//, handleRejection: require('./lib/common').rejectableRequest;
//, localPort: info.localPort
}
).api;
// TODO
// X-Forwarded-For
// X-Forwarded-Proto
console.log('api server', req.hostname, req.secure, req.ip);
apiHandler(req, res, next);
}
// TODO recase
//
// Generic Template API
//
app
.use(require('body-parser').json({
strict: true // only objects and arrays
, inflate: true
// limited to due performance issues with JSON.parse and JSON.stringify
// http://josh.zeigler.us/technology/web-development/how-big-is-too-big-for-json/
//, limit: 128 * 1024
, limit: 1.5 * 1024 * 1024
, reviver: undefined
, type: 'json'
, verify: undefined
}))
// DO NOT allow urlencoded at any point, it is expressly forbidden
//.use(require('body-parser').urlencoded({
// extended: true
//, inflate: true
//, limit: 100 * 1024
//, type: 'urlencoded'
//, verify: undefined
//}))
.use(require('connect-send-error').error())
;
app.use('/', handleApi);
return app;
});

0
backends/.gitkeep → packages/apis/.gitkeep
View File

0
packages/apps/.gitkeep Normal file
View File

0
init.public/index.html → packages/apps/com.daplie.cloud.lockscreen/index.html
View File

137
tests/schemes-config.js
View File

@ -1,9 +1,13 @@
'use strict';
var deserialize = require('../lib/schemes-config').deserialize;
var getVhostsMap = require('../lib/schemes-config').getVhostsMap;
var getDomainInfo = require('../lib/utils').getDomainInfo;
// var results = {"apis":[{"id":"oauth3-api","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"apps":[{"id":"oauth3-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"hellabit-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsio-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null},{"id":"ldsconnect-app","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"json":null}],"domains":[{"id":"oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null},{"id":"hellabit.com#connect","createdAt":null,"updatedAt":null,"deletedAt":null,"revokedAt":null,"name":null,"token":null,"accountId":null,"json":null}],"apisDomains":[{"id":"oauth3-api_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"apiId":"oauth3-api","domainId":"oauth3.org","json":null}],"appsDomains":[{"id":"oauth3-app_oauth3.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"oauth3-app","domainId":"oauth3.org","json":null},{"id":"hellabit-app_hellabit.com","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"hellabit-app","domainId":"hellabit.com","json":null},{"id":"ldsio-app_lds.io","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsio-app","domainId":"lds.io","json":null},{"id":"ldsconnect-app_ldsconnect.org","createdAt":null,"updatedAt":null,"deletedAt":null,"appId":"ldsconnect-app","domainId":"ldsconnect.org","json":null}]};
var results = {
"apis":[
{"id":"oauth3-api"}
{"id":"org.oauth3"}
]
, "apps":[
{"id":"oauth3-app"}
@ -23,7 +27,8 @@ var results = {
, {"id":"hellabit.com#connect#too"}
]
, "apisDomains":[
{"id":"oauth3-api_oauth3.org","apiId":"oauth3-api","domainId":"oauth3.org"}
{"id":"org.oauth3_oauth3.org","apiId":"org.oauth3","domainId":"oauth3.org"}
, {"id":"org.oauth3_hellabit.com#connect###","apiId":"org.oauth3","domainId":"hellabit.com#connect###"}
]
,"appsDomains":[
{"id":"oauth3-app_oauth3.org","appId":"oauth3-app","domainId":"oauth3.org"}
@ -34,127 +39,9 @@ var results = {
]
};
var deserialize = require('../lib/schemes-config').deserialize;
var getDomainInfo = require('../lib/utils').getDomainInfo;
var config = deserialize(results);
var req = { host: 'hellabit.com', url: '/connect' };
var vhosts = [];
var vhostsMap = {};
function sortApps(a, b) {
// hlen isn't important in this current use of the sorter,
// but is important for an alternate version
var hlen = b.hostname.length - a.hostname.length;
var plen = b.pathname.length - a.pathname.length;
// A directory could be named example.com, example.com# example.com##
// to indicate order of preference (for API addons, for example)
var dlen = (b.priority || b.dirname.length) - (a.priority || a.dirname.length);
if (!hlen) {
if (!plen) {
return dlen;
}
return plen;
}
return hlen;
}
Object.keys(config.domains).forEach(function (domainname) {
var domain = config.domains[domainname];
var info = getDomainInfo(domainname);
domain.hostname = info.hostname;
domain.pathname = '/' + (info.pathname || '');
domain.dirname = info.dirname;
vhosts.push(domain);
});
vhosts.sort(sortApps);
vhosts.forEach(function (domain) {
console.log(domain.hostname, domain.pathname, domain.dirname);
if (!vhostsMap[domain.hostname]) {
vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] };
}
if (!vhostsMap[domain.hostname].pathnamesMap[domain.pathname]) {
vhostsMap[domain.hostname].pathnamesMap[domain.pathname] = { pathname: domain.pathname, apps: [] };
vhostsMap[domain.hostname].pathnames.push(vhostsMap[domain.hostname].pathnamesMap[domain.pathname]);
}
vhostsMap[domain.hostname].pathnamesMap[domain.pathname].apps.push(domain);
});
if (!vhostsMap[req.host]) {
console.log("there's no app for this hostname");
return;
}
//console.log("load an app", vhosts[req.host]);
//console.log(vhosts[req.host]);
function getApp(route) {
var PromiseA = require('bluebird');
return new PromiseA(function (resolve, reject) {
console.log(route);
// route.hostname
});
}
function api(req, res, next) {
var apps;
vhostsMap[req.host].pathnames.some(function (route) {
// /connect /
if (req.url.match(route.pathname) && route.pathname.match(req.url)) {
apps = route.apps;
return true;
}
});
//console.log(apps);
function nextify(err) {
var route;
if (err) {
next(err);
return;
}
// shortest to longest
//route = apps.pop();
// longest to shortest
route = apps.shift();
if (!route) {
next();
return;
}
if (route.route) {
route.route(req, res, nextify);
return;
}
getApp(route).then(function (route) {
route.route = route;
try {
route.route(req, res, nextify);
} catch(e) {
console.error('[App Load Error]');
console.error(e.stack);
nextify(new Error("couldn't load app"));
}
});
}
nextify();
}
api(req);
module.exports.create({
apppath: '../packages/apps/'
, apipath: '../packages/apis/'
, vhostsMap: vhostsMap
}).api(req);

4
worker.js
View File

@ -12,7 +12,6 @@ function waitForInit(message) {
var msg = message.conf;
process.removeListener('message', waitForInit);
require('./lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) {
if (err) {
console.error('[ERROR] worker.js');
@ -38,6 +37,9 @@ function waitForInit(message) {
process.on('message', initWebServer);
});
});
// TODO conditional if 80 is being served by caddy
require('./lib/insecure-server').create(msg.externalPort, msg.insecurePort);
}
// We have to wait to get the configuration from the master process