WIP load apis by file structure

This commit is contained in:
AJ ONeal 2017-05-19 23:37:28 +00:00
parent 3d5eaf94bd
commit c1735d5c03
6 changed files with 211 additions and 274 deletions

View File

@ -218,7 +218,7 @@ install_my_app()
ln -sf ../node_modules /srv/walnut/core/node_modules
sudo mkdir -p /srv/walnut/etc/org.oauth3.consumer
sudo mkdir -p /srv/walnut/etc/org.oauth3.provider
sudo mkdir -p /srv/walnut/packages/{api,pages,services}
sudo mkdir -p /srv/walnut/packages/{client-api-grants,rest,api,pages,services}
#sudo chown -R $(whoami):$(whoami) /srv/walnut
sudo chown -R www-data:www-data /srv/walnut
sudo chmod -R ug+Xrw /srv/walnut

View File

@ -5,219 +5,109 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
var express = require('express');
var fs = PromiseA.promisifyAll(require('fs'));
var path = require('path');
var localCache = { apis: {}, pkgs: {} };
var localCache = { rests: {}, pkgs: {} };
// TODO xconfx.apispath
xconfx.apispath = path.join(__dirname, '..', '..', 'packages', 'apis');
xconfx.restPath = path.join(__dirname, '..', '..', 'packages', 'rest');
xconfx.appApiGrantsPath = path.join(__dirname, '..', '..', 'packages', 'client-api-grants');
function notConfigured(req, res) {
res.send({ error: { message: "api '" + req.apiId + "' not configured for domain '" + req.experienceId + "'" } });
res.send({ error: { message: "api '" + req.pkgId + "' not configured for domain '" + req.experienceId + "'" } });
}
function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
function handlePromise(p) {
return p.then(function (api) {
packagedApi._api = api;
return api;
});
}
if (!packagedApi._promise_api) {
packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
}
return handlePromise(packagedApi._promise_api);
/*
function isThisPkgInstalled(myConf, pkgId) {
}
*/
function getApi(conf, pkgConf, pkgDeps, packagedApi) {
var PromiseA = pkgDeps.Promise;
var path = require('path');
var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
function isThisClientAllowedToUseThisPkg(myConf, clientUrih, pkgId) {
var appApiGrantsPath = path.join(myConf.appApiGrantsPath, clientUrih);
// TODO needs some version stuff (which would also allow hot-loading of updates)
// TODO version could be tied to sha256sum
return new PromiseA(function (resolve, reject) {
var myApp;
var ursa;
var promise;
// TODO dynamic requires are a no-no
// can we statically generate a require-er? on each install?
// module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
// requirer[pkgpath]()
myApp = pkgDeps.express();
myApp.disable('x-powered-by');
if (pkgDeps.app.get('trust proxy')) {
myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
}
if (!pkgConf.pubkey) {
/*
return ursa.createPrivateKey(pem, password, encoding);
var pem = myKey.toPrivatePem();
return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
});
*/
ursa = require('ursa');
pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
}
try {
packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
packagedApi._apiname = packagedApi._apipkg.name;
if (packagedApi._apipkg.walnut) {
pkgpath += '/' + packagedApi._apipkg.walnut;
return fs.readFileAsync(appApiGrantsPath, 'utf8').then(function (text) {
return text.trim().split(/\n/);
}, function (/*err*/) {
return [];
}).then(function (apis) {
if (!apis.some(function (api) {
if (api === pkgId) {
return true;
}
promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
} catch(e) {
reject(e);
return;
})) {
if (clientUrih === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === pkgId) {
// fallthrough
return true;
} else {
return null;
}
}
});
}
function loadRestHelper(myConf, pkgId) {
var pkgPath = path.join(myConf.restPath, pkgId);
return fs.readFileAsync(path.join(pkgPath, 'package.json'), 'utf8').then(function (text) {
var pkg = JSON.parse(text);
var deps = {};
var myApp;
if (pkg.walnut) {
pkgPath = path.join(pkgPath, pkg.walnut);
}
promise.then(function () {
// TODO give pub/priv pair for app and all public keys
// packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
packagedApi._api = require('express-lazy')();
packagedApi._api_app = myApp;
Object.keys(apiDeps).forEach(function (key) {
deps[key] = apiDeps[key];
});
Object.keys(apiFactories).forEach(function (key) {
deps[key] = apiFactories[key];
});
//require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
pkgDeps.getOauth3Controllers =
packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
// TODO pull db stuff from package.json somehow and pass allowed data models as deps
//
// how can we tell which of these would be correct?
// deps.memstore = apiFactories.memstoreFactory.create(pkgId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + pkgId);
// DEBUG
//
/*
packagedApi._api.use('/', function (req, res, next) {
console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
next();
});
//*/
// let's go with this one for now and the api can choose to scope or not to scope
deps.memstore = apiFactories.memstoreFactory.create(pkgId);
// TODO fix backwards compat
// /api/com.example.foo (no change)
packagedApi._api.use('/', packagedApi._api_app);
// /api/com.example.foo => /api
packagedApi._api.use('/', function (req, res, next) {
var priorUrl = req.url;
req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
// console.log('api mangle 3:', req.url);
packagedApi._api_app(req, res, function (err) {
req.url = priorUrl;
next(err);
});
});
// /api/com.example.foo => /
packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
// console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
// console.log(packagedApi._api_app.toString());
packagedApi._api_app(req, res, next);
});
resolve(packagedApi._api);
}, reject);
console.log('DEBUG pkgPath', pkgPath);
myApp = express();
//
// TODO handle /accounts/:accountId
//
return PromiseA.resolve(require(pkgPath).create({
etcpath: xconfx.etcpath
}/*pkgConf*/, deps/*pkgDeps*/, myApp/*myApp*/)).then(function (handler) {
localCache.pkgs[pkgId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
return localCache.pkgs[pkgId];
});
});
}
// Read packages/apis/sub.sld.tld (forward dns) to find list of apis as tld.sld.sub (reverse dns)
// TODO packages/allowed_apis/sub.sld.tld (?)
// TODO auto-register org.oauth3.consumer for primaryDomain (and all sites?)
function loadApiHandler() {
return function handler(req, res, next) {
var name = req.experienceId;
var apiId = req.apiId;
var packagepath = path.join(xconfx.apispath, name);
function loadRestHandler(myConf, pkgId) {
return PromiseA.resolve().then(function () {
if (!localCache.pkgs[pkgId]) {
return loadRestHelper(myConf, pkgId);
}
return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
return text.trim().split(/\n/);
}, function () {
return [];
}).then(function (apis) {
return function (req, res, next) {
var apipath;
if (!apis.some(function (api) {
if (api === apiId) {
return true;
}
})) {
if (req.experienceId === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === apiId) {
// fallthrough
} else {
return null;
}
}
apipath = path.join(xconfx.apispath, apiId);
if (!localCache.pkgs[apiId]) {
return fs.readFileAsync(path.join(apipath, 'package.json'), 'utf8').then(function (text) {
var pkg = JSON.parse(text);
var deps = {};
var myApp;
if (pkg.walnut) {
apipath = path.join(apipath, pkg.walnut);
}
Object.keys(apiDeps).forEach(function (key) {
deps[key] = apiDeps[key];
});
Object.keys(apiFactories).forEach(function (key) {
deps[key] = apiFactories[key];
});
// TODO pull db stuff from package.json somehow and pass allowed data models as deps
//
// how can we tell which of these would be correct?
// deps.memstore = apiFactories.memstoreFactory.create(apiId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId);
// deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + apiId);
// let's go with this one for now and the api can choose to scope or not to scope
deps.memstore = apiFactories.memstoreFactory.create(apiId);
console.log('DEBUG apipath', apipath);
myApp = express();
//
// TODO handle /accounts/:accountId
//
return PromiseA.resolve(require(apipath).create({
etcpath: xconfx.etcpath
}/*pkgConf*/, deps/*pkgDeps*/, myApp/*myApp*/)).then(function (handler) {
localCache.pkgs[apiId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
localCache.pkgs[apiId].handler(req, res, next);
});
});
}
else {
localCache.pkgs[apiId].handler(req, res, next);
// TODO expire require cache
/*
if (Date.now() - localCache.pkgs[apiId].createdAt < (5 * 60 * 1000)) {
return;
}
*/
}
};
}, function (/*err*/) {
return null;
}).then(function (handler) {
// keep object reference intact
// DO NOT cache non-existant api
if (handler) {
localCache.apis[name].handler = handler;
} else {
handler = notConfigured;
}
handler(req, res, next);
});
};
return localCache.pkgs[pkgId];
// TODO expire require cache
/*
if (Date.now() - localCache.pkgs[pkgId].createdAt < (5 * 60 * 1000)) {
return;
}
*/
}, function (/*err*/) {
// TODO what kind of errors might we want to handle?
return null;
}).then(function (restPkg) {
return restPkg;
});
}
var CORS = require('connect-cors');
@ -228,36 +118,78 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
, 'Accept'
, 'Authorization'
], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
var staleAfter = (5 * 60 * 1000);
return function (req, res, next) {
cors(req, res, function () {
var experienceId = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
var apiId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
var clientUrih = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
var pkgId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
var now = Date.now();
var hasBeenHandled = false;
// Existing (Deprecated)
Object.defineProperty(req, 'experienceId', {
enumerable: true
, configurable: false
, writable: false
, value: clientUrih
});
Object.defineProperty(req, 'pkgId', {
enumerable: true
, configurable: false
, writable: false
, value: pkgId
});
// New
Object.defineProperty(req, 'clientUrih', {
enumerable: true
, configurable: false
, writable: false
// TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
// (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
// NOTE: probably best to alias the name logically
, value: experienceId
, value: clientUrih
});
Object.defineProperty(req, 'apiId', {
Object.defineProperty(req, 'pkgId', {
enumerable: true
, configurable: false
, writable: false
, value: apiId
, value: pkgId
});
if (!localCache.apis[experienceId]) {
localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
}
// TODO cache permission (although the FS is already cached, NBD)
return isThisClientAllowedToUseThisPkg(xconfx, clientUrih, pkgId).then(function (yes) {
if (!yes) {
notConfigured(req, res);
return null;
}
localCache.apis[experienceId].handler(req, res, next);
if (Date.now() - localCache.apis[experienceId].createdAt > (5 * 60 * 1000)) {
localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
}
if (localCache.rests[pkgId]) {
localCache.rests[pkgId].handler(req, res, next);
hasBeenHandled = true;
}
if (now - localCache.rests[pkgId].createdAt > staleAfter) {
localCache.rests[pkgId] = null;
}
if (!localCache.rests[pkgId]) {
//return doesThisPkgExist
return loadRestHandler(xconfx, pkgId).then(function (myHandler) {
if (!myHandler) {
notConfigured(req, res);
return;
}
localCache.rests[pkgId] = { handler: myHandler.handle, createdAt: now };
if (!hasBeenHandled) {
myHandler.handle(req, res, next);
}
});
}
});
});
};
};

38
lib/bootstrap.js vendored
View File

@ -40,41 +40,6 @@ module.exports.create = function (app, xconfx, models) {
var getIpAddresses = require('./ip-checker').getExternalAddresses;
var resolveInit;
function errorIfNotApi(req, res, next) {
var hostname = req.hostname || req.headers.host;
if (!/^api\.[a-z0-9\-]+/.test(hostname)) {
res.send({ error:
{ message: "API access is restricted to proper 'api'-prefixed lowercase subdomains."
+ " The HTTP 'Host' header must exist and must begin with 'api.' as in 'api.example.com'."
+ " For development you may test with api.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
, code: 'E_NOT_API'
, _hostname: hostname
}
});
return;
}
next();
}
function errorIfApi(req, res, next) {
if (!/^api\./.test(req.headers.host)) {
next();
return;
}
// has api. hostname prefix
// doesn't have /api url prefix
if (!/^\/api\//.test(req.url)) {
res.send({ error: { message: "missing /api/ url prefix" } });
return;
}
res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
}
function getConfig(req, res) {
getIpAddresses().then(function (inets) {
var results = {
@ -200,7 +165,7 @@ module.exports.create = function (app, xconfx, models) {
res.end("<!-- App bootstraping complete, but you got here somehow anyway. Let's redirect you so you get to the main app. -->");
});
});
app.use('/api', errorIfNotApi);
// NOTE Allows CORS access to API with ?access_token=
// TODO Access-Control-Max-Age: 600
// TODO How can we help apps handle this? token?
@ -208,7 +173,6 @@ module.exports.create = function (app, xconfx, models) {
app.use('/api', cors);
app.get('/api/com.daplie.walnut.init', getConfig);
app.post('/api/com.daplie.walnut.init', setConfig);
app.use('/', errorIfApi);
// TODO use package loader
//app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'com.daplie.walnut.init')));

View File

@ -9,7 +9,7 @@
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>Daplie Connect - Sign on to the Web</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; connect-src 'self' https://oauth3.org/api/; script-src 'self' https://www.google-analytics.com/; img-src https: data: blob: filesystem:; child-src 'self' https://oauth3.org/">
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; connect-src 'self' https:; script-src 'self' https://www.google-analytics.com/; img-src https: data: blob: filesystem:; child-src 'self' https://oauth3.org/">
<meta name="description" content="Sign on to the Web.">
<meta name="viewport" content="user-scalable=0, initial-scale=1.0">

View File

@ -1,19 +1,19 @@
'use strict';
module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi) {
var PromiseA = require('bluebird');
var path = require('path');
var fs = PromiseA.promisifyAll(require('fs'));
// NOTE: each process has its own cache
var localCache = { le: {}, statics: {} };
var express = require('express');
var apiApp;
var setupDomain = xconfx.setupDomain = ('cloud.' + xconfx.primaryDomain);
var apiApp;
var setupApp;
var CORS;
var cors;
function redirectSetup(reason, req, res/*, next*/) {
function redirectSetup(reason, req, res) {
console.log('xconfx', xconfx);
var url = 'https://cloud.' + xconfx.primaryDomain;
@ -194,31 +194,6 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
}
*/
// If this looks like an API, we shouldn't be here
if (/^api\./.test(req.hostname) && /\/api(\/|$)/.test(req.url)) {
// supports api.example.com/sub/app/api/com.example.xyz/
if (!apiApp) {
apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
}
if (/^OPTIONS$/i.test(req.method)) {
if (!cors) {
CORS = require('connect-cors');
cors = CORS({ credentials: true, headers: [
'X-Requested-With'
, 'X-HTTP-Method-Override'
, 'Content-Type'
, 'Accept'
, 'Authorization'
], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
}
cors(req, res, apiApp);
}
apiApp(req, res, next);
return;
}
/*
// TODO assets.example.com/sub/assets/com.example.xyz/
if (/^assets\./.test(req.hostname) && /\/assets(\/|$)/.test(req.url)) {
@ -290,6 +265,36 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
// TODO handle assets.example.com/sub/assets/com.example.xyz/
app.use('/api', function (req, res, next) {
// If this doesn't look like an API we can move along
if (!/^api\./.test(req.hostname) && !/\/api(\/|$)/.test(req.url)) {
next();
return;
}
// supports api.example.com/sub/app/api/com.example.xyz/
if (!apiApp) {
apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
}
if (/^OPTIONS$/i.test(req.method)) {
if (!cors) {
CORS = require('connect-cors');
cors = CORS({ credentials: true, headers: [
'X-Requested-With'
, 'X-HTTP-Method-Override'
, 'Content-Type'
, 'Accept'
, 'Authorization'
], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
}
cors(req, res, apiApp);
}
apiApp(req, res, next);
return;
});
app.use('/', errorIfApi);
app.use('/', serveStatic);
app.use('/', serveApps);

View File

@ -180,7 +180,7 @@ module.exports.create = function (webserver, xconfx, state) {
function setupMain() {
if (xconfx.debug) { console.log('[main] setup'); }
mainApp = express();
require('./main').create(mainApp, xconfx, apiFactories, apiDeps).then(function () {
require('./main').create(mainApp, xconfx, apiFactories, apiDeps, errorIfApi).then(function () {
if (xconfx.debug) { console.log('[main] ready'); }
// TODO process.send({});
});
@ -202,6 +202,41 @@ module.exports.create = function (webserver, xconfx, state) {
}
});
function errorIfNotApi(req, res, next) {
var hostname = req.hostname || req.headers.host;
if (!/^api\.[a-z0-9\-]+/.test(hostname)) {
res.send({ error:
{ message: "API access is restricted to proper 'api'-prefixed lowercase subdomains."
+ " The HTTP 'Host' header must exist and must begin with 'api.' as in 'api.example.com'."
+ " For development you may test with api.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
, code: 'E_NOT_API'
, _hostname: hostname
}
});
return;
}
next();
}
function errorIfApi(req, res, next) {
if (!/^api\./.test(req.headers.host)) {
next();
return;
}
// has api. hostname prefix
// doesn't have /api url prefix
if (!/^\/api\//.test(req.url)) {
res.send({ error: { message: "missing /api/ url prefix" } });
return;
}
res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
}
app.disable('x-powered-by');
app.use('/', log);
app.use('/api', require('body-parser').json({
@ -218,6 +253,7 @@ module.exports.create = function (webserver, xconfx, state) {
app.use('/api', recase);
app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal']);
app.use('/api', errorIfNotApi);
app.use('/', function (req, res) {
if (!(req.encrypted || req.secure)) {
// did not come from https