WIP load apis by file structure

install.sh

@@ -218,7 +218,7 @@ install_my_app()
   ln -sf ../node_modules /srv/walnut/core/node_modules
   sudo mkdir -p /srv/walnut/etc/org.oauth3.consumer
   sudo mkdir -p /srv/walnut/etc/org.oauth3.provider
-  sudo mkdir -p /srv/walnut/packages/{api,pages,services}
+  sudo mkdir -p /srv/walnut/packages/{client-api-grants,rest,api,pages,services}
   #sudo chown -R $(whoami):$(whoami) /srv/walnut
   sudo chown -R www-data:www-data /srv/walnut
   sudo chmod -R ug+Xrw /srv/walnut

lib/apis.js

@@ -5,163 +5,54 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
   var express = require('express');
   var fs = PromiseA.promisifyAll(require('fs'));
   var path = require('path');
-  var localCache = { apis: {}, pkgs: {} };
+  var localCache = { rests: {}, pkgs: {} };
 
   // TODO xconfx.apispath
-  xconfx.apispath = path.join(__dirname, '..', '..', 'packages', 'apis');
+  xconfx.restPath = path.join(__dirname, '..', '..', 'packages', 'rest');
+  xconfx.appApiGrantsPath = path.join(__dirname, '..', '..', 'packages', 'client-api-grants');
 
   function notConfigured(req, res) {
-    res.send({ error: { message: "api '" + req.apiId + "' not configured for domain '" + req.experienceId + "'" } });
+    res.send({ error: { message: "api '" + req.pkgId + "' not configured for domain '" + req.experienceId + "'" } });
   }
 
-  function loadApi(conf, pkgConf, pkgDeps, packagedApi) {
-    function handlePromise(p) {
-      return p.then(function (api) {
-        packagedApi._api = api;
-        return api;
-      });
-    }
-
-    if (!packagedApi._promise_api) {
-      packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi);
-    }
-
-    return handlePromise(packagedApi._promise_api);
-  }
-
-  function getApi(conf, pkgConf, pkgDeps, packagedApi) {
-    var PromiseA = pkgDeps.Promise;
-    var path = require('path');
-    var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/);
-
-    // TODO needs some version stuff (which would also allow hot-loading of updates)
-    // TODO version could be tied to sha256sum
-
-    return new PromiseA(function (resolve, reject) {
-      var myApp;
-      var ursa;
-      var promise;
-
-      // TODO dynamic requires are a no-no
-      // can we statically generate a require-er? on each install?
-      // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } }
-      // requirer[pkgpath]()
-      myApp = pkgDeps.express();
-      myApp.disable('x-powered-by');
-      if (pkgDeps.app.get('trust proxy')) {
-        myApp.set('trust proxy', pkgDeps.app.get('trust proxy'));
-      }
-      if (!pkgConf.pubkey) {
   /*
-          return ursa.createPrivateKey(pem, password, encoding);
+  function isThisPkgInstalled(myConf, pkgId) {
-          var pem = myKey.toPrivatePem();
+  }
-          return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) {
-          });
   */
-        ursa = require('ursa');
-        pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii');
-        pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey();
-      }
 
-      try {
+  function isThisClientAllowedToUseThisPkg(myConf, clientUrih, pkgId) {
-        packagedApi._apipkg = require(path.join(pkgpath, 'package.json'));
+    var appApiGrantsPath = path.join(myConf.appApiGrantsPath, clientUrih);
-        packagedApi._apiname = packagedApi._apipkg.name;
-        if (packagedApi._apipkg.walnut) {
-          pkgpath += '/' + packagedApi._apipkg.walnut;
-        }
-        promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp));
-      } catch(e) {
-        reject(e);
-        return;
-      }
 
-      promise.then(function () {
+    return fs.readFileAsync(appApiGrantsPath, 'utf8').then(function (text) {
-        // TODO give pub/priv pair for app and all public keys
-        // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp);
-        packagedApi._api = require('express-lazy')();
-        packagedApi._api_app = myApp;
-
-        //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps);
-        pkgDeps.getOauth3Controllers =
-        packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers;
-        require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps);
-
-        // DEBUG
-        //
-        /*
-        packagedApi._api.use('/', function (req, res, next) {
-          console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url);
-          next();
-        });
-        //*/
-
-        // TODO fix backwards compat
-
-        // /api/com.example.foo (no change)
-        packagedApi._api.use('/', packagedApi._api_app);
-
-        // /api/com.example.foo => /api
-        packagedApi._api.use('/', function (req, res, next) {
-          var priorUrl = req.url;
-          req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length);
-          // console.log('api mangle 3:', req.url);
-          packagedApi._api_app(req, res, function (err) {
-            req.url = priorUrl;
-            next(err);
-          });
-        });
-
-        // /api/com.example.foo => /
-        packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) {
-          // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url);
-          // console.log(packagedApi._api_app.toString());
-          packagedApi._api_app(req, res, next);
-        });
-
-        resolve(packagedApi._api);
-      }, reject);
-    });
-  }
-
-  // Read packages/apis/sub.sld.tld (forward dns) to find list of apis as tld.sld.sub (reverse dns)
-  // TODO packages/allowed_apis/sub.sld.tld (?)
-  // TODO auto-register org.oauth3.consumer for primaryDomain (and all sites?)
-  function loadApiHandler() {
-    return function handler(req, res, next) {
-      var name = req.experienceId;
-      var apiId = req.apiId;
-      var packagepath = path.join(xconfx.apispath, name);
-
-      return fs.readFileAsync(packagepath, 'utf8').then(function (text) {
       return text.trim().split(/\n/);
-      }, function () {
+    }, function (/*err*/) {
       return [];
     }).then(function (apis) {
-        return function (req, res, next) {
-          var apipath;
-
       if (!apis.some(function (api) {
-            if (api === apiId) {
+        if (api === pkgId) {
           return true;
         }
       })) {
-            if (req.experienceId === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === apiId) {
+        if (clientUrih === ('api.' + xconfx.setupDomain) && 'org.oauth3.consumer' === pkgId) {
           // fallthrough
+          return true;
         } else {
           return null;
         }
       }
+    });
+  }
 
-          apipath = path.join(xconfx.apispath, apiId);
+  function loadRestHelper(myConf, pkgId) {
+    var pkgPath = path.join(myConf.restPath, pkgId);
 
-          if (!localCache.pkgs[apiId]) {
+    return fs.readFileAsync(path.join(pkgPath, 'package.json'), 'utf8').then(function (text) {
-            return fs.readFileAsync(path.join(apipath, 'package.json'), 'utf8').then(function (text) {
       var pkg = JSON.parse(text);
       var deps = {};
       var myApp;
 
       if (pkg.walnut) {
-                apipath = path.join(apipath, pkg.walnut);
+        pkgPath = path.join(pkgPath, pkg.walnut);
       }
 
       Object.keys(apiDeps).forEach(function (key) {
@@ -174,50 +65,49 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
       // TODO pull db stuff from package.json somehow and pass allowed data models as deps
       //
       // how can we tell which of these would be correct?
-              // deps.memstore = apiFactories.memstoreFactory.create(apiId);
+      // deps.memstore = apiFactories.memstoreFactory.create(pkgId);
       // deps.memstore = apiFactories.memstoreFactory.create(req.experienceId);
-              // deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + apiId);
+      // deps.memstore = apiFactories.memstoreFactory.create(req.experienceId + pkgId);
 
       // let's go with this one for now and the api can choose to scope or not to scope
-              deps.memstore = apiFactories.memstoreFactory.create(apiId);
+      deps.memstore = apiFactories.memstoreFactory.create(pkgId);
 
-              console.log('DEBUG apipath', apipath);
+      console.log('DEBUG pkgPath', pkgPath);
       myApp = express();
       //
       // TODO handle /accounts/:accountId
       //
-              return PromiseA.resolve(require(apipath).create({
+      return PromiseA.resolve(require(pkgPath).create({
         etcpath: xconfx.etcpath
       }/*pkgConf*/, deps/*pkgDeps*/, myApp/*myApp*/)).then(function (handler) {
-                localCache.pkgs[apiId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
+        localCache.pkgs[pkgId] = { pkg: pkg, handler: handler || myApp, createdAt: Date.now() };
-                localCache.pkgs[apiId].handler(req, res, next);
+        return localCache.pkgs[pkgId];
       });
     });
   }
-          else {
+
-            localCache.pkgs[apiId].handler(req, res, next);
+  // Read packages/apis/sub.sld.tld (forward dns) to find list of apis as tld.sld.sub (reverse dns)
+  // TODO packages/allowed_apis/sub.sld.tld (?)
+  // TODO auto-register org.oauth3.consumer for primaryDomain (and all sites?)
+  function loadRestHandler(myConf, pkgId) {
+    return PromiseA.resolve().then(function () {
+      if (!localCache.pkgs[pkgId]) {
+        return loadRestHelper(myConf, pkgId);
+      }
+
+      return localCache.pkgs[pkgId];
       // TODO expire require cache
       /*
-            if (Date.now() - localCache.pkgs[apiId].createdAt < (5 * 60 * 1000)) {
+      if (Date.now() - localCache.pkgs[pkgId].createdAt < (5 * 60 * 1000)) {
         return;
       }
       */
-          }
-        };
     }, function (/*err*/) {
+      // TODO what kind of errors might we want to handle?
       return null;
-      }).then(function (handler) {
+    }).then(function (restPkg) {
-
+      return restPkg;
-        // keep object reference intact
-        // DO NOT cache non-existant api
-        if (handler) {
-          localCache.apis[name].handler = handler;
-        } else {
-          handler = notConfigured;
-        }
-        handler(req, res, next);
     });
-    };
   }
 
   var CORS = require('connect-cors');
@@ -228,36 +118,78 @@ module.exports.create = function (xconfx, apiFactories, apiDeps) {
   , 'Accept'
   , 'Authorization'
   ], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
+  var staleAfter = (5 * 60 * 1000);
 
   return function (req, res, next) {
     cors(req, res, function () {
-      var experienceId = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
+      var clientUrih = req.hostname + req.url.replace(/\/api\/.*/, '/').replace(/\/+/g, '#').replace(/#$/, '');
-      var apiId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
+      var pkgId = req.url.replace(/.*\/api\//, '').replace(/\/.*/, '');
+      var now = Date.now();
+      var hasBeenHandled = false;
 
+      // Existing (Deprecated)
       Object.defineProperty(req, 'experienceId', {
         enumerable: true
       , configurable: false
+      , writable: false
+      , value: clientUrih
+      });
+      Object.defineProperty(req, 'pkgId', {
+        enumerable: true
+      , configurable: false
+      , writable: false
+      , value: pkgId
+      });
+
+      // New
+      Object.defineProperty(req, 'clientUrih', {
+        enumerable: true
+      , configurable: false
       , writable: false
         // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app
         // (i.e. a company name change. maybe auto vs manual register - just like oauth3?)
         // NOTE: probably best to alias the name logically
-      , value: experienceId
+      , value: clientUrih
       });
-      Object.defineProperty(req, 'apiId', {
+      Object.defineProperty(req, 'pkgId', {
         enumerable: true
       , configurable: false
       , writable: false
-      , value: apiId
+      , value: pkgId
       });
 
-      if (!localCache.apis[experienceId]) {
+      // TODO cache permission (although the FS is already cached, NBD)
-        localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
+      return isThisClientAllowedToUseThisPkg(xconfx, clientUrih, pkgId).then(function (yes) {
+        if (!yes) {
+          notConfigured(req, res);
+          return null;
         }
 
-      localCache.apis[experienceId].handler(req, res, next);
+        if (localCache.rests[pkgId]) {
-      if (Date.now() - localCache.apis[experienceId].createdAt > (5 * 60 * 1000)) {
+          localCache.rests[pkgId].handler(req, res, next);
-        localCache.apis[experienceId] = { handler: loadApiHandler(experienceId), createdAt: Date.now() };
+          hasBeenHandled = true;
         }
+
+        if (now - localCache.rests[pkgId].createdAt > staleAfter) {
+          localCache.rests[pkgId] = null;
+        }
+
+        if (!localCache.rests[pkgId]) {
+          //return doesThisPkgExist
+
+          return loadRestHandler(xconfx, pkgId).then(function (myHandler) {
+            if (!myHandler) {
+              notConfigured(req, res);
+              return;
+            }
+
+            localCache.rests[pkgId] = { handler: myHandler.handle, createdAt: now };
+            if (!hasBeenHandled) {
+              myHandler.handle(req, res, next);
+            }
+          });
+        }
+      });
     });
   };
 };

lib/bootstrap.js

@@ -40,41 +40,6 @@ module.exports.create = function (app, xconfx, models) {
     var getIpAddresses = require('./ip-checker').getExternalAddresses;
     var resolveInit;
 
-    function errorIfNotApi(req, res, next) {
-      var hostname = req.hostname || req.headers.host;
-
-      if (!/^api\.[a-z0-9\-]+/.test(hostname)) {
-        res.send({ error:
-         { message: "API access is restricted to proper 'api'-prefixed lowercase subdomains."
-             + " The HTTP 'Host' header must exist and must begin with 'api.' as in 'api.example.com'."
-             + " For development you may test with api.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
-         , code: 'E_NOT_API'
-         , _hostname: hostname
-         }
-        });
-        return;
-      }
-
-      next();
-    }
-
-    function errorIfApi(req, res, next) {
-      if (!/^api\./.test(req.headers.host)) {
-        next();
-        return;
-      }
-
-      // has api. hostname prefix
-
-      // doesn't have /api url prefix
-      if (!/^\/api\//.test(req.url)) {
-        res.send({ error: { message: "missing /api/ url prefix" } });
-        return;
-      }
-
-      res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
-    }
-
     function getConfig(req, res) {
       getIpAddresses().then(function (inets) {
         var results = {
@@ -200,7 +165,7 @@ module.exports.create = function (app, xconfx, models) {
         res.end("<!-- App bootstraping complete, but you got here somehow anyway. Let's redirect you so you get to the main app. -->");
       });
     });
-    app.use('/api', errorIfNotApi);
+
     // NOTE Allows CORS access to API with ?access_token=
     // TODO Access-Control-Max-Age: 600
     // TODO How can we help apps handle this? token?
@@ -208,7 +173,6 @@ module.exports.create = function (app, xconfx, models) {
     app.use('/api', cors);
     app.get('/api/com.daplie.walnut.init', getConfig);
     app.post('/api/com.daplie.walnut.init', setConfig);
-    app.use('/', errorIfApi);
 
     // TODO use package loader
     //app.use('/', express.static(path.join(__dirname, '..', '..', 'packages', 'pages', 'com.daplie.walnut.init')));

lib/com.daplie.walnut/index.html

@@ -9,7 +9,7 @@
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 
     <title>Daplie Connect - Sign on to the Web</title>
-    <meta http-equiv="Content-Security-Policy" content="default-src 'self'; connect-src 'self' https://oauth3.org/api/; script-src 'self' https://www.google-analytics.com/; img-src https: data: blob: filesystem:; child-src 'self' https://oauth3.org/">
+    <meta http-equiv="Content-Security-Policy" content="default-src 'self'; connect-src 'self' https:; script-src 'self' https://www.google-analytics.com/; img-src https: data: blob: filesystem:; child-src 'self' https://oauth3.org/">
     <meta name="description" content="Sign on to the Web.">
     <meta name="viewport" content="user-scalable=0, initial-scale=1.0">
 

lib/main.js

@@ -1,19 +1,19 @@
 'use strict';
 
-module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
+module.exports.create = function (app, xconfx, apiFactories, apiDeps, errorIfApi) {
   var PromiseA = require('bluebird');
   var path = require('path');
   var fs = PromiseA.promisifyAll(require('fs'));
   // NOTE: each process has its own cache
   var localCache = { le: {}, statics: {} };
   var express = require('express');
-  var apiApp;
   var setupDomain = xconfx.setupDomain = ('cloud.' + xconfx.primaryDomain);
+  var apiApp;
   var setupApp;
   var CORS;
   var cors;
 
-  function redirectSetup(reason, req, res/*, next*/) {
+  function redirectSetup(reason, req, res) {
     console.log('xconfx', xconfx);
     var url = 'https://cloud.' + xconfx.primaryDomain;
 
@@ -194,31 +194,6 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
     }
     */
 
-    // If this looks like an API, we shouldn't be here
-    if (/^api\./.test(req.hostname) && /\/api(\/|$)/.test(req.url)) {
-      // supports api.example.com/sub/app/api/com.example.xyz/
-      if (!apiApp) {
-        apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
-      }
-
-      if (/^OPTIONS$/i.test(req.method)) {
-        if (!cors) {
-          CORS = require('connect-cors');
-          cors = CORS({ credentials: true, headers: [
-            'X-Requested-With'
-          , 'X-HTTP-Method-Override'
-          , 'Content-Type'
-          , 'Accept'
-          , 'Authorization'
-          ], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
-        }
-        cors(req, res, apiApp);
-      }
-
-      apiApp(req, res, next);
-      return;
-    }
-
     /*
     // TODO assets.example.com/sub/assets/com.example.xyz/
     if (/^assets\./.test(req.hostname) && /\/assets(\/|$)/.test(req.url)) {
@@ -290,6 +265,36 @@ module.exports.create = function (app, xconfx, apiFactories, apiDeps) {
 
   // TODO handle assets.example.com/sub/assets/com.example.xyz/
 
+  app.use('/api', function (req, res, next) {
+    // If this doesn't look like an API we can move along
+    if (!/^api\./.test(req.hostname) && !/\/api(\/|$)/.test(req.url)) {
+      next();
+      return;
+    }
+
+    // supports api.example.com/sub/app/api/com.example.xyz/
+    if (!apiApp) {
+      apiApp = require('./apis').create(xconfx, apiFactories, apiDeps);
+    }
+
+    if (/^OPTIONS$/i.test(req.method)) {
+      if (!cors) {
+        CORS = require('connect-cors');
+        cors = CORS({ credentials: true, headers: [
+          'X-Requested-With'
+        , 'X-HTTP-Method-Override'
+        , 'Content-Type'
+        , 'Accept'
+        , 'Authorization'
+        ], methods: [ "GET", "POST", "PATCH", "PUT", "DELETE" ] });
+      }
+      cors(req, res, apiApp);
+    }
+
+    apiApp(req, res, next);
+    return;
+  });
+  app.use('/', errorIfApi);
   app.use('/', serveStatic);
   app.use('/', serveApps);
 

lib/worker.js

@@ -180,7 +180,7 @@ module.exports.create = function (webserver, xconfx, state) {
           function setupMain() {
             if (xconfx.debug) { console.log('[main] setup'); }
             mainApp = express();
-            require('./main').create(mainApp, xconfx, apiFactories, apiDeps).then(function () {
+            require('./main').create(mainApp, xconfx, apiFactories, apiDeps, errorIfApi).then(function () {
               if (xconfx.debug) { console.log('[main] ready'); }
               // TODO process.send({});
             });
@@ -202,6 +202,41 @@ module.exports.create = function (webserver, xconfx, state) {
             }
           });
 
+          function errorIfNotApi(req, res, next) {
+            var hostname = req.hostname || req.headers.host;
+
+            if (!/^api\.[a-z0-9\-]+/.test(hostname)) {
+              res.send({ error:
+               { message: "API access is restricted to proper 'api'-prefixed lowercase subdomains."
+                   + " The HTTP 'Host' header must exist and must begin with 'api.' as in 'api.example.com'."
+                   + " For development you may test with api.localhost.daplie.me (or any domain by modifying your /etc/hosts)"
+               , code: 'E_NOT_API'
+               , _hostname: hostname
+               }
+              });
+              return;
+            }
+
+            next();
+          }
+
+          function errorIfApi(req, res, next) {
+            if (!/^api\./.test(req.headers.host)) {
+              next();
+              return;
+            }
+
+            // has api. hostname prefix
+
+            // doesn't have /api url prefix
+            if (!/^\/api\//.test(req.url)) {
+              res.send({ error: { message: "missing /api/ url prefix" } });
+              return;
+            }
+
+            res.send({ error: { code: 'E_NO_IMPL', message: "not implemented" } });
+          }
+
           app.disable('x-powered-by');
           app.use('/', log);
           app.use('/api', require('body-parser').json({
@@ -218,6 +253,7 @@ module.exports.create = function (webserver, xconfx, state) {
           app.use('/api', recase);
 
           app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal']);
+          app.use('/api', errorIfNotApi);
           app.use('/', function (req, res) {
             if (!(req.encrypted || req.secure)) {
               // did not come from https