You requested an insecure resource. Please use this instead: \n' - + ' ' + safeLocation + '
\n' - + '\n' - + '\n' - ; - - // DO NOT HTTP REDIRECT - /* - res.setHeader('Location', newLocation); - res.statusCode = 302; - */ - - // BAD NEWS BEARS - // - // When people are experimenting with the API and posting tutorials - // they'll use cURL and they'll forget to prefix with https:// - // If we allow that, then many users will be sending private tokens - // and such with POSTs in clear text and, worse, it will work! - // To minimize this, we give browser users a mostly optimal experience, - // but people experimenting with the API get a message letting them know - // that they're doing it wrong and thus forces them to ensure they encrypt. - res.setHeader('Content-Type', 'text/html; charset=utf-8'); - res.end(metaRedirect); - } - - // TODO localhost-only server shutdown mechanism - // that closes all sockets, waits for them to finish, - // and then hands control over completely to respawned server - - // - // Redirect HTTP to HTTPS - // - // This simply redirects from the current insecure location to the encrypted location - // - var insecureServer; - insecureServer = http.createServer(); - insecureServer.listen(insecurePort, function () { - console.log("\nListening on http://localhost:" + insecureServer.address().port); - console.log("(handling any explicit redirects and redirecting all other traffic to https)\n"); - if (serverCallback) { - appPromise = serverCallback(null, insecureServer); - } - }); - - if (lex) { - var LEX = require('letsencrypt-express'); - insecureServer.on('request', LEX.createAcmeResponder(lex, redirectHttps)); - } else { - insecureServer.on('request', redirectHttps); - } - - return PromiseA.resolve(insecureServer); -}; diff --git a/lib/ip-checker.js b/lib/ip-checker.js new file mode 100644 index 0000000..e3b94d4 --- /dev/null +++ b/lib/ip-checker.js @@ -0,0 +1,138 @@ +"use strict"; + +var PromiseA = require('bluebird').Promise; +var ifaces = require('os').networkInterfaces(); +var dns = PromiseA.promisifyAll(require('dns')); +var https = require('https'); + +function getExternalAddresses() { + var iftypes = {}; + var ipv4check = 'api.ipify.org'; + var ipv6check = 'myexternalip.com'; + + Object.keys(ifaces).forEach(function (ifname) { + ifaces[ifname].forEach(function (iface) { + // local addresses + if (iface.internal) { + return; + } + // auto address space + if (/^(fe80:|169\.)/.test(iface.address)) { + return; + } + /* + if (/^(fe80:|10\.|192\.168|172\.1[6-9]|172\.2[0-9]|172\.3[0-1])/.test(iface.address)) { + return; + } + */ + + iftypes[iface.family] = true; + }); + }); + + console.log(iftypes); + + var now = Date.now(); + + return PromiseA.all([ + dns.lookupAsync(ipv4check, { family: 4/*, all: true*/ }).then(function (ans) { + iftypes.IPv4 = { address: ans[0], family: ans[1], time: Date.now() - now }; + }).error(function () { + //console.log('no ipv4', Date.now() - now); + iftypes.IPv4 = false; + }) + // curl -6 https://myexternalip.com/raw + , dns.lookupAsync(ipv6check, { family: 6/*, all: true*/ }).then(function (ans) { + iftypes.IPv6 = { address: ans[0], family: ans[1], time: Date.now() - now }; + }).error(function (err) { + console.error('Error ip-checker.js'); + console.error(err.stack || err); + //console.log('no ipv6', Date.now() - now); + iftypes.IPv6 = false; + }) + ]).then(function () { + var requests = []; + + if (iftypes.IPv4) { + requests.push(new PromiseA(function (resolve) { + var req = https.request({ + method: 'GET' + , hostname: iftypes.IPv4.address + , port: 443 + , headers: { + Host: ipv4check + } + , path: '/' + //, family: 4 + // TODO , localAddress: <You requested an insecure resource. Please use this instead: \n' + + ' ' + safeLocation + '
\n' + + '\n' + + '\n' + ; + + // DO NOT HTTP REDIRECT + /* + res.setHeader('Location', newLocation); + res.statusCode = 302; + */ + + // BAD NEWS BEARS + // + // When people are experimenting with the API and posting tutorials + // they'll use cURL and they'll forget to prefix with https:// + // If we allow that, then many users will be sending private tokens + // and such with POSTs in clear text and, worse, it will work! + // To minimize this, we give browser users a mostly optimal experience, + // but people experimenting with the API get a message letting them know + // that they're doing it wrong and thus forces them to ensure they encrypt. + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end(metaRedirect); + } + + function redirectHttps(req, res) { + if (localCache.le[req.hostname]) { + if (localCache.le[req.hostname].conf) { + redirectHttpsHelper(req, res); + return; + } + else { + // TODO needs IPC to expire cache + redirectSetup(req.hostname, req, res); + return; + /* + if (Date.now() - localCache.le[req.hostname].createdAt < (5 * 60 * 1000)) { + // TODO link to dbconf.primaryDomain + res.send({ error: { message: "Security Error: Encryption for '" + req.hostname + "' has not been configured." + + " Please use the management interface to set up ACME / Let's Encrypt (or another solution)." } }); + return; + } + */ + } + } + + return xconfx.walkLe(req.hostname).then(function (leAuth) { + if (!leAuth) { + redirectSetup(req.hostname, req, res); + return; + } + + localCache.le[req.hostname] = { conf: leAuth, createdAt: Date.now() }; + redirectHttps(req, res); + }); + } + + function disallowSymLinks(req, res) { + res.end( + "Symbolic Links are not supported on all platforms and are therefore disallowed." + + " Instead, simply create a file of the same name as the link with a single line of text" + + " which should be the relative or absolute path to the target directory." + ); + } + + function disallowNonFiles(req, res) { + res.end( + "Pipes, Blocks, Sockets, FIFOs, and other such nonsense are not permitted." + + " Instead please create a directory from which to read or create a file " + + " with a single line of text which should be the target directory to read from." + ); + } + + function securityError(req, res) { + res.end("Security Error: Link points outside of packages/pages"); + } + + function notConfigured(req, res, next) { + if (setupDomain !== req.hostname) { + redirectSetup(req.hostname, req, res); + return; + } + + if (!setupApp) { + setupApp = express.static(path.join(xconfx.staticpath, 'com.daplie.walnut')); + } + setupApp(req, res, function () { + if ('/' === req.url) { + res.end('Sanity Fail: Configurator not found'); + return; + } + next(); + }); + } + + function loadHandler(name) { + return function handler(req, res, next) { + var packagepath = path.join(xconfx.staticpath, name); + + return fs.lstatAsync(packagepath).then(function (stat) { + if (stat.isSymbolicLink()) { + return disallowSymLinks; + } + + if (stat.isDirectory()) { + return express.static(packagepath); + } + + if (!stat.isFile()) { + return disallowNonFiles; + } + + return fs.readFileAsync(packagepath, 'utf8').then(function (text) { + // TODO allow cascading + text = text.trim().split(/\n/)[0]; + + // TODO rerun the above, disallowing link-style (or count or memoize to prevent infinite loop) + // TODO make safe + packagepath = path.resolve(xconfx.staticpath, text); + if (0 !== packagepath.indexOf(xconfx.staticpath)) { + return securityError; + } + + return express.static(packagepath); + }); + }, function (/*err*/) { + return notConfigured; + }).then(function (handler) { + + // keep object reference intact + localCache.statics[name].handler = handler; + handler(req, res, next); + }); + }; + } + + function staticHelper(appId, opts) { + // TODO inter-process cache expirey + // TODO add to xconfx.staticpath + xconfx.staticpath = path.join(__dirname, '..', '..', 'packages', 'pages'); + return fs.readdirAsync(xconfx.staticpath).then(function (nodes) { + if (opts && opts.clear) { + localCache.statics = {}; + } + + // longest to shortest + function shortToLong(a, b) { + return b.length - a.length; + } + nodes.sort(shortToLong); + + nodes.forEach(function (name) { + if (!localCache.statics[name]) { + localCache.statics[name] = { handler: loadHandler(name), createdAt: Date.now() }; + } + }); + + // Secure Matching + // apple.com#blah# apple.com#blah# + // apple.com.us# apple.com#foo# + // apple.com# apple.com#foo# + nodes.some(function (name) { + if (0 === (name + '#').indexOf(appId + '#')) { + if (appId !== name) { + localCache.statics[appId] = localCache.statics[name]; + } + return true; + } + }); + + if (!localCache.statics[appId]) { + localCache.statics[appId] = { handler: notConfigured, createdAt: Date.now() }; + } + + localCache.staticsKeys = Object.keys(localCache.statics).sort(shortToLong); + return localCache.statics[appId]; + }); + } + + function redirectSetup(reason, req, res/*, next*/) { + var url = 'https://cloud.' + xconfx.primaryDomain; + + if (443 !== xconfx.externalPort) { + url += ':' + xconfx.externalPort; + } + + url += '#referrer=' + reason; + + res.statusCode = 302; + res.setHeader('Location', url); + res.end(); + } + + function serveStatic(req, res, next) { + // If we get this far we can be pretty confident that + // the domain was already set up because it's encrypted + var appId = req.hostname + req.url.replace(/\/+/g, '#').replace(/#$/, ''); + var appIdParts = appId.split('#'); + var appIdPart; + + if (!req.secure) { + // did not come from https + if (/\.(appcache|manifest)\b/.test(req.url)) { + require('./unbrick-appcache').unbrick(req, res); + return; + } + return redirectHttps(req, res); + } + + // TODO configuration for allowing www + if (/^www\./.test(req.hostname)) { + // NOTE: acme responder and appcache unbricker must come before scrubTheDub + if (/\.(appcache|manifest)\b/.test(req.url)) { + require('./unbrick-appcache').unbrick(req, res); + return; + } + require('./no-www').scrubTheDub(req, res); + return; + } + /* + if (!redirectives && config.redirects) { + redirectives = require('./hostname-redirects').compile(config.redirects); + } + */ + + // TODO assets.example.com/sub/assets/com.example.xyz/ + if (/^api\./.test(req.hostname) && /\/api(\/|$)/.test(req.url)) { + // supports api.example.com/sub/app/api/com.example.xyz/ + if (!apiApp) { + apiApp = require('./apis').create(xconfx, apiFactories, apiDeps); + } + apiApp(req, res, next); + return; + } + + while (appIdParts.length) { + // TODO needs IPC to expire cache + appIdPart = appIdParts.join('#'); + if (localCache.statics[appIdPart]) { + break; + } + // TODO test via staticsKeys + + appIdParts.pop(); + } + + if (!appIdPart || !localCache.statics[appIdPart]) { + return staticHelper(appId).then(function () { + localCache.statics[appId].handler(req, res, next); + }); + } + + localCache.statics[appIdPart].handler(req, res, next); + if (Date.now() - localCache.statics[appIdPart].createdAt > (5 * 60 * 1000)) { + staticHelper(appId, { clear: true }); + } + } + + app.use('/', serveStatic); + + return PromiseA.resolve(); +}; diff --git a/lib/master.js b/lib/master.js index 5b79575..916f1a6 100644 --- a/lib/master.js +++ b/lib/master.js @@ -2,18 +2,17 @@ var cluster = require('cluster'); var PromiseA = require('bluebird'); -// TODO -// var rootMasterKey; function init(conf, state) { + var newConf = {}; if (!conf.ipcKey) { - conf.ipcKey = require('crypto').randomBytes(16).toString('base64'); + conf.ipcKey = newConf.ipcKey = require('crypto').randomBytes(16).toString('base64'); } if (!conf.sqlite3Sock) { - conf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; + conf.sqlite3Sock = newConf.sqlite3Sock = '/tmp/sqlite3.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; } if (!conf.memstoreSock) { - conf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; + conf.memstoreSock = newConf.memstoreSock = '/tmp/memstore.' + require('crypto').randomBytes(4).toString('hex') + '.sock'; } try { @@ -49,15 +48,14 @@ function init(conf, state) { verbose: null , sock: conf.sqlite3Sock , ipcKey: conf.ipcKey - }) + })/*.then(function () { + var sqlite3 = require('sqlite3-cluster/client'); + return sqliet3.createClientFactory(...); + })*/ ]).then(function (args) { state.memstore = args[0]; - state.sqlstore = args[1]; - return { - conf: conf - , memstore: args[0] - , sqlstore: args[1] - }; + //state.sqlstore = args[1]; + return newConf; }); return promise; @@ -69,10 +67,10 @@ function touch(conf, state) { } // TODO if no xyz worker, start on xyz worker (unlock, for example) - return state.initialize.then(function () { + return state.initialize.then(function (newConf) { // TODO conf.locked = true|false; conf.initialized = true; - return conf; + return newConf; }); } diff --git a/lib/package-server-apis.js b/lib/package-server-apis.js new file mode 100644 index 0000000..945933b --- /dev/null +++ b/lib/package-server-apis.js @@ -0,0 +1,215 @@ +'use strict'; + +var escapeStringRegexp = require('escape-string-regexp'); +//var apiHandlers = {}; + +function getApi(conf, pkgConf, pkgDeps, packagedApi) { + var PromiseA = pkgDeps.Promise; + var path = require('path'); + var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/); + + // TODO needs some version stuff (which would also allow hot-loading of updates) + // TODO version could be tied to sha256sum + + return new PromiseA(function (resolve, reject) { + var myApp; + var ursa; + var promise; + + // TODO dynamic requires are a no-no + // can we statically generate a require-er? on each install? + // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } } + // requirer[pkgpath]() + myApp = pkgDeps.express(); + myApp.disable('x-powered-by'); + if (pkgDeps.app.get('trust proxy')) { + myApp.set('trust proxy', pkgDeps.app.get('trust proxy')); + } + if (!pkgConf.pubkey) { + /* + return ursa.createPrivateKey(pem, password, encoding); + var pem = myKey.toPrivatePem(); + return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) { + }); + */ + ursa = require('ursa'); + pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii'); + pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey(); + } + + try { + packagedApi._apipkg = require(path.join(pkgpath, 'package.json')); + packagedApi._apiname = packagedApi._apipkg.name; + if (packagedApi._apipkg.walnut) { + pkgpath += '/' + packagedApi._apipkg.walnut; + } + promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp)); + } catch(e) { + reject(e); + return; + } + + promise.then(function () { + // TODO give pub/priv pair for app and all public keys + // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp); + packagedApi._api = require('express-lazy')(); + packagedApi._api_app = myApp; + + //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps); + pkgDeps.getOauth3Controllers = + packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers; + require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps); + + // DEBUG + // + /* + packagedApi._api.use('/', function (req, res, next) { + console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url); + next(); + }); + //*/ + + // TODO fix backwards compat + + // /api/com.example.foo (no change) + packagedApi._api.use('/', packagedApi._api_app); + + // /api/com.example.foo => /api + packagedApi._api.use('/', function (req, res, next) { + var priorUrl = req.url; + req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length); + // console.log('api mangle 3:', req.url); + packagedApi._api_app(req, res, function (err) { + req.url = priorUrl; + next(err); + }); + }); + + // /api/com.example.foo => / + packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) { + // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url); + // console.log(packagedApi._api_app.toString()); + packagedApi._api_app(req, res, next); + }); + + resolve(packagedApi._api); + }, reject); + }); +} + +function loadApi(conf, pkgConf, pkgDeps, packagedApi) { + function handlePromise(p) { + return p.then(function (api) { + packagedApi._api = api; + return api; + }); + } + + if (!packagedApi._promise_api) { + packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi); + } + + return handlePromise(packagedApi._promise_api); +} + +function runApi(opts, router, req, res, next) { + var path = require('path'); + var pkgConf = opts.config; + var pkgDeps = opts.deps; + //var Services = opts.Services; + var packagedApi; + var pathname; + + // TODO compile packagesMap + // TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name) + router.packagedApis.some(function (_packagedApi) { + // console.log('[DEBUG _packagedApi.id]', _packagedApi.id); + pathname = router.pathname; + if ('/' === pathname) { + pathname = ''; + } + // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc) + if (!_packagedApi._api_re) { + _packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)'); + //console.log('[api re 2]', _packagedApi._api_re); + } + if (_packagedApi._api_re.test(req.url)) { + packagedApi = _packagedApi; + return true; + } + }); + + if (!packagedApi) { + console.log("[ODD] no api for '" + req.url + "'"); + next(); + return; + } + + // Reaching this point means that there are APIs for this pathname + // it is important to identify this host + pathname (example.com/foo) as the app + Object.defineProperty(req, 'experienceId', { + enumerable: true + , configurable: false + , writable: false + // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app + // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) + // NOTE: probably best to alias the name logically + , value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '') + }); + Object.defineProperty(req, 'escapedExperienceId', { + enumerable: true + , configurable: false + , writable: false + // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app + // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) + // NOTE: probably best to alias the name logically + , value: req.experienceId.replace(/\//g, ':') + }); + // packageId should mean hash(api.id + host + path) - also called "api" + Object.defineProperty(req, 'packageId', { + enumerable: true + , configurable: false + , writable: false + // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app + // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) + // NOTE: probably best to alias the name logically + , value: packagedApi.domain.id + }); + Object.defineProperty(req, 'appConfig', { + enumerable: true + , configurable: false + , writable: false + , value: {} // TODO just the app-scoped config + }); + Object.defineProperty(req, 'appDeps', { + enumerable: true + , configurable: false + , writable: false + , value: {} // TODO app-scoped deps + // i.e. when we need to use things such as stripe id + // without exposing them to the app + }); + + // + // TODO user authentication should go right about here + // + + // + // TODO freeze objects for passing them into app + // + + if (packagedApi._api) { + packagedApi._api(req, res, next); + return; + } + + // console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id); + loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) { + api(req, res, next); + }, function (err) { + console.error('[App Promise Error]'); + next(err); + }); +} + +module.exports.runApi = runApi; diff --git a/lib/package-server-static.js b/lib/package-server-static.js new file mode 100644 index 0000000..44997c7 --- /dev/null +++ b/lib/package-server-static.js @@ -0,0 +1,87 @@ +'use strict'; + +var staticHandlers = {}; + +function loadPages(pkgConf, packagedPage, req, res, next) { + var PromiseA = require('bluebird'); + var fs = require('fs'); + var path = require('path'); + var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || '')); + + // TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc) + + function handlePromise(p) { + p.then(function (app) { + app(req, res, next); + packagedPage._page = app; + }, function (err) { + console.error('[App Promise Error]'); + next(err); + }); + } + + if (staticHandlers[pkgpath]) { + packagedPage._page = staticHandlers[pkgpath]; + packagedPage._page(req, res, next); + return; + } + + if (!packagedPage._promise_page) { + packagedPage._promise_page = new PromiseA(function (resolve, reject) { + fs.exists(pkgpath, function (exists) { + var staticServer; + + if (!exists) { + reject(new Error("package '" + pkgpath + "' is registered but does not exist")); + return; + } + + //console.log('[static mount]', pkgpath); + // https://github.com/expressjs/serve-static/issues/54 + // https://github.com/pillarjs/send/issues/91 + // https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx + staticServer = require('serve-static')(pkgpath, { dotfiles: undefined }); + resolve(staticServer); + }); + }); + } + + handlePromise(packagedPage._promise_page); +} + +function layerItUp(pkgConf, router, req, res, next) { + var nexti = -1; + // Layers exist so that static apps can use them like a virtual filesystem + // i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html + // or perhaps some dynamic content (like application cache) + function nextify(err) { + var packagedPage; + nexti += 1; + + if (err) { + next(err); + return; + } + + // shortest to longest + //route = packages.pop(); + // longest to shortest + packagedPage = router.packagedPages[nexti]; + if (!packagedPage) { + next(); + return; + } + + if (packagedPage._page) { + packagedPage._page(req, res, nextify); + return; + } + + // could attach to req.{ pkgConf, pkgDeps, Services} + loadPages(pkgConf, packagedPage, req, res, next); + } + + nextify(); +} + +module.exports.layerItUp = layerItUp; diff --git a/lib/package-server.js b/lib/package-server.js index d1451c0..6a2488b 100644 --- a/lib/package-server.js +++ b/lib/package-server.js @@ -1,8 +1,8 @@ 'use strict'; var escapeStringRegexp = require('escape-string-regexp'); -var staticHandlers = {}; -//var apiHandlers = {}; +var runApi = require('./package-server-apis').runApi; +var layerItUp = require('./package-server-static').layerItUp; function compileVhosts(vhostsMap) { var results = { @@ -62,297 +62,6 @@ function compileVhosts(vhostsMap) { return results; } -function loadPages(pkgConf, packagedPage, req, res, next) { - var PromiseA = require('bluebird'); - var fs = require('fs'); - var path = require('path'); - var pkgpath = path.join(pkgConf.pagespath, (packagedPage.package || packagedPage.id), (packagedPage.version || '')); - - // TODO special cases for /.well_known/ and similar (oauth3.html, oauth3.json, webfinger, etc) - - function handlePromise(p) { - p.then(function (app) { - app(req, res, next); - packagedPage._page = app; - }, function (err) { - console.error('[App Promise Error]'); - next(err); - }); - } - - if (staticHandlers[pkgpath]) { - packagedPage._page = staticHandlers[pkgpath]; - packagedPage._page(req, res, next); - return; - } - - if (!packagedPage._promise_page) { - packagedPage._promise_page = new PromiseA(function (resolve, reject) { - fs.exists(pkgpath, function (exists) { - var staticServer; - - if (!exists) { - reject(new Error("package '" + pkgpath + "' is registered but does not exist")); - return; - } - - //console.log('[static mount]', pkgpath); - // https://github.com/expressjs/serve-static/issues/54 - // https://github.com/pillarjs/send/issues/91 - // https://example.com/.well-known/acme-challenge/xxxxxxxxxxxxxxx - staticServer = require('serve-static')(pkgpath, { dotfiles: undefined }); - resolve(staticServer); - }); - }); - } - - handlePromise(packagedPage._promise_page); -} - -function getApi(conf, pkgConf, pkgDeps, packagedApi) { - var PromiseA = pkgDeps.Promise; - var path = require('path'); - var pkgpath = path.join(pkgConf.apipath, packagedApi.id/*, (packagedApi.api.version || '')*/); - - // TODO needs some version stuff (which would also allow hot-loading of updates) - // TODO version could be tied to sha256sum - - return new PromiseA(function (resolve, reject) { - var myApp; - var ursa; - var promise; - - // TODO dynamic requires are a no-no - // can we statically generate a require-er? on each install? - // module.exports = { {{pkgpath}}: function () { return require({{pkgpath}}) } } - // requirer[pkgpath]() - myApp = pkgDeps.express(); - myApp.disable('x-powered-by'); - if (pkgDeps.app.get('trust proxy')) { - myApp.set('trust proxy', pkgDeps.app.get('trust proxy')); - } - if (!pkgConf.pubkey) { - /* - return ursa.createPrivateKey(pem, password, encoding); - var pem = myKey.toPrivatePem(); - return jwt.verifyAsync(token, myKey.toPublicPem(), { ignoreExpiration: false && true }).then(function (decoded) { - }); - */ - ursa = require('ursa'); - pkgConf.keypair = ursa.createPrivateKey(pkgConf.privkey, 'ascii'); - pkgConf.pubkey = ursa.createPublicKey(pkgConf.pubkey, 'ascii'); //conf.keypair.toPublicKey(); - } - - try { - packagedApi._apipkg = require(path.join(pkgpath, 'package.json')); - packagedApi._apiname = packagedApi._apipkg.name; - if (packagedApi._apipkg.walnut) { - pkgpath += '/' + packagedApi._apipkg.walnut; - } - promise = PromiseA.resolve(require(pkgpath).create(pkgConf, pkgDeps, myApp)); - } catch(e) { - reject(e); - return; - } - - promise.then(function () { - // TODO give pub/priv pair for app and all public keys - // packagedApi._api = require(pkgpath).create(pkgConf, pkgDeps, myApp); - packagedApi._api = require('express-lazy')(); - packagedApi._api_app = myApp; - - //require('./oauth3-auth').inject(conf, packagedApi._api, pkgConf, pkgDeps); - pkgDeps.getOauth3Controllers = - packagedApi._getOauth3Controllers = require('oauthcommon/example-oauthmodels').create(conf).getControllers; - require('oauthcommon').inject(packagedApi._getOauth3Controllers, packagedApi._api, pkgConf, pkgDeps); - - // DEBUG - // - /* - packagedApi._api.use('/', function (req, res, next) { - console.log('[DEBUG pkgApiApp]', req.method, req.hostname, req.url); - next(); - }); - //*/ - - // TODO fix backwards compat - - // /api/com.example.foo (no change) - packagedApi._api.use('/', packagedApi._api_app); - - // /api/com.example.foo => /api - packagedApi._api.use('/', function (req, res, next) { - var priorUrl = req.url; - req.url = '/api' + req.url.slice(('/api/' + packagedApi.id).length); - // console.log('api mangle 3:', req.url); - packagedApi._api_app(req, res, function (err) { - req.url = priorUrl; - next(err); - }); - }); - - // /api/com.example.foo => / - packagedApi._api.use('/api/' + packagedApi.id, function (req, res, next) { - // console.log('api mangle 2:', '/api/' + packagedApi.id, req.url); - // console.log(packagedApi._api_app.toString()); - packagedApi._api_app(req, res, next); - }); - - resolve(packagedApi._api); - }, reject); - }); -} - -function loadApi(conf, pkgConf, pkgDeps, packagedApi) { - function handlePromise(p) { - return p.then(function (api) { - packagedApi._api = api; - return api; - }); - } - - if (!packagedApi._promise_api) { - packagedApi._promise_api = getApi(conf, pkgConf, pkgDeps, packagedApi); - } - - return handlePromise(packagedApi._promise_api); -} - -function layerItUp(pkgConf, router, req, res, next) { - var nexti = -1; - // Layers exist so that static apps can use them like a virtual filesystem - // i.e. oauth3.html isn't in *your* app but you may use it and want it mounted at /.well-known/oauth3.html - // or perhaps some dynamic content (like application cache) - function nextify(err) { - var packagedPage; - nexti += 1; - - if (err) { - next(err); - return; - } - - // shortest to longest - //route = packages.pop(); - // longest to shortest - packagedPage = router.packagedPages[nexti]; - if (!packagedPage) { - next(); - return; - } - - if (packagedPage._page) { - packagedPage._page(req, res, nextify); - return; - } - - // could attach to req.{ pkgConf, pkgDeps, Services} - loadPages(pkgConf, packagedPage, req, res, next); - } - - nextify(); -} - -function runApi(opts, router, req, res, next) { - var path = require('path'); - var pkgConf = opts.config; - var pkgDeps = opts.deps; - //var Services = opts.Services; - var packagedApi; - var pathname; - - // TODO compile packagesMap - // TODO people may want to use the framework in a non-framework way (i.e. to conceal the module name) - router.packagedApis.some(function (_packagedApi) { - // console.log('[DEBUG _packagedApi.id]', _packagedApi.id); - pathname = router.pathname; - if ('/' === pathname) { - pathname = ''; - } - // TODO allow for special apis that do not follow convention (.well_known, webfinger, oauth3.html, etc) - if (!_packagedApi._api_re) { - _packagedApi._api_re = new RegExp(escapeStringRegexp(pathname + '/api/' + _packagedApi.id) + '\/([\\w\\.\\-]+)(\\/|\\?|$)'); - //console.log('[api re 2]', _packagedApi._api_re); - } - if (_packagedApi._api_re.test(req.url)) { - packagedApi = _packagedApi; - return true; - } - }); - - if (!packagedApi) { - console.log("[ODD] no api for '" + req.url + "'"); - next(); - return; - } - - // Reaching this point means that there are APIs for this pathname - // it is important to identify this host + pathname (example.com/foo) as the app - Object.defineProperty(req, 'experienceId', { - enumerable: true - , configurable: false - , writable: false - // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app - // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) - // NOTE: probably best to alias the name logically - , value: (path.join(req.hostname, pathname || '')).replace(/\/$/, '') - }); - Object.defineProperty(req, 'escapedExperienceId', { - enumerable: true - , configurable: false - , writable: false - // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app - // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) - // NOTE: probably best to alias the name logically - , value: req.experienceId.replace(/\//g, ':') - }); - // packageId should mean hash(api.id + host + path) - also called "api" - Object.defineProperty(req, 'packageId', { - enumerable: true - , configurable: false - , writable: false - // TODO this identifier may need to be non-deterministic as to transfer if a domain name changes but is still the "same" app - // (i.e. a company name change. maybe auto vs manual register - just like oauth3?) - // NOTE: probably best to alias the name logically - , value: packagedApi.domain.id - }); - Object.defineProperty(req, 'appConfig', { - enumerable: true - , configurable: false - , writable: false - , value: {} // TODO just the app-scoped config - }); - Object.defineProperty(req, 'appDeps', { - enumerable: true - , configurable: false - , writable: false - , value: {} // TODO app-scoped deps - // i.e. when we need to use things such as stripe id - // without exposing them to the app - }); - - // - // TODO user authentication should go right about here - // - - // - // TODO freeze objects for passing them into app - // - - if (packagedApi._api) { - packagedApi._api(req, res, next); - return; - } - - // console.log("[DEBUG pkgpath]", pkgConf.apipath, packagedApi.id); - loadApi(opts.conf, pkgConf, pkgDeps, packagedApi).then(function (api) { - api(req, res, next); - }, function (err) { - console.error('[App Promise Error]'); - next(err); - }); -} - function mapToApp(opts, req, res, next) { // opts = { config, deps, services } var vhost; @@ -450,6 +159,5 @@ function mapToApp(opts, req, res, next) { return runApi(opts, router, req, res, next); } -module.exports.runApi = runApi; module.exports.compileVhosts = compileVhosts; module.exports.mapToApp = mapToApp; diff --git a/lib/worker.js b/lib/worker.js index dea2221..08cc48c 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -1,137 +1,50 @@ 'use strict'; -module.exports.create = function (webserver, conf, state) { +module.exports.create = function (webserver, xconfx, state) { + console.log('DEBUG create worker'); + if (!state) { state = {}; } var PromiseA = state.Promise || require('bluebird'); - var path = require('path'); - //var vhostsdir = path.join(__dirname, 'vhosts'); - var express = require('express-lazy'); - var app = express(); var memstore; var sqlstores = {}; - var models = {}; var systemFactory = require('sqlite3-cluster/client').createClientFactory({ - dirname: path.join(__dirname, '..', '..', 'var') // TODO conf - , prefix: 'com.example.' + dirname: xconfx.varpath + , prefix: 'com.daplie.walnut.' //, dbname: 'config' , suffix: '' , ext: '.sqlite3' - , sock: conf.sqlite3Sock - , ipcKey: conf.ipcKey + , sock: xconfx.sqlite3Sock + , ipcKey: xconfx.ipcKey }); + /* var clientFactory = require('sqlite3-cluster/client').createClientFactory({ algorithm: 'aes' , bits: 128 , mode: 'cbc' - , dirname: path.join(__dirname, '..', '..', 'var') // TODO conf - , prefix: 'com.example.' + , dirname: xconfx.varpath // TODO conf + , prefix: 'com.daplie.walnut.' //, dbname: 'cluster' , suffix: '' , ext: '.sqlcipher' - , sock: conf.sqlite3Sock - , ipcKey: conf.ipcKey + , sock: xconfx.sqlite3Sock + , ipcKey: xconfx.ipcKey }); - var cstore = require('cluster-store'); - var redirectives; - - app.disable('x-powered-by'); - if (conf.trustProxy) { - console.info('[Trust Proxy]'); - app.set('trust proxy', ['loopback']); - //app.set('trust proxy', function (ip) { console.log('[ip]', ip); return true; }); - } else { - console.info('[DO NOT trust proxy]'); - // TODO make sure the gzip module loads if there isn't a proxy gzip-ing for us - // app.use(compression()) - } - - /* - function unlockDevice(conf, state) { - return require('./lib/unlock-device').create().then(function (result) { - result.promise.then(function (_rootMasterKey) { - process.send({ - type: 'walnut.keys.root' - conf: { - rootMasterKey: _rootMasterkey - } - }); - conf.locked = false; - if (state.caddy) { - state.caddy.update(conf); - } - conf.rootMasterKey = _rootMasterKey; - }); - - return result.app; - }); - } */ - - // TODO handle insecure to actual redirect - // blog.coolaj86.com -> coolaj86.com/blog - // hmm... that won't really matter with hsts - // I guess I just needs letsencrypt - - function scrubTheDub(req, res, next) { - var host = req.hostname; - - if (!host || 'string' !== typeof host) { - next(); - return; - } - - // TODO test if this is even necessary - host = host.toLowerCase(); - - // TODO this should be hot loadable / changeable - if (!redirectives && conf.redirects) { - redirectives = require('./hostname-redirects').compile(conf.redirects); - } - - if (!/^www\./.test(host) && !redirectives) { - next(); - return; - } - - // TODO misnomer, handles all exact redirects - if (!require('./no-www').scrubTheDub(req, res, redirectives)) { - next(); - return; - } - } - - function caddyBugfix(req, res, next) { - // workaround for Caddy - // https://github.com/mholt/caddy/issues/341 - if (app.get('trust proxy')) { - if (req.headers['x-forwarded-proto']) { - req.headers['x-forwarded-proto'] = (req.headers['x-forwarded-proto'] || '').split(/,\s+/g)[0] || undefined; - } - if (req.headers['x-forwarded-host']) { - req.headers['x-forwarded-host'] = (req.headers['x-forwarded-host'] || '').split(/,\s+/g)[0] || undefined; - } - } - - next(); - } - - // TODO misnomer, this can handle nowww, yeswww, and exact hostname redirects - app.use('/', scrubTheDub); - app.use('/', caddyBugfix); + var cstore = require('cluster-store'); return PromiseA.all([ // TODO security on memstore // TODO memstoreFactory.create cstore.create({ - sock: conf.memstoreSock - , connect: conf.memstoreSock + sock: xconfx.memstoreSock + , connect: xconfx.memstoreSock // TODO implement - , key: conf.ipcKey + , key: xconfx.ipcKey }).then(function (_memstore) { - memstore = _memstore; + memstore = PromiseA.promisifyAll(_memstore); return memstore; }) // TODO mark a device as lost, stolen, missing in DNS records @@ -140,101 +53,127 @@ module.exports.create = function (webserver, conf, state) { init: true , dbname: 'config' }) - , clientFactory.create({ - init: true - , key: '00000000000000000000000000000000' - // TODO only complain if the values are different - //, algo: 'aes' - , dbname: 'auth' - }) - , clientFactory.create({ - init: false - , dbname: 'system' - }) ]).then(function (args) { memstore = args[0]; sqlstores.config = args[1]; - sqlstores.auth = args[2]; - sqlstores.system = args[3]; - sqlstores.create = clientFactory.create; - return require('../lib/schemes-config').create(sqlstores.config).then(function (tables) { - models.Config = tables; - return models.Config.Config.get().then(function (vhostsMap) { - // TODO the core needs to be replacable in one shot - // rm -rf /tmp/walnut/; tar xvf -C /tmp/walnut/; mv /srv/walnut /srv/walnut.{{version}}; mv /tmp/walnut /srv/ - // this means that any packages must be outside, perhaps /srv/walnut/{boot,core,packages} - var pkgConf = { - pagespath: path.join(__dirname, '..', '..', 'packages', 'pages') + path.sep - , apipath: path.join(__dirname, '..', '..', 'packages', 'apis') + path.sep - , servicespath: path.join(__dirname, '..', '..', 'packages', 'services') - , vhostsMap: vhostsMap - , vhostPatterns: null - , server: webserver - , externalPort: conf.externalPort - , privkey: conf.privkey - , pubkey: conf.pubkey - , redirects: conf.redirects - , apiPrefix: '/api' - , 'org.oauth3.consumer': conf['org.oauth3.consumer'] - , 'org.oauth3.provider': conf['org.oauth3.provider'] - , keys: conf.keys - }; - var pkgDeps = { - memstore: memstore - , sqlstores: sqlstores - , clientSqlFactory: clientFactory - , systemSqlFactory: systemFactory - //, handlePromise: require('./lib/common').promisableRequest; - //, handleRejection: require('./lib/common').rejectableRequest; - //, localPort: conf.localPort - , Promise: PromiseA - , express: express - , app: app - //, oauthmodels: require('oauthcommon/example-oauthmodels').create(conf) - }; - var Services = require('./services-loader').create(pkgConf, { - memstore: memstore - , sqlstores: sqlstores - , clientSqlFactory: clientFactory - , systemSqlFactory: systemFactory - , Promise: PromiseA - }); - var recase = require('connect-recase')({ - // TODO allow explicit and or default flag - explicit: false - , default: 'snake' - , prefixes: ['/api'] - // TODO allow exclude - //, exclusions: [config.oauthPrefix] - , exceptions: {} - //, cancelParam: 'camel' - }); + var wrap = require('masterquest-sqlite3'); + var dir = [ + { tablename: 'com_daplie_walnut_config' + , idname: 'id' + , unique: [ 'id' ] + , indices: [ 'createdAt', 'updatedAt' ] + } + , { tablename: 'com_daplie_walnut_redirects' + , idname: 'id' // blog.example.com:sample.net/blog + , unique: [ 'id' ] + , indices: [ 'createdAt', 'updatedAt' ] + } + ]; - function handlePackages(req, res, next) { - // TODO move to caddy parser? - if (/(^|\.)proxyable\./.test(req.hostname)) { - // device-id-12345678.proxyable.myapp.mydomain.com => myapp.mydomain.com - // proxyable.myapp.mydomain.com => myapp.mydomain.com - // TODO myapp.mydomain.com.example.proxyable.com => myapp.mydomain.com - req.hostname = req.hostname.replace(/.*\.?proxyable\./, ''); - } - - require('./package-server').mapToApp({ - config: pkgConf - , deps: pkgDeps - , services: Services - , conf: conf - }, req, res, next); + function scopeMemstore(expId) { + var scope = expId + '|'; + return { + getAsync: function (id) { + return memstore.getAsync(scope + id); + } + , setAsync: function (id, data) { + return memstore.setAsync(scope + id, data); + } + , touchAsync: function (id, data) { + return memstore.touchAsync(scope + id, data); + } + , destroyAsync: function (id) { + return memstore.destroyAsync(scope + id); } - // TODO recase + // helpers + , allAsync: function () { + return memstore.allASync().then(function (db) { + return Object.keys(db).filter(function (key) { + return 0 === key.indexOf(scope); + }).map(function (key) { + return db[key]; + }); + }); + } + , lengthAsync: function () { + return memstore.allASync().then(function (db) { + return Object.keys(db).filter(function (key) { + return 0 === key.indexOf(scope); + }).length; + }); + } + , clearAsync: function () { + return memstore.allASync().then(function (db) { + return Object.keys(db).filter(function (key) { + return 0 === key.indexOf(scope); + }).map(function (key) { + return memstore.destroyAsync(key); + }); + }).then(function () { + return null; + }); + } + }; + } - // - // Generic Template API - // - app - .use('/api', require('body-parser').json({ + return wrap.wrap(sqlstores.config, dir).then(function (models) { + return models.ComDaplieWalnutConfig.find(null, { limit: 100 }).then(function (results) { + return models.ComDaplieWalnutConfig.find(null, { limit: 10000 }).then(function (redirects) { + var express = require('express-lazy'); + var app = express(); + var recase = require('connect-recase')({ + // TODO allow explicit and or default flag + explicit: false + , default: 'snake' + , prefixes: ['/api'] + // TODO allow exclude + //, exclusions: [config.oauthPrefix] + , exceptions: {} + //, cancelParam: 'camel' + }); + var bootstrapApp; + var mainApp; + var apiDeps = { + models: models + // TODO don't let packages use this directly + , Promise: PromiseA + }; + var apiFactories = { + memstoreFactory: { create: scopeMemstore } + , systemSqlFactory: systemFactory + }; + + function log(req, res, next) { + console.log('[worker/log]', req.method, req.headers.host, req.url); + next(); + } + + function setupMain() { + mainApp = express(); + require('./main').create(mainApp, xconfx, apiFactories, apiDeps).then(function () { + // TODO process.send({}); + }); + } + + if (!bootstrapApp) { + bootstrapApp = express(); + require('./bootstrap').create(bootstrapApp, xconfx, models).then(function () { + // TODO process.send({}); + setupMain(); + }); + } + + process.on('message', function (data) { + if ('com.daplie.walnut.bootstrap' === data.type) { + setupMain(); + } + }); + + app.disable('x-powered-by'); + app.use('/', log); + app.use('/api', require('body-parser').json({ strict: true // only objects and arrays , inflate: true // limited to due performance issues with JSON.parse and JSON.stringify @@ -244,38 +183,40 @@ module.exports.create = function (webserver, conf, state) { , reviver: undefined , type: 'json' , verify: undefined - })) - // DO NOT allow urlencoded at any point, it is expressly forbidden - //.use(require('body-parser').urlencoded({ - // extended: true - //, inflate: true - //, limit: 100 * 1024 - //, type: 'urlencoded' - //, verify: undefined - //})) - .use(require('connect-send-error').error()) - ; + })); + app.use('/api', recase); - app.use('/api', recase); + app.use('/', function (req, res) { + if (!req.secure) { + // did not come from https + if (/\.(appcache|manifest)\b/.test(req.url)) { + require('./unbrick-appcache').unbrick(req, res); + return; + } + } - app.use('/', handlePackages); - app.use('/', function (err, req, res, next) { - console.error('[Error Handler]'); - console.error(err.stack); - if (req.xhr) { - res.send({ error: { message: "kinda unknownish error" } }); - } else { - res.send('