diff --git a/master.js b/boot/master.js similarity index 77% rename from master.js rename to boot/master.js index 0664fda..d2d2a05 100644 --- a/master.js +++ b/boot/master.js @@ -3,15 +3,15 @@ // TODO if RAM is very low we should not fork at all, // but use a different process altogether -console.log('pid:', process.pid); -console.log('title:', process.title); -console.log('arch:', process.arch); -console.log('platform:', process.platform); -console.log('\n\n\n[MASTER] Welcome to WALNUT!'); +console.info('pid:', process.pid); +console.info('title:', process.title); +console.info('arch:', process.arch); +console.info('platform:', process.platform); +console.info('\n\n\n[MASTER] Welcome to WALNUT!'); var cluster = require('cluster'); var path = require('path'); -var minWorkers = 2; +//var minWorkers = 2; var numCores = 1; // Math.max(minWorkers, require('os').cpus().length); var workers = []; var caddypath = '/usr/local/bin/caddy'; @@ -23,8 +23,8 @@ var conf = { // TODO externalInsecurePort? , locked: false // TODO XXX , ipcKey: null -, caddyfilepath: path.join(__dirname, 'Caddyfile') -, sitespath: path.join(__dirname, 'sites-enabled') +, caddyfilepath: path.join(__dirname, '..', 'Caddyfile') +, sitespath: path.join(__dirname, '..', 'sites-enabled') }; var state = {}; var caddy; @@ -45,9 +45,10 @@ cluster.on('online', function (worker) { var certPaths = [path.join(__dirname, 'certs', 'live')]; var info; - console.log('[MASTER] Worker ' + worker.process.pid + ' is online'); + console.info('[MASTER] Worker ' + worker.process.pid + ' is online'); fork(); + // TODO communicate config with environment vars? info = { type: 'com.daplie.walnut.init' , conf: { @@ -72,7 +73,7 @@ cluster.on('online', function (worker) { // calls init if init has not been called state.caddy = caddy; state.workers = workers; - require('./lib/master').touch(conf, state).then(function () { + require('../lib/master').touch(conf, state).then(function () { info.type = 'com.daplie.walnut.webserver.onrequest'; info.conf.ipcKey = conf.ipcKey; info.conf.memstoreSock = conf.memstoreSock; @@ -84,7 +85,7 @@ cluster.on('online', function (worker) { }); cluster.on('exit', function (worker, code, signal) { - console.log('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal); + console.info('[MASTER] Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal); workers = workers.map(function (w) { if (worker !== w) { @@ -102,7 +103,7 @@ cluster.on('exit', function (worker, code, signal) { fork(); if (useCaddy) { - caddy = require('./lib/spawn-caddy').create(conf); + caddy = require('../lib/spawn-caddy').create(conf); // relies on { localPort, locked } caddy.spawn(conf); } diff --git a/boot/worker.js b/boot/worker.js new file mode 100644 index 0000000..146d931 --- /dev/null +++ b/boot/worker.js @@ -0,0 +1,113 @@ +'use strict'; + +module.exports.create = function (opts) { + var id = '0'; + + function createAndBindServers(message, cb) { + var msg = message.conf; + + require('../lib/local-server').create(msg.certPaths, msg.localPort, function (err, webserver) { + if (err) { + console.error('[ERROR] worker.js'); + console.error(err.stack); + throw err; + } + + console.info("#" + id + " Listening on " + msg.protocol + "://" + webserver.address().address + ":" + webserver.address().port, '\n'); + + return cb(webserver); + }); + + // TODO conditional if 80 is being served by caddy + require('../lib/insecure-server').create(msg.externalPort, msg.insecurePort); + } + + // + // Worker Mode + // + function waitForConfig(message) { + if ('com.daplie.walnut.init' !== message.type) { + console.warn('[Worker] 0 got unexpected message:'); + console.warn(message); + return; + } + + process.removeListener('message', waitForConfig); + + // NOTE: this callback must return a promise for an express app + createAndBindServers(message, function (webserver) { + var PromiseA = require('bluebird'); + return new PromiseA(function (resolve) { + function initWebServer(srvmsg) { + if ('com.daplie.walnut.webserver.onrequest' !== srvmsg.type) { + console.warn('[Worker] 1 got unexpected message:'); + console.warn(srvmsg); + return; + } + + process.removeListener('message', initWebServer); + + resolve(require('../lib/worker').create(webserver, srvmsg)); + } + + process.send({ type: 'com.daplie.walnut.webserver.listening' }); + process.on('message', initWebServer); + }).then(function (app) { + console.info('[Worker Ready]'); + return app; + }); + }); + } + + // + // Standalone Mode + // + if (opts) { + // NOTE: this callback must return a promise for an express app + createAndBindServers(opts, function (webserver) { + var PromiseA = require('bluebird'); + return new PromiseA(function (resolve) { + opts.getConfig(function (srvmsg) { + resolve(require('../lib/worker').create(webserver, srvmsg)); + }); + }).then(function (app) { + console.info('[Standalone Ready]'); + return app; + }); + }); + } else { + // we are in cluster mode, as opposed to standalone mode + id = require('cluster').worker.id.toString(); + // We have to wait to get the configuration from the master process + // before we can start our webserver + console.info('[Worker #' + id + '] online!'); + process.on('message', waitForConfig); + } + + // + // Debugging + // + process.on('exit', function (code) { + // only sync code can run here + console.info('uptime:', process.uptime()); + console.info(process.memoryUsage()); + console.info('[exit] process.exit() has been called (or master has killed us).'); + console.info(code); + }); + process.on('beforeExit', function () { + // async can be scheduled here + console.info('[beforeExit] Event Loop is empty. Process will end.'); + }); + process.on('unhandledRejection', function (err) { + // this should always throw + // (it means somewhere we're not using bluebird by accident) + console.error('[caught] [unhandledRejection]'); + console.error(Object.keys(err)); + console.error(err); + console.error(err.stack); + }); + process.on('rejectionHandled', function (msg) { + console.error('[rejectionHandled]'); + console.error(msg); + }); +}; diff --git a/etc/init/caddy.conf b/etc/init/caddy.conf new file mode 100644 index 0000000..4535e67 --- /dev/null +++ b/etc/init/caddy.conf @@ -0,0 +1,23 @@ +# sudo rsync -av etc/init/caddy.conf /etc/init/caddy.conf + +description "Caddy Server" +version "1.0" +author "AJ ONeal" + +# Upstart has nothing in $PATH by default +env PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Keep the server running on crash or machine reboot +respawn +respawn limit 10 120 +start on runlevel [2345] + +# Start the server using spark and redirect output to log files +script + DATE=`date '+%F_%H-%M-%S'` + cd /srv/walnut + # exec /usr/local/bin/caddy -conf /srv/walnut/Caddyfile -pidfile /tmp/caddy.pid \ + exec start-stop-daemon --start --pidfile /tmp/caddy.pid --exec /usr/local/bin/caddy -- -conf /srv/walnut/Caddyfile --pidfile /tmp/caddy.pid \ + > "./logs/access.caddy.${DATE}.log" \ + 2> "./logs/error.caddy.${DATE}.log" +end script diff --git a/lib/api-server.js b/lib/api-server.js index 3dfca8a..50ae29c 100644 --- a/lib/api-server.js +++ b/lib/api-server.js @@ -14,17 +14,13 @@ module.exports.create = function (conf, deps, app) { var PromiseA = require('bluebird'); var path = require('path'); - console.log(route); // TODO needs some version stuff (which would also allow hot-loading of updates) // TODO version could be tied to sha256sum var pkgpath = path.join(conf.apipath, (route.api.package || route.api.id), (route.api.version || '')); - console.log('pkgpath'); - console.log(pkgpath); - return new PromiseA(function (resolve, reject) { try { - route.route = require(pkgpath).create(conf, deps.app, app); + route.route = require(pkgpath).create(conf, deps, app); } catch(e) { reject(e); return; @@ -37,9 +33,6 @@ module.exports.create = function (conf, deps, app) { function api(req, res, next) { var apps; - console.log('hostname', req.hostname); - console.log('headers', req.headers); - if (!vhostsMap[req.hostname]) { // TODO keep track of match-only vhosts, such as '*.example.com', // separate from exact matches @@ -71,14 +64,10 @@ module.exports.create = function (conf, deps, app) { }); if (!apps) { - console.log('No apps to try for this hostname'); - console.log(vhostsMap[req.hostname]); next(); return; } - //console.log(apps); - function nextify(err) { var route; diff --git a/lib/schemes-config.js b/lib/schemes-config.js index aa3772b..a261831 100644 --- a/lib/schemes-config.js +++ b/lib/schemes-config.js @@ -109,8 +109,6 @@ function getVhostsMap(config) { vhosts.sort(sortApps); vhosts.forEach(function (domain) { - console.log(domain.hostname, domain.pathname, domain.dirname); - if (!vhostsMap[domain.hostname]) { vhostsMap[domain.hostname] = { pathnamesMap: {}, pathnames: [] }; } @@ -129,7 +127,6 @@ function getVhostsMap(config) { module.exports.deserialize = deserialize; module.exports.getVhostsMap = getVhostsMap; module.exports.create = function (db) { - console.log('[DB -1]'); var wrap = require('dbwrap'); var dir = [ diff --git a/lib/vhost-server.js b/lib/vhost-server.js deleted file mode 100644 index cf26ae6..0000000 --- a/lib/vhost-server.js +++ /dev/null @@ -1,343 +0,0 @@ -'use strict'; - -module.exports.create = function (securePort, vhostsdir) { - var PromiseA = require('bluebird').Promise; - var serveStatic; - var fs = require('fs'); - var path = require('path'); - var dummyCerts; - var loopbackToken = require('crypto').randomBytes(32).toString('hex'); - - function handleAppScopedError(tag, domaininfo, req, res, fn) { - function next(err) { - if (!err) { - fn(req, res); - return; - } - - if (res.headersSent) { - console.error('[ERROR] handleAppScopedError headersSent'); - console.log(err); - console.log(err.stack); - return; - } - - console.error('[ERROR] handleAppScopedError'); - console.log(err); - console.log(err.stack); - - res.writeHead(500); - res.end( - "" - + "
" - + '' - + "" - + "" - + ""
- + ""
- + "Method: " + encodeURI(req.method)
- + '\n'
- + "Hostname: " + encodeURI(domaininfo.hostname)
- + '\n'
- + "App: " + encodeURI(domaininfo.pathname ? (domaininfo.pathname + '/') : '')
- + '\n'
- + "Route: " + encodeURI(req.url)//.replace(/^\//, '')
- + '\n'
- // TODO better sanatization
- + 'Error: ' + (err.message || err.toString()).replace(/"
- + "
"
- + ""
- + ""
- );
- }
-
- return next;
- }
-
- function createPromiseApps(secureServer) {
- return new PromiseA(function (resolve) {
- var forEachAsync = require('foreachasync').forEachAsync.create(PromiseA);
- var connect = require('connect');
- // TODO make lazy
- var app = connect().use(require('compression')());
- var vhost = require('vhost');
-
- var domainMergeMap = {};
- var domainMerged = [];
-
- function getDomainInfo(apppath) {
- var parts = apppath.split(/[#%]+/);
- var hostname = parts.shift();
- var pathname = parts.join('/').replace(/\/+/g, '/').replace(/^\//, '');
-
- return {
- hostname: hostname
- , pathname: pathname
- , dirpathname: parts.join('#')
- , dirname: apppath
- , isRoot: apppath === hostname
- };
- }
-
- function loadDomainMounts(domaininfo) {
- var connectContext = {};
- var appContext;
-
- // should order and group by longest domain, then longest path
- if (!domainMergeMap[domaininfo.hostname]) {
- // create an connect / express app exclusive to this domain
- // TODO express??
- domainMergeMap[domaininfo.hostname] = {
- hostname: domaininfo.hostname
- , apps: connect()
- , mountsMap: {}
- };
- domainMerged.push(domainMergeMap[domaininfo.hostname]);
- }
-
- if (domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname]) {
- return;
- }
-
- console.log('[log] [once] Preparing mount for', domaininfo.hostname + '/' + domaininfo.dirpathname);
- domainMergeMap[domaininfo.hostname].mountsMap['/' + domaininfo.dirpathname] = function (req, res, next) {
- res.setHeader('Strict-Transport-Security', 'max-age=10886400; includeSubDomains; preload');
- function loadThatApp() {
- var time = Date.now();
-
- console.log('[log] LOADING "' + domaininfo.hostname + '/' + domaininfo.pathname + '"', req.url);
- return getAppContext(domaininfo).then(function (localApp) {
- console.info((Date.now() - time) + 'ms Loaded ' + domaininfo.hostname + ':' + securePort + '/' + domaininfo.pathname);
- //if (localApp.arity >= 2) { /* connect uses .apply(null, arguments)*/ }
- if ('function' !== typeof localApp) {
- localApp = getDummyAppContext(null, "[ERROR] no connect-style export from " + domaininfo.dirname);
- }
-
- function fourohfour(req, res) {
- res.writeHead(404);
- res.end(
- ""
- + ""
- + ''
- + ""
- + ""
- + "Cannot "
- + encodeURI(req.method)
- + " 'https://"
- + encodeURI(domaininfo.hostname)
- + '/'
- + encodeURI(domaininfo.pathname ? (domaininfo.pathname + '/') : '')
- + encodeURI(req.url.replace(/^\//, ''))
- + "'"
- + "