Merge branch 'v1' of git.daplie.com:Daplie/Goldilocks.js into v1

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "packages/assets/org.oauth3"]
-  path = packages/assets/org.oauth3
-  url = git@git.daplie.com:OAuth3/oauth3.js.git

bin/goldilocks.js

@@ -137,6 +137,8 @@ function readConfigAndRun(args) {
   config.addresses = addresses;
   config.device = { hostname: 'daplien-pod' };
 
+  config.tunnel = args.tunnel || config.tunnel;
+
   var PromiseA = require('bluebird');
   var tcpProm, dnsProm;
 

etc/goldilocks/goldilocks.example.yml

@@ -69,7 +69,7 @@ http:
     - name: proxy
       domains:
         - localhost.daplie.me
-      host: locahost
+      host: localhost
       port: 4000
     - name: static
       domains:

lib/app.js

@@ -15,7 +15,6 @@ module.exports = function (myDeps, conf, overrideHttp) {
   //var server;
   var serveInit;
   var app;
-  var tun;
   var request;
 
   /*
@@ -43,7 +42,6 @@ module.exports = function (myDeps, conf, overrideHttp) {
 
   function createServeInit() {
     var PromiseA = require('bluebird');
-    var stunnel = require('stunnel');
     var OAUTH3 = require('../packages/assets/org.oauth3');
     require('../packages/assets/org.oauth3/oauth3.domains.js');
     require('../packages/assets/org.oauth3/oauth3.dns.js');
@@ -96,7 +94,13 @@ module.exports = function (myDeps, conf, overrideHttp) {
         obj.id = id;
         owners[id] = obj;
 
-        return fs.writeFileAsync(ownersPath, JSON.stringify(owners), 'utf8');
+        return fs.mkdirAsync(path.dirname(ownersPath)).catch(function (err) {
+          if (err.code !== 'EEXIST') {
+            console.error('failed to mkdir', path.dirname(ownersPath), err.toString());
+          }
+        }).then(function () {
+          return fs.writeFileAsync(ownersPath, JSON.stringify(owners), 'utf8');
+        });
       }
     };
 
@@ -143,8 +147,7 @@ module.exports = function (myDeps, conf, overrideHttp) {
           providerUri: providerUri
         , session: session
         });
-        //var crypto = require('crypto');
+
-        //var id = crypto.createHash('sha256').update(session.token.sub).digest('hex');
         return oauth3.setProvider(providerUri).then(function () {
           /*
           return oauth3.api('domains.list').then(function (domains) {
@@ -173,55 +176,13 @@ module.exports = function (myDeps, conf, overrideHttp) {
               }
             }).then(function (result) {
               console.log('got a token from the tunnel server?');
-              console.log(result);
+              result.owner = session.id;
-              if (!result.tunnelUrl) {
+              return deps.tunneler.add(result);
-                result.tunnelUrl = ('wss://' + (new Buffer(result.jwt.split('.')[1], 'base64').toString('ascii')).aud + '/');
-              }
-              var services = { https: { '*': 443 }, http: { '*': 80 }, smtp: { '*': 25}, smtps: { '*': 587 /*also 465/starttls*/ } /*, ssh: { '*': 22 }*/ };
-              /*
-              console.log('blah');
-              console.log(result.jwt);
-              console.log(result.tunnelUrl);
-              console.log(services);
-              console.log('deps.tunnel');
-              console.log(deps.tunnel);
-              console.log('deps.tunnel.net');
-              console.log(deps.tunnel.net.toString());
-              console.log('deps.net');
-              console.log(deps.net);
-              */
-              var opts3 = {
-                token: result.jwt
-              , stunneld: result.tunnelUrl
-                // we'll provide faux networking and pipe as we please
-              , services: services
-              , net: myDeps.tunnel.net
-              };
-
-              console.log('blah 2');
-              if (tun) {
-                console.log('balh 3');
-                if (tun.append) {
-                  tun.append(result.jwt);
-                }
-                else if (tun.end) {
-                  tun.end();
-                  tun = null;
-                }
-              }
-
-              console.log('might have tunnel?');
-              if (!tun) {
-                console.log('connecting to the tunnel');
-                tun = stunnel.connect(opts3);
-                conf.tun = true;
-              }
             });
           /*
           });
           */
         });
-        //, { token: token, refresh: refresh });
       }
     };
 

lib/goldilocks.js

@@ -68,7 +68,13 @@ module.exports.create = function (deps, config) {
       return;
     }
 
+    function onError(err) {
+      console.error('[error] socket errored peeking -', err);
+      conn.destroy();
+    }
+    conn.once('error', onError);
     conn.once('data', function (chunk) {
+      conn.removeListener('error', onError);
       peek(conn, chunk, opts);
     });
   }
@@ -184,6 +190,7 @@ module.exports.create = function (deps, config) {
       return writer;
     }
   };
+  deps.tunneler = require('./tunnel-manager').create(deps, config);
 
   var listenPromises = [];
   var tcpPortMap = {};

lib/mdns.js

@@ -2,7 +2,8 @@
 
 var PromiseA = require('bluebird');
 var fs = PromiseA.promisifyAll(require('fs'));
-var idFilename = require('path').join(__dirname, '..', 'var', 'mdns-id');
+var path = require('path');
+var idFilename = path.join(__dirname, '..', 'var', 'mdns-id');
 var queryName = '_cloud._tcp.local';
 
 var randomId = {
@@ -18,10 +19,15 @@ var randomId = {
   }
 
 , set: function (value) {
-    return fs.writeFileAsync(idFilename, value)
+    return fs.mkdirAsync(path.dirname(idFilename)).catch(function (err) {
-      .then(function () {
+      if (err.code !== 'EEXIST') {
+        console.error('failed to mkdir', path.dirname(idFilename), err.toString());
+      }
+    }).then(function () {
+      return fs.writeFileAsync(idFilename, value).then(function () {
         return value;
       });
+    });
   }
 };
 
@@ -104,6 +110,7 @@ function createResponse(name, packet, ttl) {
 module.exports.start = function (deps, config) {
   var socket = require('dgram').createSocket({ type: 'udp4', reuseAddr: true });
   var dns = require('dns-suite');
+  var nextBroadcast = -1;
 
   socket.on('message', function (message, rinfo) {
     // console.log('Received %d bytes from %s:%d', message.length, rinfo.address, rinfo.port);
@@ -130,7 +137,13 @@ module.exports.start = function (deps, config) {
 
     randomId.get().then(function (name) {
       var resp = createResponse(name, packet, config.mdns.ttl);
-      socket.send(resp, config.mdns.port, config.mdns.broadcast);
+      var now = Date.now();
+      if (now > nextBroadcast) {
+        socket.send(resp, config.mdns.port, config.mdns.broadcast);
+        nextBroadcast = now + config.mdns.ttl * 1000;
+      } else {
+        socket.send(resp, rinfo.port, rinfo.address);
+      }
     });
   });
 

lib/modules/admin.js

@@ -1,3 +1,12 @@
+var adminDomains = [
+  'localhost.alpha.daplie.me'
+, 'localhost.admin.daplie.me'
+, 'alpha.localhost.daplie.me'
+, 'admin.localhost.daplie.me'
+, 'localhost.daplie.invalid'
+];
+module.exports.adminDomains = adminDomains;
+
 module.exports.create = function (deps, conf) {
   'use strict';
 
@@ -43,20 +52,14 @@ module.exports.create = function (deps, conf) {
       */
     ]
   };
-  opts.sites.push({
+  adminDomains.forEach(function (id) {
-    // greenlock: {}
+    opts.sites.push({
-    $id: 'localhost.alpha.daplie.me'
+      $id: id
-  , paths: [
+    , paths: [
-      { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] }
+        { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] }
-    , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] }
+      , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] }
-    ]
+      ]
-  });
+    });
-  opts.sites.push({
-    $id: 'localhost.daplie.invalid'
-  , paths: [
-      { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] }
-    , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] }
-    ]
   });
 
   var app = require('../app.js')(deps, conf, opts);

lib/modules/http.js

@@ -6,13 +6,6 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
   var domainMatches = require('../domain-utils').match;
   var separatePort = require('../domain-utils').separatePort;
 
-  var adminDomains = [
-    /\blocalhost\.admin\./
-  , /\blocalhost\.alpha\./
-  , /\badmin\.localhost\./
-  , /\balpha\.localhost\./
-  ];
-
   function parseHeaders(conn, opts) {
     // There should already be a `firstChunk` on the opts, but because we might sometimes
     // need more than that to get all the headers it's easier to always read the data off
@@ -51,7 +44,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
       var result = {};
 
       lines.slice(1).forEach(function (line) {
-        var match = /(.*)\s*:\s*(.*)/.exec(line);
+        var match = /([^:]*?)\s*:\s*(.*)/.exec(line);
         if (match) {
           result[match[1].toLowerCase()] = match[2];
         } else {
@@ -186,8 +179,9 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
 
   var adminServer;
   function checkAdmin(conn, opts, headers) {
-    var admin = adminDomains.some(function (re) {
+    var host = separatePort(headers.host).host;
-      return re.test(headers.host);
+    var admin = require('./admin').adminDomains.some(function (domain) {
+      return host === domain;
     });
 
     if (admin) {
@@ -199,7 +193,50 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     return false;
   }
 
-  function checkProxy(mod, conn, opts, headers) {
+  var proxyServer;
+  function createProxyServer() {
+    var http = require('http');
+    var agent = new http.Agent();
+    agent.createConnection = deps.net.createConnection;
+
+    var proxy = require('http-proxy').createProxyServer({
+      agent: agent,
+      toProxy: true
+    });
+
+    proxyServer = http.createServer(function (req, res) {
+      proxy.web(req, res, req.connection.proxyOpts);
+    });
+    proxyServer.on('upgrade', function (req, socket, head) {
+      proxy.ws(req, socket, head, socket.proxyOpts);
+    });
+  }
+  function proxyRequest(mod, conn, opts, headers) {
+    if (!proxyServer) {
+      createProxyServer();
+    }
+
+    var xHeaders = {};
+    // Then add our own `X-Forwarded` headers at the end.
+    if (conf.http.trustProxy && headers['x-forwarded-proto']) {
+      xHeaders['X-Forwarded-Proto'] = headers['x-forwarded-proto'];
+    } else {
+      xHeaders['X-Forwarded-Proto'] = conn.encrypted ? 'https' : 'http';
+    }
+    var proxyChain = (headers['x-forwarded-for'] || '').split(/ *, */).filter(Boolean);
+    proxyChain.push(opts.remoteAddress || opts.address || conn.remoteAddress);
+    xHeaders['X-Forwarded-For'] = proxyChain.join(', ');
+    xHeaders['X-Forwarded-Host'] = headers.host;
+
+    conn.proxyOpts = {
+      target: 'http://'+(mod.address || (mod.host || 'localhost')+':'+mod.port),
+      headers: xHeaders
+    };
+    proxyServer.emit('connection', conn);
+    conn.unshift(opts.firstChunk);
+  }
+
+  function proxyWebsocket(mod, conn, opts, headers) {
     var index = opts.firstChunk.indexOf('\r\n\r\n');
     var body = opts.firstChunk.slice(index);
 
@@ -213,7 +250,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     if (conf.http.trustProxy && headers['x-forwarded-proto']) {
       headLines.push('X-Forwarded-Proto: ' + headers['x-forwarded-proto']);
     } else {
-      headLines.push('X-Forwarded-Proto: ' + conn.encrypted ? 'https' : 'http');
+      headLines.push('X-Forwarded-Proto: ' + (conn.encrypted ? 'https' : 'http'));
     }
     var proxyChain = (headers['x-forwarded-for'] || '').split(/ *, */).filter(Boolean);
     proxyChain.push(opts.remoteAddress || opts.address || conn.remoteAddress);
@@ -235,6 +272,14 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     newConnOpts.remotePort    = opts.port    || conn.remotePort;
 
     deps.proxy(conn, newConnOpts, opts.firstChunk);
+  }
+
+  function checkProxy(mod, conn, opts, headers) {
+    if ((headers.connection || '').toLowerCase() === 'upgrade') {
+      proxyWebsocket(mod, conn, opts, headers);
+    } else {
+      proxyRequest(mod, conn, opts, headers);
+    }
     return true;
   }
 

lib/modules/tls.js

@@ -36,40 +36,17 @@ module.exports.create = function (deps, config, netHandler) {
   , 'localPort'
   ];
   function wrapSocket(socket, opts) {
-    var reader = require('socket-pair').create(function (err, writer) {
+    if (!opts.hyperPeek) {
-      if (err) {
-        reader.emit('error', err);
-        return;
-      }
-
       process.nextTick(function () {
         socket.unshift(opts.firstChunk);
       });
+    }
 
-      socket.pipe(writer);
+    var wrapped = require('tunnel-packer').wrapSocket(socket);
-      writer.pipe(socket);
-
-      socket.on('error', function (err) {
-        console.log('wrapped TLS socket error', err);
-        reader.emit('error', err);
-      });
-      writer.on('error', function (err) {
-        console.error('socket-pair writer error', err);
-        // If the writer had an error the reader probably did too, and I don't think we'll
-        // get much out of emitting this on the original socket, so logging is enough.
-      });
-    });
-
-    // We can't set these properties the normal way because there is a getter without a setter,
-    // but we can use defineProperty. We reuse the descriptor even though we will be manipulating
-    // it because we will only ever set the value and we set it every time.
-    var descriptor = {enumerable: true, configurable: true, writable: true};
     addressNames.forEach(function (name) {
-      descriptor.value = opts[name] || extractSocketProp(socket, name);
+      wrapped[name] = opts[name] || wrapped[name];
-      Object.defineProperty(reader, name, descriptor);
     });
-
+    return wrapped;
-    return reader;
   }
 
   var le = greenlock.create({
@@ -206,7 +183,7 @@ module.exports.create = function (deps, config, netHandler) {
   };
 
   var terminateServer = tls.createServer(terminatorOpts, function (socket) {
-    console.log('(pre-terminated) tls connection, addr:', socket.remoteAddress);
+    console.log('(post-terminated) tls connection, addr:', extractSocketProp(socket, 'remoteAddress'));
 
     netHandler(socket, {
       servername: socket.servername
@@ -217,6 +194,9 @@ module.exports.create = function (deps, config, netHandler) {
     , remoteFamily:  extractSocketProp(socket, 'remoteFamily')
     });
   });
+  terminateServer.on('error', function (err) {
+    console.log('[error] TLS termination server', err);
+  });
 
   function proxy(socket, opts, mod) {
     var newConnOpts = require('../domain-utils').separatePort(mod.address || '');

lib/proxy-conn.js

@@ -34,6 +34,7 @@ module.exports.sendBadGateway = sendBadGateway;
 module.exports.create = function (deps, config) {
   return function proxy(conn, newConnOpts, firstChunk, decrypt) {
     var connected = false;
+    newConnOpts.allowHalfOpen = true;
     var newConn = deps.net.createConnection(newConnOpts, function () {
       connected = true;
 

lib/servers.js

@@ -29,7 +29,7 @@ module.exports.addTcpListener = function (port, handler) {
     var enableDestroy = require('server-destroy');
     var net = require('net');
     var resolved;
-    var server = net.createServer();
+    var server = net.createServer({allowHalfOpen: true});
 
     stat = serversMap[port] = {
       server: server

lib/tunnel-manager.js

@@ -0,0 +1,161 @@
+'use strict';
+
+module.exports.create = function (deps, config) {
+  var PromiseA = require('bluebird');
+  var fs = PromiseA.promisifyAll(require('fs'));
+  var stunnel = require('stunnel');
+  var activeTunnels = {};
+
+  var path = require('path');
+  var tokensPath = path.join(__dirname, '..', 'var', 'tokens.json');
+  var storage = {
+    _read: function () {
+      var tokens;
+      try {
+        tokens = require(tokensPath);
+      } catch (err) {
+        tokens = {};
+      }
+      return tokens;
+    }
+  , _write: function (tokens) {
+      return fs.mkdirAsync(path.dirname(tokensPath)).catch(function (err) {
+        if (err.code !== 'EEXIST') {
+          console.error('failed to mkdir', path.dirname(tokensPath), err.toString());
+        }
+      }).then(function () {
+        return fs.writeFileAsync(tokensPath, JSON.stringify(tokens), 'utf8');
+      });
+    }
+
+  , all: function () {
+      var tokens = storage._read();
+      return PromiseA.resolve(Object.keys(tokens).map(function (key) {
+        return tokens[key];
+      }));
+    }
+  , save: function (result) {
+      var tokens = storage._read();
+      tokens[result.jwt] = result;
+      storage._write(tokens);
+    }
+  , del: function (id) {
+      var tokens = storage._read();
+      delete tokens[id];
+      storage._write(tokens);
+    }
+  };
+
+  function addToken(data) {
+    if (!data.tunnelUrl) {
+      var decoded;
+      try {
+        decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii'));
+      } catch (err) {
+        console.warn('invalid web token given to tunnel manager', err);
+        return PromiseA.reject(err);
+      }
+      if (!decoded.aud) {
+        console.warn('tunnel manager given token with no tunnelUrl or audience');
+        var err = new Error('missing tunnelUrl and audience');
+        return PromiseA.reject(err);
+      }
+      data.tunnelUrl = 'wss://' + decoded.aud + '/';
+    }
+
+    if (!activeTunnels[data.tunnelUrl]) {
+      console.log('creating new tunnel client for', data.tunnelUrl);
+      // We create the tunnel without an initial token so we can append the token and
+      // get the promise that should tell us more about if it worked or not.
+      activeTunnels[data.tunnelUrl] = stunnel.connect({
+        stunneld: data.tunnelUrl
+      , net: deps.tunnel.net
+        // NOTE: the ports here aren't that important since we are providing a custom
+        // `net.createConnection` that doesn't actually use the port. What is important
+        // is that any services we are interested in are listed in this object and have
+        // a '*' sub-property.
+      , services: {
+          https: { '*': 443 }
+        , http:  { '*': 80 }
+        , smtp:  { '*': 25 }
+        , smtps: { '*': 587 /*also 465/starttls*/ }
+        , ssh:   { '*': 22 }
+        }
+      });
+    }
+
+    console.log('appending token to tunnel at', data.tunnelUrl);
+    return activeTunnels[data.tunnelUrl].append(data.jwt);
+  }
+
+  function removeToken(data) {
+    if (!data.tunnelUrl) {
+      var decoded;
+      try {
+        decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii'));
+      } catch (err) {
+        console.warn('invalid web token given to tunnel manager', err);
+        return PromiseA.reject(err);
+      }
+      if (!decoded.aud) {
+        console.warn('tunnel manager given token with no tunnelUrl or audience');
+        var err = new Error('missing tunnelUrl and audience');
+        return PromiseA.reject(err);
+      }
+      data.tunnelUrl = 'wss://' + decoded.aud + '/';
+    }
+
+    // Not sure if we actually want to return an error that the token didn't even belong to a
+    // server that existed, but since it never existed we can consider it as "removed".
+    if (!activeTunnels[data.tunnelUrl]) {
+      return PromiseA.resolve();
+    }
+
+    console.log('removing token from tunnel at', data.tunnelUrl);
+    return activeTunnels[data.tunnelUrl].clear(data.jwt);
+  }
+
+  if (typeof config.tunnel === 'string') {
+    config.tunnel.split(',').forEach(function (jwt) {
+      addToken({ jwt: jwt, owner: 'config' });
+    });
+  }
+  storage.all().then(function (stored) {
+    stored.forEach(function (result) {
+      addToken(result);
+    });
+  });
+
+  return {
+    add: function (data) {
+      return addToken(data).then(function () {
+        return storage.save(data);
+      });
+    }
+  , remove: function (data) {
+      return storage.del(data.jwt).then(function () {
+        return removeToken(data);
+      });
+    }
+  , get: function (owner) {
+      return storage.all().then(function (tokens) {
+        var result = {};
+        tokens.forEach(function (data) {
+          if (!result[data.owner]) {
+            result[data.owner] = {};
+          }
+          if (!result[data.owner][data.tunnelUrl]) {
+            result[data.owner][data.tunnelUrl] = [];
+          }
+          data.decoded = JSON.parse(new Buffer(data.jwt.split('.')[0], 'base64'));
+          result[data.owner][data.tunnelUrl].push(data);
+        });
+
+        if (owner) {
+          return result[owner] || {};
+        }
+        return result;
+      });
+    }
+  };
+};

package.json

@@ -45,6 +45,7 @@
     "express": "git+https://github.com/expressjs/express.git#4.x",
     "finalhandler": "^0.4.0",
     "greenlock": "git+https://git.daplie.com/Daplie/node-greenlock.git#master",
+    "http-proxy": "^1.16.2",
     "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0",
     "ipify": "^1.1.0",
     "js-yaml": "^3.8.3",
@@ -62,8 +63,9 @@
     "serve-static": "^1.10.0",
     "server-destroy": "^1.0.1",
     "sni": "^1.0.0",
-    "socket-pair": "^1.0.0",
+    "socket-pair": "^1.0.1",
     "stream-pair": "^1.0.3",
-    "stunnel": "git+https://git.daplie.com/Daplie/node-tunnel-client.git#v1"
+    "stunnel": "git+https://git.daplie.com/Daplie/node-tunnel-client.git#v1",
+    "tunnel-packer": "^1.3.0"
   }
 }

packages/apis/com.daplie.goldilocks/index.js

@@ -118,6 +118,15 @@ module.exports.create = function (deps, conf) {
     }
   , tunnel: function (req, res) {
       isAuthorized(req, res, function () {
+        if ('POST' !== req.method) {
+          res.setHeader('Content-Type', 'application/json');
+          return deps.tunneler.get(req.userId).then(function (result) {
+            res.end(JSON.stringify(result));
+          }, function (err) {
+            res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } }));
+          });
+        }
+
         jsonParser(req, res, function () {
 
           console.log('req.body', req.body);
@@ -153,8 +162,8 @@ module.exports.create = function (deps, conf) {
       });
     }
   , request: function (req, res) {
-      jsonParser(req, res, function () {
       isAuthorized(req, res, function () {
+      jsonParser(req, res, function () {
 
         deps.request({
           method: req.body.method || 'GET'

packages/assets/org.oauth3

@@ -1 +0,0 @@
-Subproject commit c4cc61992805469f86ecd4d74afc18cf6506155b