Merge branch 'loopback'

# Conflicts:
#	lib/worker.js
#	packages/apis/com.daplie.goldilocks/index.js

bin/goldilocks.js

@@ -251,7 +251,7 @@ function run(args) {
   var cachedConfig;
 
   cluster.on('message', function (worker, message) {
-    if (message.type !== 'com.daplie.goldilocks.config-change') {
+    if (message.type !== 'com.daplie.goldilocks/config') {
       return;
     }
     configStorage.save(message.changes)

lib/app.js

@@ -13,11 +13,11 @@ module.exports = function (myDeps, conf, overrideHttp) {
   var serveIndexMap = {};
   var content = conf.content;
   //var server;
-  var serveInit;
+  var goldilocksApis;
   var app;
   var request;
 
-  function createServeInit() {
+  function createGoldilocksApis() {
     var PromiseA = require('bluebird');
     var OAUTH3 = require('../packages/assets/org.oauth3');
     require('../packages/assets/org.oauth3/oauth3.domains.js');
@@ -31,35 +31,6 @@ module.exports = function (myDeps, conf, overrideHttp) {
     myDeps.OAUTH3 = OAUTH3;
     myDeps.recase = require('recase').create({});
     myDeps.request = request;
-    myDeps.api = {
-      // TODO move loopback to oauth3.api('tunnel:loopback')
-      loopback: function (deps, session, opts2) {
-        var crypto = require('crypto');
-        var token = crypto.randomBytes(16).toString('hex');
-        var keyAuthorization = crypto.randomBytes(16).toString('hex');
-        var nonce = crypto.randomBytes(16).toString('hex');
-
-        // TODO set token and keyAuthorization to /.well-known/cloud-challenge/:token
-        return request({
-          method: 'POST'
-        , url: 'https://oauth3.org/api/org.oauth3.tunnel/loopback'
-        , json: {
-            address: opts2.address
-          , port: opts2.port
-          , token: token
-          , keyAuthorization: keyAuthorization
-          , servername: opts2.servername
-          , nonce: nonce
-          , scheme: 'https'
-          , iat: Date.now()
-          }
-        }).then(function (result) {
-          // TODO this will always fail at the moment
-          console.log('loopback result:');
-          return result;
-        });
-      }
-    };
 
     return require('../packages/apis/com.daplie.goldilocks').create(myDeps, conf);
   }
@@ -143,31 +114,19 @@ module.exports = function (myDeps, conf, overrideHttp) {
     path.modules.forEach(mapMap);
   });
 
-  return app.use('/', function (req, res, next) {
+  return app.use('/api/com.daplie.goldilocks/:name', function (req, res, next) {
-    if (!req.headers.host) {
+    if (!goldilocksApis) {
-      next(new Error('missing HTTP Host header'));
+      goldilocksApis = createGoldilocksApis();
-      return;
     }
 
-    if (0 === req.url.indexOf('/api/com.daplie.goldilocks/')) {
+    if (typeof goldilocksApis[req.params.name] === 'function') {
-      if (!serveInit) {
+      goldilocksApis[req.params.name](req, res);
-        serveInit = createServeInit();
+    } else {
-      }
+      next();
     }
-    if ('/api/com.daplie.goldilocks/init' === req.url) {
+  }).use('/', function (req, res, next) {
-      serveInit.init(req, res);
+    if (!req.headers.host) {
-      return;
+      next(new Error('missing HTTP Host header'));
-    }
-    if ('/api/com.daplie.goldilocks/tunnel' === req.url) {
-      serveInit.tunnel(req, res);
-      return;
-    }
-    if ('/api/com.daplie.goldilocks/config' === req.url) {
-      serveInit.config(req, res);
-      return;
-    }
-    if ('/api/com.daplie.goldilocks/request' === req.url) {
-      serveInit.request(req, res);
       return;
     }
 

lib/goldilocks.js

@@ -186,11 +186,11 @@ module.exports.create = function (deps, config) {
       return;
     }
     if (Array.isArray(bindList)) {
-      bindList.forEach(function (port) {
+      bindList.filter(Number).forEach(function (port) {
         tcpPortMap[port] = true;
       });
     }
-    else {
+    else if (Number(bindList)) {
       tcpPortMap[bindList] = true;
     }
   }

lib/loopback.js

@@ -0,0 +1,95 @@
+'use strict';
+
+module.exports.create = function (deps) {
+  var PromiseA = require('bluebird');
+  var request = PromiseA.promisify(require('request'));
+  var pending = {};
+
+  function checkPublicAddr(host) {
+    return request({
+      method: 'GET'
+    , url: host+'/api/org.oauth3.tunnel/checkip'
+    , json: true
+    }).then(function (result) {
+      if (!result.body) {
+        return PromiseA.reject(new Error('No response body in request for public address'));
+      }
+      if (result.body.error) {
+        var err = new Error(result.body.error.message);
+        return PromiseA.reject(Object.assign(err, result.body.error));
+      }
+      return result.body.address;
+    });
+  }
+
+  function checkSinglePort(host, address, port) {
+    var crypto = require('crypto');
+    var token   = crypto.randomBytes(8).toString('hex');
+    var keyAuth = crypto.randomBytes(32).toString('hex');
+    pending[token] = keyAuth;
+
+    var opts = {
+      address: address
+    , port: port
+    , token: token
+    , keyAuthorization: keyAuth
+    , iat: Date.now()
+    };
+
+    return request({
+      method: 'POST'
+    , url: host+'/api/org.oauth3.tunnel/loopback'
+    , json: opts
+    })
+    .then(function (result) {
+      delete pending[token];
+      if (!result.body) {
+        return PromiseA.reject(new Error('No response body in loopback request for port '+port));
+      }
+      // If the loopback requests don't go to us then there are all kinds of ways it could
+      // error, but none of them really provide much extra information so we don't do
+      // anything that will break the PromiseA.all out and mask the other results.
+      if (result.body.error) {
+        console.log('error on remote side of port '+port+' loopback', result.body.error);
+      }
+      return !!result.body.success;
+    }, function (err) {
+      delete pending[token];
+      throw err;
+    });
+  }
+
+  function loopback(provider) {
+    return deps.OAUTH3.discover(provider).then(function (directives) {
+      return checkPublicAddr(directives.api).then(function (address) {
+        console.log('checking to see if', address, 'gets back to us');
+        var ports = require('./servers').listeners.tcp.list();
+        return PromiseA.all(ports.map(function (port) {
+          return checkSinglePort(directives.api, address, port);
+        }))
+        .then(function (values) {
+          console.log(pending);
+          var result = {error: null, address: address};
+          ports.forEach(function (port, ind) {
+            result[port] = values[ind];
+          });
+          return result;
+        });
+      });
+    });
+  }
+
+  loopback.server = require('http').createServer(function (req, res) {
+    var parsed = require('url').parse(req.url);
+    var token = parsed.pathname.replace('/.well-known/cloud-challenge/', '');
+    if (pending[token]) {
+      res.setHeader('Content-Type', 'text/plain');
+      res.end(pending[token]);
+    } else {
+      res.statusCode = 404;
+      res.end();
+    }
+  });
+
+  return loopback;
+};

lib/modules/http.js

@@ -64,7 +64,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
   }
 
   function hostMatchesDomains(req, domains) {
-    var host = separatePort((req.headers || req).host).host;
+    var host = separatePort((req.headers || req).host).host.toLowerCase();
 
     return domains.some(function (pattern) {
       return domainMatches(pattern, host);
@@ -170,6 +170,13 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     return emitConnection(acmeServer, conn, opts);
   }
 
+  function checkLoopback(conn, opts, headers) {
+    if (headers.url.indexOf('/.well-known/cloud-challenge/') !== 0) {
+      return false;
+    }
+    return emitConnection(deps.loopback.server, conn, opts);
+  }
+
   var httpsRedirectServer;
   function checkHttps(conn, opts, headers) {
     if (conf.http.allowInsecure || conn.encrypted) {
@@ -398,6 +405,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     parseHeaders(conn, opts)
       .then(function (headers) {
         if (checkAcme(conn, opts, headers))  { return; }
+        if (checkLoopback(conn, opts, headers))  { return; }
         if (checkHttps(conn, opts, headers)) { return; }
         if (checkAdmin(conn, opts, headers)) { return; }
 

lib/modules/tls.js

@@ -9,14 +9,18 @@ module.exports.create = function (deps, config, netHandler) {
 
   function extractSocketProp(socket, propName) {
     // remoteAddress, remotePort... ugh... https://github.com/nodejs/node/issues/8854
-    var value = socket[propName] || socket['_' + propName];
+    var altName = '_' + propName;
+    var value = socket[propName] || socket[altName];
     try {
       value = value || socket._handle._parent.owner.stream[propName];
+      value = value || socket._handle._parent.owner.stream[altName];
     } catch (e) {}
 
     try {
       value = value || socket._handle._parentWrap[propName];
+      value = value || socket._handle._parentWrap[altName];
       value = value || socket._handle._parentWrap._handle.owner.stream[propName];
+      value = value || socket._handle._parentWrap._handle.owner.stream[altName];
     } catch (e) {}
 
     return value || '';
@@ -160,18 +164,21 @@ module.exports.create = function (deps, config, netHandler) {
   var secureContexts = {};
   var terminatorOpts = require('localhost.daplie.me-certificates').merge({});
   terminatorOpts.SNICallback = function (sni, cb) {
+    sni = sni.toLowerCase();
     console.log("[tlsOptions.SNICallback] SNI: '" + sni + "'");
 
     var tlsOptions;
 
     // Static Certs
-    if (/.*localhost.*\.daplie\.me/.test(sni.toLowerCase())) {
+    if (/\.invalid$/.test(sni)) {
-      // TODO implement
+      sni = 'localhost.daplie.me';
+    }
+    if (/.*localhost.*\.daplie\.me/.test(sni)) {
       if (!secureContexts[sni]) {
         tlsOptions = localhostCerts.mergeTlsOptions(sni, {});
-      }
+        if (tlsOptions) {
-      if (tlsOptions) {
+          secureContexts[sni] = tls.createSecureContext(tlsOptions);
-        secureContexts[sni] = tls.createSecureContext(tlsOptions);
+        }
       }
       if (secureContexts[sni]) {
         console.log('Got static secure context:', sni, secureContexts[sni]);

lib/servers.js

@@ -46,20 +46,18 @@ module.exports.addTcpListener = function (port, handler) {
       conn.__proto = 'tcp';
       stat.handler(conn);
     });
-    server.on('error', function (e) {
+    server.on('close', function () {
+      console.log('TCP server on port %d closed', port);
       delete serversMap[port];
-
+    });
+    server.on('error', function (e) {
       if (!resolved) {
         reject(e);
-        return;
+      } else if (handler.onError) {
-      }
-
-      if (handler.onError) {
         handler.onError(e);
-        return;
+      } else {
+        throw e;
       }
-
-      throw e;
     });
 
     server.listen(port, function () {
@@ -75,29 +73,20 @@ module.exports.closeTcpListener = function (port) {
       resolve();
       return;
     }
-    stat.server.on('close', function () {
+    stat.server.once('close', resolve);
-      // once the clients close too
-      delete serversMap[port];
-      if (stat._closing) {
-        stat._closing(); // resolve
-        stat._closing = null;
-      }
-      stat = null;
-    });
-    stat._closing = resolve;
     stat.server.close();
   });
 };
 module.exports.destroyTcpListener = function (port) {
   var stat = serversMap[port];
-  delete serversMap[port];
+  if (stat) {
-  stat.server.destroy();
+    stat.server.destroy();
-  if (stat._closing) {
-    stat._closing();
-    stat._closing = null;
   }
-  stat = null;
 };
+module.exports.listTcpListeners = function () {
+  return Object.keys(serversMap).map(Number).filter(Boolean);
+};
+
 
 module.exports.addUdpListener = function (port, handler) {
   return new PromiseA(function (resolve, reject) {
@@ -162,6 +151,9 @@ module.exports.closeUdpListener = function (port) {
     stat.server.close();
   });
 };
+module.exports.listUdpListeners = function () {
+  return Object.keys(dgramMap).map(Number).filter(Boolean);
+};
 
 
 module.exports.listeners = {
@@ -169,9 +161,11 @@ module.exports.listeners = {
     add: module.exports.addTcpListener
   , close: module.exports.closeTcpListener
   , destroy: module.exports.destroyTcpListener
+  , list: module.exports.listTcpListeners
   }
 , udp: {
     add: module.exports.addUdpListener
   , close: module.exports.closeUdpListener
+  , list: module.exports.listUdpListeners
   }
 };

lib/socks5-server.js

@@ -0,0 +1,73 @@
+'use strict';
+
+module.exports.create = function () {
+  var PromiseA = require('bluebird');
+  var enableDestroy = require('server-destroy');
+  var server;
+
+  function curState() {
+    if (!server) {
+      return PromiseA.resolve({running: false});
+    }
+    return PromiseA.resolve({
+      running: true
+    , port: server.address().port
+    });
+  }
+
+  function start() {
+    if (server) {
+      return curState();
+    }
+
+    server = require('socksv5').createServer(function (info, accept) {
+      accept();
+    });
+
+    enableDestroy(server);
+    server.on('close', function () {
+      server = null;
+    });
+
+    server.useAuth(require('socksv5').auth.None());
+
+    return new PromiseA(function (resolve, reject) {
+      server.on('error', function (err) {
+        if (err.code === 'EADDRINUSE') {
+          server.listen(0);
+        } else {
+          server = null;
+          reject(err);
+        }
+      });
+      server.listen(1080, function () {
+        resolve(curState());
+      });
+    });
+  }
+
+  function stop() {
+    if (!server) {
+      return curState();
+    }
+    return new PromiseA(function (resolve, reject) {
+      var timeoutId = setTimeout(function () {
+        server.destroy();
+      }, 1000);
+      server.close(function (err) {
+        clearTimeout(timeoutId);
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  return {
+    isRunning: curState
+  , start: start
+  , stop: stop
+  };
+};

lib/storage.js

@@ -67,7 +67,7 @@ module.exports.create = function (deps, conf) {
   var config = {
     save: function (changes) {
       deps.messenger.send({
-        type: 'com.daplie.goldilocks.config-change'
+        type: 'com.daplie.goldilocks/config'
       , changes: changes
       });
     }

lib/worker.js

@@ -29,6 +29,8 @@ function create(conf) {
   };
   deps.storage = require('./storage').create(deps, conf);
   deps.proxy = require('./proxy-conn').create(deps, conf);
+  deps.socks5 = require('./socks5-server').create(deps, conf);
+  deps.loopback = require('./loopback').create(deps, conf);
 
   require('./goldilocks.js').create(deps, conf);
   process.removeListener('message', create);

packages/apis/com.daplie.goldilocks/index.js

@@ -10,8 +10,6 @@ module.exports.create = function (deps, conf) {
     inflate: true, limit: '100kb', reviver: null, strict: true /* type, verify */
   });
 
-  var api = deps.api;
-
   function handleCors(req, res, methods) {
     if (!methods) {
       methods = ['GET', 'POST'];
@@ -24,13 +22,21 @@ module.exports.create = function (deps, conf) {
     res.setHeader('Access-Control-Allow-Methods', methods.join(', '));
     res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
 
-    if (req.method.toUpperCase() !== 'OPTIONS') {
+    if (req.method.toUpperCase() === 'OPTIONS') {
-      return false;
+      res.setHeader('Allow', methods.join(', '));
+      res.end();
+      return true;
     }
 
-    res.setHeader('Allow', methods.join(', '));
+    if (methods.indexOf('*') >= 0) {
-    res.end();
+      return false;
-    return true;
+    }
+    if (methods.indexOf(req.method.toUpperCase()) < 0) {
+      res.statusCode = 405;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(JSON.stringify({ error: { message: 'method '+req.method+' not allowed', code: 'EBADMETHOD'}}));
+      return true;
+    }
   }
 
   function isAuthorized(req, res, fn) {
@@ -56,17 +62,86 @@ module.exports.create = function (deps, conf) {
     });
   }
 
+  function checkPaywall() {
+    var PromiseA = require('bluebird');
+    var testDomains = [
+      'daplie.com'
+    , 'duckduckgo.com'
+    , 'google.com'
+    , 'amazon.com'
+    , 'facebook.com'
+    , 'msn.com'
+    , 'yahoo.com'
+    ];
+
+    // While this is not being developed behind a paywall the current idea is that
+    // a paywall will either manipulate DNS queries to point to the paywall gate,
+    // or redirect HTTP requests to the paywall gate. So we check for both and
+    // hope we can detect most hotel/ISP paywalls out there in the world.
+
+    return PromiseA.resolve()
+    .then(function () {
+      var dns = PromiseA.promisifyAll(require('dns'));
+      var proms = testDomains.map(function (dom) {
+        return dns.resolve6Async(dom)
+          .catch(function (err) {
+            if (err.code === 'ENODATA') {
+              return dns.resolve4Async(dom);
+            } else {
+              return PromiseA.reject(err);
+            }
+          })
+          .then(function (result) {
+            return result[0];
+          });
+      });
+
+      return PromiseA.all(proms).then(function (addrs) {
+        var unique = addrs.filter(function (value, ind, self) {
+          return value && self.indexOf(value) === ind;
+        });
+        // It is possible some walls might have exceptions that leave some of the domains
+        // we test alone, so we might have more than one unique address even behind an
+        // active paywall.
+        return unique.length < addrs.length;
+      });
+    })
+    .then(function (paywall) {
+      if (paywall) {
+        return paywall;
+      }
+      var request = deps.request.defaults({
+        followRedirect: false
+      , headers: {
+          connection: 'close'
+        }
+      });
+
+      var proms = testDomains.map(function (dom) {
+        return request('https://'+dom).then(function (resp) {
+          if (resp.statusCode >= 300 && resp.statusCode < 400) {
+            return resp.headers.location;
+          } else {
+            return 'https://'+dom;
+          }
+        });
+      });
+
+      return PromiseA.all(proms).then(function (urls) {
+        var unique = urls.filter(function (value, ind, self) {
+          return value && self.indexOf(value) === ind;
+        });
+        return unique.length < urls.length;
+      });
+    })
+    ;
+  }
+
   return {
     init: function (req, res) {
       if (handleCors(req, res, ['GET', 'POST'])) {
         return;
       }
-      if (req.method !== 'POST') {
-        res.statusCode = 405;
-        res.setHeader('Content-Type', 'application/json');
-        res.end(JSON.stringify({ error: { message: 'method '+req.method+' not allowed'}}));
-        return;
-      }
 
       if ('POST' !== req.method) {
         // It should be safe to give the list of owner IDs to an un-authenticated
@@ -238,6 +313,70 @@ module.exports.create = function (deps, conf) {
       });
       });
     }
-  , _api: api
+  , loopback: function (req, res) {
+      if (handleCors(req, res, 'GET')) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+        var prom;
+        var query = require('querystring').parse(require('url').parse(req.url).query);
+        if (query.provider) {
+          prom = deps.loopback(query.provider);
+        } else {
+          prom = deps.storage.owners.get(req.userId).then(function (session) {
+            return deps.loopback(session.token.aud);
+          });
+        }
+
+        res.setHeader('Content-Type', 'application/json');
+        prom.then(function (result) {
+          res.end(JSON.stringify(result));
+        }, function (err) {
+          res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+        });
+      });
+    }
+  , paywall_check: function (req, res) {
+      if (handleCors(req, res, 'GET')) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+        res.setHeader('Content-Type', 'application/json;');
+
+        checkPaywall().then(function (paywall) {
+          res.end(JSON.stringify({paywall: paywall}));
+        }, function (err) {
+          err.message = err.message || err.toString();
+          res.statusCode = 500;
+          res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+        });
+      });
+    }
+  , socks5: function (req, res) {
+      if (handleCors(req, res, ['GET', 'POST', 'DELETE'])) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+        var method = req.method.toUpperCase();
+        var prom;
+
+        if (method === 'POST') {
+          prom = deps.socks5.start();
+        } else if (method === 'DELETE') {
+          prom = deps.socks5.stop();
+        } else {
+          prom = deps.socks5.curState();
+        }
+
+        res.setHeader('Content-Type', 'application/json;');
+        prom.then(function (result) {
+          res.end(JSON.stringify(result));
+        }, function (err) {
+          err.message = err.message || err.toString();
+          res.statusCode = 500;
+          res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+        });
+      });
+    }
   };
 };