Merge branch 'master' of git.daplie.com:Daplie/goldilocks.js

README.md

@@ -370,6 +370,41 @@ tunnel:
     tunnelUrl: 'wss://api.tunnel.example.com/'
 ```
 
+**NOTE**: The more common way to use the tunnel with goldilocks is to use the
+API to have goldilocks get a token from `oauth3.org`. In order to do this you
+will need to have initialized goldilocks with a token that has the `dns` and
+`domains` scopes. This is probably easiest to do with the `daplie-desktop-app`,
+which will also get the first tunnel token for you.
+
+**If you want to add more domains** to handle on your device while using the tunnel
+you will need to manually get a new token that will tell the tunnel server to
+deliver the requests to the new domain(s) to your device. The first step in this
+is to attach the new domains to your device. To get the name of the device you
+can use the `config` API, but it's probably easiest to `ssh` onto the device and
+get the hostname. You can also use the daplie cli tool to see what device name
+your other domains are routed to.
+
+```bash
+# for every new domain you want to route attach the domain to your device
+daplie devices:attach -n $new_domain -d $device_name
+```
+
+After that step you will need to use the API to get goldilocks to get a new token
+that includes the new domains you attached. It is also recommended but not
+required to remove the older token with the incomplete list of domains. Run the
+following commands from the unit.
+
+```bash
+# remove the old token
+rm /opt/goldilocks/lib/node_modules/goldilocks/var/tokens.json
+
+# set the "refresh_token" to a bash variable `token`
+TOKEN=$(python -mjson.tool /opt/goldilocks/lib/node_modules/goldilocks/var/owners.json | sed -n 's|\s*"refresh_token": "\(.*\)",|\1|p')
+
+# tell goldilocks to get a new tunnel token
+curl -H "authorization: bearer $TOKEN" -X POST https://localhost.admin.daplie.me/api/goldilocks@daplie.com/tunnel
+```
+
 ### ddns
 
 TODO

bin/goldilocks.js

@@ -41,15 +41,12 @@ function createStorage(filename, filetype) {
   }
 
   function read() {
-    return fs.readFileAsync(filename)
+    return fs.readFileAsync(filename).then(parse).catch(function (err) {
-      .catch(function (err) {
       if (err.code === 'ENOENT') {
         return '';
       }
       return PromiseA.reject(err);
-      })
+    });
-      .then(parse)
-      ;
   }
 
   var result = {
@@ -154,6 +151,9 @@ function fillConfig(config, args) {
   if (!config.dns) {
     config.dns = { bind: [ 53 ], modules: [{ name: 'proxy', port: 3053 }] };
   }
+  if (!config.ddns) {
+    config.ddns = { enabled: false };
+  }
   // Use Object.assign to add any properties needed but not defined in the mdns config.
   // It will first copy the defaults into an empty object, then copy any real config over that.
   var mdnsDefaults = { port: 5353, broadcast: '224.0.0.251', ttl: 300 };
@@ -250,25 +250,29 @@ function run(args) {
   var workers = {};
   var cachedConfig;
 
-  cluster.on('message', function (worker, message) {
+  function updateConfig(config) {
-    if (message.type !== 'com.daplie.goldilocks/config') {
+    fillConfig(config, args).then(function (config) {
-      return;
-    }
-    configStorage.save(message.changes)
-      .then(function (config) {
-        return fillConfig(config, args);
-      })
-      .then(function (config) {
       cachedConfig = config;
       console.log('changed config', config);
       Object.keys(workers).forEach(function (key) {
         workers[key].send(cachedConfig);
       });
-      })
+    });
-      .catch(function (err) {
+  }
+
+  process.on('SIGHUP', function () {
+    configStorage.read().then(updateConfig).catch(function (err) {
+      console.error('error updating config after SIGHUP', err);
+    });
+  });
+
+  cluster.on('message', function (worker, message) {
+    if (message.type !== 'com.daplie.goldilocks/config') {
+      return;
+    }
+    configStorage.save(message.changes).then(updateConfig).catch(function (err) {
       console.error('error changing config', err);
-      })
+    });
-      ;
   });
 
   cluster.on('online', function (worker) {

lib/ddns.js

@@ -1,88 +1,149 @@
 'use strict';
 
-module.exports.create = function (opts/*, servers*/) {
+module.exports.create = function (deps, conf) {
-  var PromiseA = opts.PromiseA;
+  var PromiseA = deps.PromiseA;
-  var dns = PromiseA.promisifyAll(require('dns'));
+  var request = PromiseA.promisify(require('request'));
+  var OAUTH3 = require('../packages/assets/org.oauth3');
+  require('../packages/assets/org.oauth3/oauth3.dns.js');
+  OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js');
 
-  return PromiseA.all([
+  function dnsType(addr) {
-    dns.resolve4Async(opts._old_server_name).then(function (results) {
+    if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) {
-      return results;
+      return 'A';
-    }, function () {})
+    }
-  , dns.resolve6Async(opts._old_server_name).then(function (results) {
+    if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) {
-      return results;
+      return 'AAAA';
-    }, function () {})
+    }
-  ]).then(function (results) {
+  }
-    var ipv4 = results[0] || [];
-    var ipv6 = results[1] || [];
-    var record;
 
-    opts.dnsRecords = {
+  async function getSession() {
-      A: ipv4
+    var sessions = await deps.storage.owners.all();
-    , AAAA: ipv6
+    var session = sessions.filter(function (sess) {
+      return sess.token.scp.indexOf('dns') >= 0;
+    })[0];
+
+    if (!session) {
+      throw new Error('no sessions with DNS grants');
+    }
+
+    // The OAUTH3 library stores some things on the root session object that we usually
+    // just leave inside the token, but we need to pull those out before we use it here
+    session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss;
+    session.client_uri = session.client_uri || session.token.azp;
+    session.scope = session.scope || session.token.scp;
+    return session;
+  }
+
+  async function setDeviceAddress(addr) {
+    var session = await getSession();
+    var directives = await OAUTH3.discover(session.token.aud);
+
+    // Set the address of the device to our public address.
+    await request({
+      url: directives.api+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname
+    , method: 'POST'
+    , headers: {
+        'Authorization': 'Bearer ' + session.refresh_token
+      , 'Accept': 'application/json; charset=utf-8'
+      }
+    , json: {
+        addresses: [
+          { value: addr, type:  dnsType(addr) }
+        ]
+      }
+    });
+
+    // Then update all of the records attached to our hostname, first removing the old records
+    // to remove the reference to the old address, then creating new records for the same domains
+    // using our new address.
+    var allDns = OAUTH3.api(directives.api, {session: session, api: 'dns.list'});
+    var ourDomains = allDns.filter(function (record) {
+      return record.device === conf.device.hostname;
+    }).map(function (record) {
+      var zoneSplit = record.zone.split('.');
+      return {
+        tld: zoneSplit.slice(1).join('.')
+      , sld: zoneSplit[0]
+      , sub: record.host.slice(0, -(record.zone.length + 1))
       };
-
-    Object.keys(opts.ifaces).some(function (ifacename) {
-      var iface = opts.ifaces[ifacename];
-
-      return iface.ipv4.some(function (localIp) {
-        return ipv4.some(function (remoteIp) {
-          if (localIp.address === remoteIp) {
-            record = localIp;
-            return record;
-          }
-        });
-      }) || iface.ipv6.some(function (localIp) {
-        return ipv6.forEach(function (remoteIp) {
-          if (localIp.address === remoteIp) {
-            record = localIp;
-            return record;
-          }
-        });
-      });
     });
 
-    if (!record) {
+    var common = {
-      console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address.");
+      api: 'devices.detach'
-      console.info("Use --ddns to allow the people of the Internet to access your server.");
+    , session: session
-    }
+    , device: conf.device.hostname
-
-    opts.externalIps.ipv4.some(function (localIp) {
-      return ipv4.some(function (remoteIp) {
-        if (localIp.address === remoteIp) {
-          record = localIp;
-          return record;
-        }
-      });
-    });
-
-    opts.externalIps.ipv6.some(function (localIp) {
-      return ipv6.some(function (remoteIp) {
-        if (localIp.address === remoteIp) {
-          record = localIp;
-          return record;
-        }
-      });
-    });
-
-    if (!record) {
-      console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address.");
-      console.info("Use --ddns to allow the people of the Internet to access your server.");
-    }
-  });
     };
+    await PromiseA.all(ourDomains.map(function (record) {
+      return OAUTH3.api(directives.api, Object.assign({}, common, record));
+    }));
 
-if (require.main === module) {
+    common = {
-  var opts = {
+      api: 'devices.attach'
-    _old_server_name: 'aj.daplie.me'
+    , session: session
-  , PromiseA: require('bluebird')
+    , device: conf.device.hostname
+    , ip: addr
+    , ttl: 300
     };
-  // ifaces
+    await PromiseA.all(ourDomains.map(function (record) {
-  opts.ifaces = require('./local-ip.js').find();
+      return OAUTH3.api(directives.api, Object.assign({}, common, record));
-  console.log('opts.ifaces');
+    }));
-  console.log(opts.ifaces);
-  require('./match-ips.js').match(opts._old_server_name, opts).then(function (ips) {
-    opts.matchingIps = ips.matchingIps || [];
-    opts.externalIps = ips.externalIps;
-    module.exports.create(opts);
-  });
   }
+
+  async function getDeviceAddresses() {
+    var session = await getSession();
+    var directives = await OAUTH3.discover(session.token.aud);
+
+    var result = await request({
+      url: directives.api+'/api/org.oauth3.dns/acl/devices'
+    , method: 'GET'
+    , headers: {
+        'Authorization': 'Bearer ' + session.refresh_token
+      , 'Accept': 'application/json; charset=utf-8'
+      }
+    , json: true
+    });
+
+    if (!result.body) {
+      throw new Error('No response body in request for device addresses');
+    }
+    if (result.body.error) {
+      throw Object.assign(new Error('error getting device list'), result.body.error);
+    }
+
+    var dev = result.body.devices.filter(function (dev) {
+      return dev.name === conf.device.hostname;
+    })[0];
+    return (dev || {}).addresses || [];
+  }
+
+  var publicAddress;
+  async function recheckPubAddr() {
+    if (!conf.ddns.enabled) {
+      return;
+    }
+
+    var session = await getSession();
+    var directives = await OAUTH3.discover(session.token.aud);
+    var addr = await deps.loopback.checkPublicAddr(directives.api);
+
+    if (publicAddress === addr) {
+      return;
+    }
+
+    if (conf.debug) {
+      console.log('previous public address',publicAddress, 'does not match current public address', addr);
+    }
+
+    await setDeviceAddress(addr);
+    publicAddress = addr;
+  }
+
+  recheckPubAddr();
+  setInterval(recheckPubAddr, 5*60*1000);
+
+  return {
+    setDeviceAddress:   setDeviceAddress
+  , getDeviceAddresses: getDeviceAddresses
+  , recheckPubAddr:     recheckPubAddr
+  };
+};

lib/goldilocks.js

@@ -50,7 +50,7 @@ module.exports.create = function (deps, config) {
     }
 
     console.warn('failed to identify protocol from first chunk', firstChunk);
-    conn.close();
+    conn.destroy();
   }
   function netHandler(conn, opts) {
     function getProp(name) {

lib/loopback.js

@@ -1,84 +1,88 @@
 'use strict';
 
-module.exports.create = function (deps) {
+module.exports.create = function (deps, conf) {
   var PromiseA = require('bluebird');
   var request = PromiseA.promisify(require('request'));
   var pending = {};
 
-  function checkPublicAddr(host) {
+  async function checkPublicAddr(host) {
-    return request({
+    var result = await request({
       method: 'GET'
     , url: host+'/api/org.oauth3.tunnel/checkip'
     , json: true
-    }).then(function (result) {
+    });
+
     if (!result.body) {
-        return PromiseA.reject(new Error('No response body in request for public address'));
+      throw new Error('No response body in request for public address');
     }
     if (result.body.error) {
-        var err = new Error(result.body.error.message);
+      // Note that the error on the body will probably have a message that overwrites the default
-        return PromiseA.reject(Object.assign(err, result.body.error));
+      throw Object.assign(new Error('error in check IP response'), result.body.error);
     }
     return result.body.address;
-    });
   }
 
-  function checkSinglePort(host, address, port) {
+  async function checkSinglePort(host, address, port) {
     var crypto = require('crypto');
     var token   = crypto.randomBytes(8).toString('hex');
     var keyAuth = crypto.randomBytes(32).toString('hex');
     pending[token] = keyAuth;
 
-    var opts = {
+    var reqObj = {
+      method: 'POST'
+    , url: host+'/api/org.oauth3.tunnel/loopback'
+    , json: {
         address: address
       , port: port
       , token: token
       , keyAuthorization: keyAuth
       , iat: Date.now()
+      }
     };
 
-    return request({
+    var result;
-      method: 'POST'
+    try {
-    , url: host+'/api/org.oauth3.tunnel/loopback'
+      result = await request(reqObj);
-    , json: opts
+    } catch (err) {
-    })
+      delete pending[token];
-    .then(function (result) {
+      throw err;
+    }
+
     delete pending[token];
     if (!result.body) {
-        return PromiseA.reject(new Error('No response body in loopback request for port '+port));
+      throw new Error('No response body in loopback request for port '+port);
     }
     // If the loopback requests don't go to us then there are all kinds of ways it could
     // error, but none of them really provide much extra information so we don't do
     // anything that will break the PromiseA.all out and mask the other results.
-      if (result.body.error) {
+    if (conf.debug && result.body.error) {
       console.log('error on remote side of port '+port+' loopback', result.body.error);
     }
     return !!result.body.success;
-    }, function (err) {
-      delete pending[token];
-      throw err;
-    });
   }
 
-  function loopback(provider) {
+  async function loopback(provider) {
-    return deps.OAUTH3.discover(provider).then(function (directives) {
+    var directives = await deps.OAUTH3.discover(provider);
-      return checkPublicAddr(directives.api).then(function (address) {
+    var address = await checkPublicAddr(directives.api);
     console.log('checking to see if', address, 'gets back to us');
+
     var ports = require('./servers').listeners.tcp.list();
-        return PromiseA.all(ports.map(function (port) {
+    var values = await PromiseA.all(ports.map(function (port) {
       return checkSinglePort(directives.api, address, port);
-        }))
+    }));
-        .then(function (values) {
+
-          console.log(pending);
+    if (conf.debug) {
+      console.log('remaining loopback tokens', pending);
+    }
+
     var result = {error: null, address: address};
     ports.forEach(function (port, ind) {
       result[port] = values[ind];
     });
     return result;
-        });
-      });
-    });
   }
 
+  loopback.checkPublicAddr = checkPublicAddr;
   loopback.server = require('http').createServer(function (req, res) {
     var parsed = require('url').parse(req.url);
     var token = parsed.pathname.replace('/.well-known/cloud-challenge/', '');

lib/match-ips.js

@@ -1,117 +0,0 @@
-'use strict';
-
-var PromiseA = require('bluebird');
-
-module.exports.match = function (servername, opts) {
-  return PromiseA.promisify(require('ipify'))().then(function (externalIp) {
-    var dns = PromiseA.promisifyAll(require('dns'));
-
-    opts.externalIps = [ { address: externalIp, family: 'IPv4' } ];
-    opts.ifaces = require('./local-ip.js').find({ externals: opts.externalIps });
-    opts.externalIfaces = Object.keys(opts.ifaces).reduce(function (all, iname) {
-      var iface = opts.ifaces[iname];
-
-      iface.ipv4.forEach(function (addr) {
-        if (addr.external) {
-          addr.iface = iname;
-          all.push(addr);
-        }
-      });
-      iface.ipv6.forEach(function (addr) {
-        if (addr.external) {
-          addr.iface = iname;
-          all.push(addr);
-        }
-      });
-
-      return all;
-    }, []).filter(Boolean);
-
-    function resolveIps(hostname) {
-      var allIps = [];
-
-      return PromiseA.all([
-        dns.resolve4Async(hostname).then(function (records) {
-            records.forEach(function (ip) {
-              allIps.push({
-                address: ip
-              , family: 'IPv4'
-              });
-            });
-          }, function () {})
-        , dns.resolve6Async(hostname).then(function (records) {
-            records.forEach(function (ip) {
-              allIps.push({
-                address: ip
-              , family: 'IPv6'
-              });
-            });
-          }, function () {})
-      ]).then(function () {
-        return allIps;
-      });
-    }
-
-    function resolveIpsAndCnames(hostname) {
-      return PromiseA.all([
-        resolveIps(hostname)
-      , dns.resolveCnameAsync(hostname).then(function (records) {
-          return PromiseA.all(records.map(function (hostname) {
-            return resolveIps(hostname);
-          })).then(function (allIps) {
-            return allIps.reduce(function (all, ips) {
-              return all.concat(ips);
-            }, []);
-          });
-        }, function () {
-          return [];
-        })
-      ]).then(function (ips) {
-        return ips.reduce(function (all, set) {
-          return all.concat(set);
-        }, []);
-      });
-    }
-
-    return resolveIpsAndCnames(servername).then(function (allIps) {
-      var matchingIps = [];
-
-      if (!allIps.length) {
-        console.warn("Could not resolve '" + servername + "'");
-      }
-
-      // { address, family }
-      allIps.some(function (ip) {
-        function match(addr) {
-          if (ip.address === addr.address) {
-            matchingIps.push(addr);
-          }
-        }
-
-        opts.externalIps.forEach(match);
-        // opts.externalIfaces.forEach(match);
-
-        Object.keys(opts.ifaces).forEach(function (iname) {
-          var iface = opts.ifaces[iname];
-
-          iface.ipv4.forEach(match);
-          iface.ipv6.forEach(match);
-        });
-
-        return matchingIps.length;
-      });
-
-      matchingIps.externalIps = {
-        ipv4: [
-          { address: externalIp
-          , family: 'IPv4'
-          }
-        ]
-      , ipv6: [
-        ]
-      };
-      matchingIps.matchingIps = matchingIps;
-      return matchingIps;
-    });
-  });
-};

lib/modules/http.js

@@ -29,12 +29,14 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
       function handleChunk(chunk) {
         if (!errored) {
           opts.firstChunk = Buffer.concat([opts.firstChunk, chunk]);
-          if (opts.firstChunk.includes('\r\n\r\n')) {
+          if (!opts.firstChunk.includes('\r\n\r\n')) {
-            resolve(opts.firstChunk.toString());
-            conn.removeListener('error', handleErr);
-          } else {
             conn.once('data', handleChunk);
+            return;
           }
+
+          conn.removeListener('error', handleErr);
+          conn.pause();
+          resolve(opts.firstChunk.toString());
         }
       }
       conn.once('data', handleChunk);
@@ -144,6 +146,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
     // data comes in.
     process.nextTick(function () {
       conn.unshift(opts.firstChunk);
+      conn.resume();
     });
 
     // Convenience return for all the check* functions.
@@ -160,6 +163,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
       deps.tunnelServer.handleClientConn(conn);
       process.nextTick(function () {
         conn.unshift(opts.firstChunk);
+        conn.resume();
       });
       return true;
     }
@@ -211,6 +215,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
       deps.tunnelServer.handleAdminConn(conn);
       process.nextTick(function () {
         conn.unshift(opts.firstChunk);
+        conn.resume();
       });
       return true;
     }

lib/worker.js

@@ -22,6 +22,7 @@ function create(conf) {
   config = conf;
   var deps = {
     messenger: process
+  , PromiseA: require('bluebird')
     // Note that if a custom createConnections is used it will be called with different
     // sets of custom options based on what is actually being proxied. Most notably the
     // HTTP proxying connection creation is not something we currently control.
@@ -31,6 +32,7 @@ function create(conf) {
   deps.proxy = require('./proxy-conn').create(deps, conf);
   deps.socks5 = require('./socks5-server').create(deps, conf);
   deps.loopback = require('./loopback').create(deps, conf);
+  deps.ddns = require('./ddns').create(deps, conf);
 
   require('./goldilocks.js').create(deps, conf);
   process.removeListener('message', create);

package.json

@@ -48,7 +48,6 @@
     "http-proxy": "^1.16.2",
     "human-readable-ids": "git+https://git.daplie.com/Daplie/human-readable-ids-js#master",
     "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0",
-    "ipify": "^1.1.0",
     "js-yaml": "^3.8.3",
     "jsonwebtoken": "^7.4.0",
     "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master",