Merge branch 'master' of git.daplie.com:Daplie/goldilocks.js

CHANGELOG

@@ -1,3 +1,5 @@
+v1.1.5 - Implemented dns-01 ACME challenges
+
 v1.1.4 - Improved responsiveness to config updates
   * changed which TCP/UDP ports are bound to on config update
   * update tunnel server settings on config update

README.md

@@ -50,8 +50,16 @@ npm install -g goldilocks@v1
 
 ### Uninstall
 
+Remove goldilocks and services:
+
 ```
-rm -rf /srv/goldilocks/ /var/goldilocks/ /etc/goldilocks/ /opt/goldilocks/ /var/log/goldilocks/ /etc/tmpfiles.d/goldilocks.conf /etc/systemd/system/goldilocks.service /etc/ssl/goldilocks
+rm -rf /opt/goldilocks/ /srv/goldilocks/ /var/goldilocks/ /var/log/goldilocks/ /etc/tmpfiles.d/goldilocks.conf /etc/systemd/system/goldilocks.service
+```
+
+Remove config as well
+
+```
+rm -rf /etc/goldilocks/ /etc/ssl/goldilocks
 ```
 
 Usage
@@ -297,6 +305,12 @@ tls:
       challenge_type: 'http-01'
 ```
 
+**NOTE:** If you specify `dns-01` as the challenge type there must also be a
+[DDNS module](#ddns) defined for all of the relevant domains (though not all
+domains handled by a single TLS module need to be handled by the same DDNS
+module). The DDNS module provides all of the information needed to actually
+set the DNS records needed to verify ownership.
+
 ### tcp
 
 The tcp system handles both *raw* and *tls-terminated* tcp network traffic

lib/ddns/challenge-responder.js

@@ -0,0 +1,122 @@
+'use strict';
+
+// Much of this file was based on the `le-challenge-ddns` library (which we are not using
+// here because it's method of setting records requires things we don't really want).
+module.exports.create = function (deps, conf, utils) {
+
+  function getReleventSessionId(domain) {
+    var sessId;
+
+    utils.iterateAllModules(function (mod, domainList) {
+      // We return a truthy value in these cases because of the way the iterate function
+      // handles modules grouped by domain. By returning true we are saying these domains
+      // are "handled" and so if there are multiple modules we won't be given the rest.
+      if (sessId) { return true; }
+      if (domainList.indexOf(domain) < 0) { return true; }
+
+      // But if the domains are relevant but we don't know how to handle the module we
+      // return false to allow us to look at any other modules that might exist here.
+      if (mod.type !== 'dns@oauth3.org')  { return false; }
+
+      sessId = mod.tokenId || mod.token_id;
+      return true;
+    });
+
+    return sessId;
+  }
+
+  function get(args, domain, challenge, done) {
+    done(new Error("Challenge.get() does not need an implementation for dns-01. (did you mean Challenge.loopback?)"));
+  }
+  // same as get, but external
+  function loopback(args, domain, challenge, done) {
+    var challengeDomain = (args.test || '') + args.acmeChallengeDns + domain;
+    require('dns').resolveTxt(challengeDomain, done);
+  }
+
+  var activeChallenges = {};
+  async function removeAsync(args, domain) {
+    var data = activeChallenges[domain];
+    if (!data) {
+      console.warn(new Error('cannot remove DNS challenge for ' + domain + ': already removed'));
+      return;
+    }
+
+    var session = await utils.getSession(data.sessId);
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+    var apiOpts = {
+      api: 'dns.unset'
+    , session: session
+    , type: 'TXT'
+    , value: data.keyAuthDigest
+    };
+    await deps.OAUTH3.api(directives.api, Object.assign({}, apiOpts, data.splitDomain));
+
+    delete activeChallenges[domain];
+  }
+  async function setAsync(args, domain, challenge, keyAuth) {
+    if (activeChallenges[domain]) {
+      await removeAsync(args, domain, challenge);
+    }
+
+    var sessId = getReleventSessionId(domain);
+    if (!sessId) {
+      throw new Error('no DDNS module handles the domain ' + domain);
+    }
+    var session = await utils.getSession(sessId);
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+
+    // I'm not sure what role challenge is supposed to play since even in the library
+    // this code is based on it was never used, but check for it anyway because ...
+    if (!challenge || keyAuth) {
+      console.warn(new Error('DDNS challenge missing challenge or keyAuth'));
+    }
+    var keyAuthDigest = require('crypto').createHash('sha256').update(keyAuth || '').digest('base64')
+      .replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+
+    var challengeDomain = (args.test || '') + args.acmeChallengeDns + domain;
+    var splitDomain = (await utils.splitDomains(directives.api, [challengeDomain]))[0];
+
+    var apiOpts = {
+      api: 'dns.set'
+    , session: session
+    , type: 'TXT'
+    , value: keyAuthDigest
+    , ttl: args.ttl || 0
+    };
+    await deps.OAUTH3.api(directives.api, Object.assign({}, apiOpts, splitDomain));
+
+    activeChallenges[domain] = {
+      sessId
+    , keyAuthDigest
+    , splitDomain
+    };
+
+    return new Promise(res => setTimeout(res, 1000));
+  }
+
+  // It might be slightly easier to use arguments and apply, but the library that will use
+  // this function counts the arguments we expect.
+  function set(a, b, c, d, done) {
+    setAsync(a, b, c, d).then(result => done(null, result), done);
+  }
+  function remove(a, b, c, done) {
+    removeAsync(a, b, c).then(result => done(null, result), done);
+  }
+
+  function getOptions() {
+    return {
+      oauth3: 'oauth3.org'
+    , debug: conf.debug
+    , acmeChallengeDns: '_acme-challenge.'
+    };
+  }
+
+  return {
+    getOptions
+  , set
+  , get
+  , remove
+  , loopback
+  };
+};

lib/ddns/dns-ctrl.js

@@ -1,6 +1,6 @@
 'use strict';
 
-module.exports.create = function (deps, conf) {
+module.exports.create = function (deps, conf, utils) {
   function dnsType(addr) {
     if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) {
       return 'A';
@@ -10,62 +10,6 @@ module.exports.create = function (deps, conf) {
     }
   }
 
-  var tldCache = {};
-  async function getTlds(provider) {
-    async function updateCache() {
-      var reqObj = {
-        url: deps.OAUTH3.url.normalize(provider)+'/api/com.daplie.domains/prices'
-      , method: 'GET'
-      , json: true
-      };
-
-      var resp = await deps.OAUTH3.request(reqObj);
-      var tldObj = {};
-      resp.data.forEach(function (tldInfo) {
-        if (tldInfo.enabled) {
-          tldObj[tldInfo.tld] = true;
-        }
-      });
-
-      tldCache[provider] = {
-        time: Date.now()
-      , tlds: tldObj
-      };
-      return tldObj;
-    }
-
-    // If we've never cached the results we need to return the promise that will fetch the recult,
-    // otherwise we can return the cached value. If the cached value has "expired", we can still
-    // return the cached value we just want to update the cache in parellel (making sure we only
-    // update once).
-    if (!tldCache[provider]) {
-      return updateCache();
-    }
-    if (!tldCache[provider].updating && Date.now() - tldCache[provider].time > 24*60*60*1000) {
-      tldCache[provider].updating = true;
-      updateCache();
-    }
-
-    return tldCache[provider].tlds;
-  }
-
-  async function splitDomains(provider, domains) {
-    var tlds = await getTlds(provider);
-    return domains.map(function (domain) {
-      var split = domain.split('.');
-      var tldSegCnt = tlds[split.slice(-2).join('.')] ? 2 : 1;
-
-      // Currently assuming that the sld can't contain dots, and that the tld can have at
-      // most one dot. Not 100% sure this is a valid assumption, but exceptions should be
-      // rare even if the assumption isn't valid.
-      return {
-        tld: split.slice(-tldSegCnt).join('.')
-      , sld: split.slice(-tldSegCnt-1, -tldSegCnt).join('.')
-      , sub: split.slice(0, -tldSegCnt-1).join('.')
-      };
-    });
-  }
-
   async function setDeviceAddress(session, addr, domains) {
     var directives = await deps.OAUTH3.discover(session.token.aud);
 
@@ -111,7 +55,7 @@ module.exports.create = function (deps, conf) {
       return goodAddrDomains.indexOf(domain) < 0;
     });
 
-    var oldDns = await splitDomains(directives.api, badAddrDomains);
+    var oldDns = await utils.splitDomains(directives.api, badAddrDomains);
     var common = {
       api: 'devices.detach'
     , session: session
@@ -124,7 +68,7 @@ module.exports.create = function (deps, conf) {
       console.log('removed bad DNS records for ' + badAddrDomains.join(', '));
     }
 
-    var newDns = await splitDomains(directives.api, requiredUpdates);
+    var newDns = await utils.splitDomains(directives.api, requiredUpdates);
     common = {
       api: 'devices.attach'
     , session: session
@@ -169,7 +113,7 @@ module.exports.create = function (deps, conf) {
   async function removeDomains(session, domains) {
     var directives = await deps.OAUTH3.discover(session.token.aud);
 
-    var oldDns = await splitDomains(directives.api, domains);
+    var oldDns = await utils.splitDomains(directives.api, domains);
     var common = {
       api: 'devices.detach'
     , session: session

lib/ddns/index.js

@@ -3,48 +3,20 @@
 module.exports.create = function (deps, conf) {
   var dns = deps.PromiseA.promisifyAll(require('dns'));
   var network = deps.PromiseA.promisifyAll(deps.recase.camelCopy(require('network')));
-  var loopback = require('./loopback').create(deps, conf);
-  var dnsCtrl = require('./dns-ctrl').create(deps, conf);
-  var tunnelClients = require('./tunnel-client-manager').create(deps, conf);
   var equal = require('deep-equal');
 
+  var utils = require('./utils').create(deps, conf);
+  var loopback = require('./loopback').create(deps, conf, utils);
+  var dnsCtrl = require('./dns-ctrl').create(deps, conf, utils);
+  var challenge = require('./challenge-responder').create(deps, conf, utils);
+  var tunnelClients = require('./tunnel-client-manager').create(deps, conf, utils);
+
   var loopbackDomain;
 
-  function iterateAllModules(action, curConf) {
-    curConf = curConf || conf;
-    var promises = curConf.ddns.modules.map(function (mod) {
-      return action(mod, mod.domains);
-    });
-
-    curConf.domains.forEach(function (dom) {
-      if (!dom.modules || !Array.isArray(dom.modules.ddns) || !dom.modules.ddns.length) {
-        return null;
-      }
-
-      // For the time being all of our things should only be tried once (regardless if it succeeded)
-      // TODO: revisit this behavior when we support multiple ways of setting records, and/or
-      // if we want to allow later modules to run if early modules fail.
-      promises.push(dom.modules.ddns.reduce(function (prom, mod) {
-        if (prom) { return prom; }
-        return action(mod, dom.names);
-      }, null));
-    });
-
-    return deps.PromiseA.all(promises.filter(Boolean));
-  }
-
-  async function getSession(id) {
-    var session = await deps.storage.tokens.get(id);
-    if (!session) {
-      throw new Error('no user token with ID "'+id+'"');
-    }
-    return session;
-  }
-
   var tunnelActive = false;
   async function startTunnel(tunnelSession, mod, domainList) {
     try {
-      var dnsSession = await getSession(mod.tokenId);
+      var dnsSession = await utils.getSession(mod.tokenId);
       var tunnelDomain = await tunnelClients.start(tunnelSession || dnsSession, domainList);
 
       var addrList;
@@ -60,7 +32,9 @@ module.exports.create = function (deps, conf) {
         throw new Error('failed to lookup IP for tunnel domain "' + tunnelDomain + '"');
       }
 
-      await dnsCtrl.setDeviceAddress(dnsSession, addrList[0], domainList);
+      if (!mod.disabled) {
+        await dnsCtrl.setDeviceAddress(dnsSession, addrList[0], domainList);
+      }
     } catch (err) {
       console.log('error starting tunnel for', domainList.join(', '));
       console.log(err);
@@ -74,7 +48,7 @@ module.exports.create = function (deps, conf) {
       tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId);
     }
 
-    await iterateAllModules(function (mod, domainList) {
+    await utils.iterateAllModules(function (mod, domainList) {
       if (mod.type !== 'dns@oauth3.org') { return null; }
 
       return startTunnel(tunnelSession, mod, domainList);
@@ -90,7 +64,7 @@ module.exports.create = function (deps, conf) {
   async function checkTunnelTokens() {
     var oldTokens = tunnelClients.current();
 
-    var newTokens = await iterateAllModules(function checkTokens(mod, domainList) {
+    var newTokens = await utils.iterateAllModules(function checkTokens(mod, domainList) {
       if (mod.type !== 'dns@oauth3.org') { return null; }
 
       var domainStr = domainList.slice().sort().join(',');
@@ -188,10 +162,10 @@ module.exports.create = function (deps, conf) {
     }
     publicAddress = addr;
 
-    await iterateAllModules(function setModuleDNS(mod, domainList) {
+    await utils.iterateAllModules(function setModuleDNS(mod, domainList) {
       if (mod.type !== 'dns@oauth3.org' || mod.disabled) { return null; }
 
-      return getSession(mod.tokenId).then(function (session) {
+      return utils.getSession(mod.tokenId).then(function (session) {
         return dnsCtrl.setDeviceAddress(session, addr, domainList);
       }).catch(function (err) {
         console.log('error setting DNS records for', domainList.join(', '));
@@ -206,13 +180,13 @@ module.exports.create = function (deps, conf) {
 
     // this returns a Promise, but since the functions we use are synchronous
     // and change our enclosed variables we don't need to wait for the return.
-    iterateAllModules(function (mod, domainList) {
+    utils.iterateAllModules(function (mod, domainList) {
       if (mod.type !== 'dns@oauth3.org') { return; }
 
       prevMods[mod.id] = { mod, domainList };
       return true;
     }, prevConf);
-    iterateAllModules(function (mod, domainList) {
+    utils.iterateAllModules(function (mod, domainList) {
       if (mod.type !== 'dns@oauth3.org') { return; }
 
       curMods[mod.id] = { mod, domainList };
@@ -235,8 +209,11 @@ module.exports.create = function (deps, conf) {
 
     // Then remove DNS records for the domains that we are no longer responsible for.
     await Promise.all(Object.values(prevMods).map(function ({mod, domainList}) {
+      // If the module was disabled before there should be any records that we need to clean up
+      if (mod.disabled) { return; }
+
       var oldDomains;
-      if (!curMods[mod.id] || mod.tokenId !== curMods[mod.id].mod.tokenId) {
+      if (!curMods[mod.id] || curMods[mod.id].disabled || mod.tokenId !== curMods[mod.id].mod.tokenId) {
         oldDomains = domainList.slice();
       } else {
         oldDomains = domainList.filter(function (domain) {
@@ -250,7 +227,7 @@ module.exports.create = function (deps, conf) {
         return;
       }
 
-      return getSession(mod.tokenId).then(function (session) {
+      return utils.getSession(mod.tokenId).then(function (session) {
         return dnsCtrl.removeDomains(session, oldDomains);
       });
     }).filter(Boolean));
@@ -260,6 +237,9 @@ module.exports.create = function (deps, conf) {
 
     // And add DNS records for any newly added domains.
     await Promise.all(Object.values(curMods).map(function ({mod, domainList}) {
+      // Don't set any new records if the module has been disabled.
+      if (mod.disabled) { return; }
+
       var newDomains;
       if (!prevMods[mod.id] || mod.tokenId !== prevMods[mod.id].mod.tokenId) {
         newDomains = domainList.slice();
@@ -275,7 +255,7 @@ module.exports.create = function (deps, conf) {
         return;
       }
 
-      return getSession(mod.tokenId).then(function (session) {
+      return utils.getSession(mod.tokenId).then(function (session) {
         return dnsCtrl.setDeviceAddress(session, publicAddress, newDomains);
       });
     }).filter(Boolean));
@@ -341,5 +321,6 @@ module.exports.create = function (deps, conf) {
   , getDeviceAddresses: dnsCtrl.getDeviceAddresses
   , recheckPubAddr:     recheckPubAddr
   , updateConf:         updateConf
+  , challenge
   };
 };

lib/ddns/utils.js

@@ -0,0 +1,102 @@
+'use strict';
+
+module.exports.create = function (deps, conf) {
+
+  async function getSession(id) {
+    var session = await deps.storage.tokens.get(id);
+    if (!session) {
+      throw new Error('no user token with ID "' + id + '"');
+    }
+    return session;
+  }
+
+  function iterateAllModules(action, curConf) {
+    curConf = curConf || conf;
+    var promises = [];
+
+    curConf.domains.forEach(function (dom) {
+      if (!dom.modules || !Array.isArray(dom.modules.ddns) || !dom.modules.ddns.length) {
+        return null;
+      }
+
+      // For the time being all of our things should only be tried once (regardless if it succeeded)
+      // TODO: revisit this behavior when we support multiple ways of setting records, and/or
+      // if we want to allow later modules to run if early modules fail.
+      promises.push(dom.modules.ddns.reduce(function (prom, mod) {
+        if (prom) { return prom; }
+        return action(mod, dom.names);
+      }, null));
+    });
+
+    curConf.ddns.modules.forEach(function (mod) {
+      promises.push(action(mod, mod.domains));
+    });
+
+    return Promise.all(promises.filter(Boolean));
+  }
+
+  var tldCache = {};
+  async function updateTldCache(provider) {
+    var reqObj = {
+      url: deps.OAUTH3.url.normalize(provider) + '/api/com.daplie.domains/prices'
+    , method: 'GET'
+    , json: true
+    };
+
+    var resp = await deps.OAUTH3.request(reqObj);
+    var tldObj = {};
+    resp.data.forEach(function (tldInfo) {
+      if (tldInfo.enabled) {
+        tldObj[tldInfo.tld] = true;
+      }
+    });
+
+    tldCache[provider] = {
+      time: Date.now()
+    , tlds: tldObj
+    };
+    return tldObj;
+  }
+  async function getTlds(provider) {
+    // If we've never cached the results we need to return the promise that will fetch the result,
+    // otherwise we can return the cached value. If the cached value has "expired", we can still
+    // return the cached value we just want to update the cache in parellel (making sure we only
+    // update once).
+    if (!tldCache[provider]) {
+      tldCache[provider] = {
+        updating: true
+      , tlds: updateTldCache(provider)
+      };
+    }
+    if (!tldCache[provider].updating && Date.now() - tldCache[provider].time > 24 * 60 * 60 * 1000) {
+      tldCache[provider].updating = true;
+      updateTldCache(provider);
+    }
+
+    return tldCache[provider].tlds;
+  }
+
+  async function splitDomains(provider, domains) {
+    var tlds = await getTlds(provider);
+    return domains.map(function (domain) {
+      var split = domain.split('.');
+      var tldSegCnt = tlds[split.slice(-2).join('.')] ? 2 : 1;
+
+      // Currently assuming that the sld can't contain dots, and that the tld can have at
+      // most one dot. Not 100% sure this is a valid assumption, but exceptions should be
+      // rare even if the assumption isn't valid.
+      return {
+        tld: split.slice(-tldSegCnt).join('.')
+      , sld: split.slice(-tldSegCnt - 1, -tldSegCnt).join('.')
+      , sub: split.slice(0, -tldSegCnt - 1).join('.')
+      };
+    });
+  }
+
+  return {
+    getSession
+  , iterateAllModules
+  , getTlds
+  , splitDomains
+  };
+};

lib/tcp/index.js

@@ -159,11 +159,13 @@ module.exports.create = function (deps, config) {
     });
   }
 
-  modules = {};
+  process.nextTick(function () {
-  modules.tcpHandler = tcpHandler;
+    modules = {};
-  modules.proxy = require('./proxy-conn').create(deps, config);
+    modules.tcpHandler = tcpHandler;
-  modules.tls   = require('./tls').create(deps, config, modules);
+    modules.proxy = require('./proxy-conn').create(deps, config);
-  modules.http  = require('./http').create(deps, config, modules);
+    modules.tls   = require('./tls').create(deps, config, modules);
+    modules.http  = require('./http').create(deps, config, modules);
+  });
 
   function updateListeners() {
     var current = listeners.list();

lib/tcp/tls.js

@@ -86,8 +86,7 @@ module.exports.create = function (deps, config, tcpMods) {
   , challenges: {
       'http-01': require('le-challenge-fs').create({ debug: config.debug })
     , 'tls-sni-01': require('le-challenge-sni').create({ debug: config.debug })
-      // TODO dns-01
+    , 'dns-01': deps.ddns.challenge
-      //, 'dns-01': require('le-challenge-ddns').create({ debug: config.debug })
     }
   , challengeType: 'http-01'
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "goldilocks",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "The node.js webserver that's just right, Greenlock (HTTPS/TLS/SSL via ACME/Let's Encrypt) and tunneling (RVPN) included.",
   "main": "bin/goldilocks.js",
   "repository": {
@@ -52,7 +52,6 @@
     "js-yaml": "^3.8.3",
     "jsonschema": "^1.2.0",
     "jsonwebtoken": "^7.4.0",
-    "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master",
     "le-challenge-fs": "git+https://git.daplie.com/Daplie/le-challenge-webroot.git#master",
     "le-challenge-sni": "^2.0.1",
     "le-store-certbot": "git+https://git.daplie.com/Daplie/le-store-certbot.git#master",

terms.sh

@@ -1,3 +0,0 @@
-# adding TOS to TXT DNS Record
-daplie dns:set -n _terms._cloud.localhost.foo.daplie.me -t TXT -a '{"url":"oauth3.org/tos/draft","explicit":true}' --ttl 3600
-daplie dns:set -n _terms._cloud.localhost.alpha.daplie.me -t TXT -a '{"url":"oauth3.org/tos/draft","explicit":true}' --ttl 3600

test-chain.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-node serve.js \
-  --port 8443 \
-  --key node_modules/localhost.daplie.me-certificates/privkey.pem \
-  --cert node_modules/localhost.daplie.me-certificates/fullchain.pem \
-  --root node_modules/localhost.daplie.me-certificates/root.pem \
-  -c "$(cat node_modules/localhost.daplie.me-certificates/root.pem)" &
-
-PID=$!
-
-sleep 1
-curl -s --insecure http://localhost.daplie.me:8443 > ./root.pem
-curl -s https://localhost.daplie.me:8443 --cacert ./root.pem
-
-rm ./root.pem
-kill $PID 2>/dev/null