diff --git a/README.md b/README.md index bf53848..277f941 100644 --- a/README.md +++ b/README.md @@ -370,6 +370,41 @@ tunnel: tunnelUrl: 'wss://api.tunnel.example.com/' ``` +**NOTE**: The more common way to use the tunnel with goldilocks is to use the +API to have goldilocks get a token from `oauth3.org`. In order to do this you +will need to have initialized goldilocks with a token that has the `dns` and +`domains` scopes. This is probably easiest to do with the `daplie-desktop-app`, +which will also get the first tunnel token for you. + +**If you want to add more domains** to handle on your device while using the tunnel +you will need to manually get a new token that will tell the tunnel server to +deliver the requests to the new domain(s) to your device. The first step in this +is to attach the new domains to your device. To get the name of the device you +can use the `config` API, but it's probably easiest to `ssh` onto the device and +get the hostname. You can also use the daplie cli tool to see what device name +your other domains are routed to. + +```bash +# for every new domain you want to route attach the domain to your device +daplie devices:attach -n $new_domain -d $device_name +``` + +After that step you will need to use the API to get goldilocks to get a new token +that includes the new domains you attached. It is also recommended but not +required to remove the older token with the incomplete list of domains. Run the +following commands from the unit. + +```bash +# remove the old token +rm /opt/goldilocks/lib/node_modules/goldilocks/var/tokens.json + +# set the "refresh_token" to a bash variable `token` +TOKEN=$(python -mjson.tool /opt/goldilocks/lib/node_modules/goldilocks/var/owners.json | sed -n 's|\s*"refresh_token": "\(.*\)",|\1|p') + +# tell goldilocks to get a new tunnel token +curl -H "authorization: bearer $TOKEN" -X POST https://localhost.admin.daplie.me/api/goldilocks@daplie.com/tunnel +``` + ### ddns TODO diff --git a/bin/goldilocks.js b/bin/goldilocks.js index ab680d0..51d77d2 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -41,15 +41,12 @@ function createStorage(filename, filetype) { } function read() { - return fs.readFileAsync(filename) - .catch(function (err) { - if (err.code === 'ENOENT') { - return ''; - } - return PromiseA.reject(err); - }) - .then(parse) - ; + return fs.readFileAsync(filename).then(parse).catch(function (err) { + if (err.code === 'ENOENT') { + return ''; + } + return PromiseA.reject(err); + }); } var result = { @@ -154,6 +151,9 @@ function fillConfig(config, args) { if (!config.dns) { config.dns = { bind: [ 53 ], modules: [{ name: 'proxy', port: 3053 }] }; } + if (!config.ddns) { + config.ddns = { enabled: false }; + } // Use Object.assign to add any properties needed but not defined in the mdns config. // It will first copy the defaults into an empty object, then copy any real config over that. var mdnsDefaults = { port: 5353, broadcast: '224.0.0.251', ttl: 300 }; @@ -250,25 +250,29 @@ function run(args) { var workers = {}; var cachedConfig; + function updateConfig(config) { + fillConfig(config, args).then(function (config) { + cachedConfig = config; + console.log('changed config', config); + Object.keys(workers).forEach(function (key) { + workers[key].send(cachedConfig); + }); + }); + } + + process.on('SIGHUP', function () { + configStorage.read().then(updateConfig).catch(function (err) { + console.error('error updating config after SIGHUP', err); + }); + }); + cluster.on('message', function (worker, message) { if (message.type !== 'com.daplie.goldilocks/config') { return; } - configStorage.save(message.changes) - .then(function (config) { - return fillConfig(config, args); - }) - .then(function (config) { - cachedConfig = config; - console.log('changed config', config); - Object.keys(workers).forEach(function (key) { - workers[key].send(cachedConfig); - }); - }) - .catch(function (err) { - console.error('error changing config', err); - }) - ; + configStorage.save(message.changes).then(updateConfig).catch(function (err) { + console.error('error changing config', err); + }); }); cluster.on('online', function (worker) { diff --git a/lib/ddns.js b/lib/ddns.js index 2ed1cca..6985e2a 100644 --- a/lib/ddns.js +++ b/lib/ddns.js @@ -1,88 +1,149 @@ 'use strict'; -module.exports.create = function (opts/*, servers*/) { - var PromiseA = opts.PromiseA; - var dns = PromiseA.promisifyAll(require('dns')); +module.exports.create = function (deps, conf) { + var PromiseA = deps.PromiseA; + var request = PromiseA.promisify(require('request')); + var OAUTH3 = require('../packages/assets/org.oauth3'); + require('../packages/assets/org.oauth3/oauth3.dns.js'); + OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); - return PromiseA.all([ - dns.resolve4Async(opts._old_server_name).then(function (results) { - return results; - }, function () {}) - , dns.resolve6Async(opts._old_server_name).then(function (results) { - return results; - }, function () {}) - ]).then(function (results) { - var ipv4 = results[0] || []; - var ipv6 = results[1] || []; - var record; + function dnsType(addr) { + if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) { + return 'A'; + } + if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) { + return 'AAAA'; + } + } - opts.dnsRecords = { - A: ipv4 - , AAAA: ipv6 + async function getSession() { + var sessions = await deps.storage.owners.all(); + var session = sessions.filter(function (sess) { + return sess.token.scp.indexOf('dns') >= 0; + })[0]; + + if (!session) { + throw new Error('no sessions with DNS grants'); + } + + // The OAUTH3 library stores some things on the root session object that we usually + // just leave inside the token, but we need to pull those out before we use it here + session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss; + session.client_uri = session.client_uri || session.token.azp; + session.scope = session.scope || session.token.scp; + return session; + } + + async function setDeviceAddress(addr) { + var session = await getSession(); + var directives = await OAUTH3.discover(session.token.aud); + + // Set the address of the device to our public address. + await request({ + url: directives.api+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname + , method: 'POST' + , headers: { + 'Authorization': 'Bearer ' + session.refresh_token + , 'Accept': 'application/json; charset=utf-8' + } + , json: { + addresses: [ + { value: addr, type: dnsType(addr) } + ] + } + }); + + // Then update all of the records attached to our hostname, first removing the old records + // to remove the reference to the old address, then creating new records for the same domains + // using our new address. + var allDns = OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); + var ourDomains = allDns.filter(function (record) { + return record.device === conf.device.hostname; + }).map(function (record) { + var zoneSplit = record.zone.split('.'); + return { + tld: zoneSplit.slice(1).join('.') + , sld: zoneSplit[0] + , sub: record.host.slice(0, -(record.zone.length + 1)) + }; + }); + + var common = { + api: 'devices.detach' + , session: session + , device: conf.device.hostname }; + await PromiseA.all(ourDomains.map(function (record) { + return OAUTH3.api(directives.api, Object.assign({}, common, record)); + })); - Object.keys(opts.ifaces).some(function (ifacename) { - var iface = opts.ifaces[ifacename]; + common = { + api: 'devices.attach' + , session: session + , device: conf.device.hostname + , ip: addr + , ttl: 300 + }; + await PromiseA.all(ourDomains.map(function (record) { + return OAUTH3.api(directives.api, Object.assign({}, common, record)); + })); + } - return iface.ipv4.some(function (localIp) { - return ipv4.some(function (remoteIp) { - if (localIp.address === remoteIp) { - record = localIp; - return record; - } - }); - }) || iface.ipv6.some(function (localIp) { - return ipv6.forEach(function (remoteIp) { - if (localIp.address === remoteIp) { - record = localIp; - return record; - } - }); - }); + async function getDeviceAddresses() { + var session = await getSession(); + var directives = await OAUTH3.discover(session.token.aud); + + var result = await request({ + url: directives.api+'/api/org.oauth3.dns/acl/devices' + , method: 'GET' + , headers: { + 'Authorization': 'Bearer ' + session.refresh_token + , 'Accept': 'application/json; charset=utf-8' + } + , json: true }); - if (!record) { - console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address."); - console.info("Use --ddns to allow the people of the Internet to access your server."); + if (!result.body) { + throw new Error('No response body in request for device addresses'); + } + if (result.body.error) { + throw Object.assign(new Error('error getting device list'), result.body.error); } - opts.externalIps.ipv4.some(function (localIp) { - return ipv4.some(function (remoteIp) { - if (localIp.address === remoteIp) { - record = localIp; - return record; - } - }); - }); + var dev = result.body.devices.filter(function (dev) { + return dev.name === conf.device.hostname; + })[0]; + return (dev || {}).addresses || []; + } - opts.externalIps.ipv6.some(function (localIp) { - return ipv6.some(function (remoteIp) { - if (localIp.address === remoteIp) { - record = localIp; - return record; - } - }); - }); - - if (!record) { - console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address."); - console.info("Use --ddns to allow the people of the Internet to access your server."); + var publicAddress; + async function recheckPubAddr() { + if (!conf.ddns.enabled) { + return; } - }); -}; -if (require.main === module) { - var opts = { - _old_server_name: 'aj.daplie.me' - , PromiseA: require('bluebird') + var session = await getSession(); + var directives = await OAUTH3.discover(session.token.aud); + var addr = await deps.loopback.checkPublicAddr(directives.api); + + if (publicAddress === addr) { + return; + } + + if (conf.debug) { + console.log('previous public address',publicAddress, 'does not match current public address', addr); + } + + await setDeviceAddress(addr); + publicAddress = addr; + } + + recheckPubAddr(); + setInterval(recheckPubAddr, 5*60*1000); + + return { + setDeviceAddress: setDeviceAddress + , getDeviceAddresses: getDeviceAddresses + , recheckPubAddr: recheckPubAddr }; - // ifaces - opts.ifaces = require('./local-ip.js').find(); - console.log('opts.ifaces'); - console.log(opts.ifaces); - require('./match-ips.js').match(opts._old_server_name, opts).then(function (ips) { - opts.matchingIps = ips.matchingIps || []; - opts.externalIps = ips.externalIps; - module.exports.create(opts); - }); -} +}; diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 732678b..a0b3fcc 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -50,7 +50,7 @@ module.exports.create = function (deps, config) { } console.warn('failed to identify protocol from first chunk', firstChunk); - conn.close(); + conn.destroy(); } function netHandler(conn, opts) { function getProp(name) { diff --git a/lib/loopback.js b/lib/loopback.js index fd827cf..6b4c389 100644 --- a/lib/loopback.js +++ b/lib/loopback.js @@ -1,84 +1,88 @@ 'use strict'; -module.exports.create = function (deps) { +module.exports.create = function (deps, conf) { var PromiseA = require('bluebird'); var request = PromiseA.promisify(require('request')); var pending = {}; - function checkPublicAddr(host) { - return request({ + async function checkPublicAddr(host) { + var result = await request({ method: 'GET' , url: host+'/api/org.oauth3.tunnel/checkip' , json: true - }).then(function (result) { - if (!result.body) { - return PromiseA.reject(new Error('No response body in request for public address')); - } - if (result.body.error) { - var err = new Error(result.body.error.message); - return PromiseA.reject(Object.assign(err, result.body.error)); - } - return result.body.address; }); + + if (!result.body) { + throw new Error('No response body in request for public address'); + } + if (result.body.error) { + // Note that the error on the body will probably have a message that overwrites the default + throw Object.assign(new Error('error in check IP response'), result.body.error); + } + return result.body.address; } - function checkSinglePort(host, address, port) { + async function checkSinglePort(host, address, port) { var crypto = require('crypto'); var token = crypto.randomBytes(8).toString('hex'); var keyAuth = crypto.randomBytes(32).toString('hex'); pending[token] = keyAuth; - var opts = { - address: address - , port: port - , token: token - , keyAuthorization: keyAuth - , iat: Date.now() - }; - - return request({ + var reqObj = { method: 'POST' , url: host+'/api/org.oauth3.tunnel/loopback' - , json: opts - }) - .then(function (result) { - delete pending[token]; - if (!result.body) { - return PromiseA.reject(new Error('No response body in loopback request for port '+port)); + , json: { + address: address + , port: port + , token: token + , keyAuthorization: keyAuth + , iat: Date.now() } - // If the loopback requests don't go to us then there are all kinds of ways it could - // error, but none of them really provide much extra information so we don't do - // anything that will break the PromiseA.all out and mask the other results. - if (result.body.error) { - console.log('error on remote side of port '+port+' loopback', result.body.error); - } - return !!result.body.success; - }, function (err) { + }; + + var result; + try { + result = await request(reqObj); + } catch (err) { delete pending[token]; throw err; - }); + } + + delete pending[token]; + if (!result.body) { + throw new Error('No response body in loopback request for port '+port); + } + // If the loopback requests don't go to us then there are all kinds of ways it could + // error, but none of them really provide much extra information so we don't do + // anything that will break the PromiseA.all out and mask the other results. + if (conf.debug && result.body.error) { + console.log('error on remote side of port '+port+' loopback', result.body.error); + } + return !!result.body.success; } - function loopback(provider) { - return deps.OAUTH3.discover(provider).then(function (directives) { - return checkPublicAddr(directives.api).then(function (address) { - console.log('checking to see if', address, 'gets back to us'); - var ports = require('./servers').listeners.tcp.list(); - return PromiseA.all(ports.map(function (port) { - return checkSinglePort(directives.api, address, port); - })) - .then(function (values) { - console.log(pending); - var result = {error: null, address: address}; - ports.forEach(function (port, ind) { - result[port] = values[ind]; - }); - return result; - }); - }); + async function loopback(provider) { + var directives = await deps.OAUTH3.discover(provider); + var address = await checkPublicAddr(directives.api); + console.log('checking to see if', address, 'gets back to us'); + + var ports = require('./servers').listeners.tcp.list(); + var values = await PromiseA.all(ports.map(function (port) { + return checkSinglePort(directives.api, address, port); + })); + + if (conf.debug) { + console.log('remaining loopback tokens', pending); + } + + var result = {error: null, address: address}; + ports.forEach(function (port, ind) { + result[port] = values[ind]; }); + return result; } + loopback.checkPublicAddr = checkPublicAddr; loopback.server = require('http').createServer(function (req, res) { var parsed = require('url').parse(req.url); var token = parsed.pathname.replace('/.well-known/cloud-challenge/', ''); diff --git a/lib/match-ips.js b/lib/match-ips.js deleted file mode 100644 index dbb3ff1..0000000 --- a/lib/match-ips.js +++ /dev/null @@ -1,117 +0,0 @@ -'use strict'; - -var PromiseA = require('bluebird'); - -module.exports.match = function (servername, opts) { - return PromiseA.promisify(require('ipify'))().then(function (externalIp) { - var dns = PromiseA.promisifyAll(require('dns')); - - opts.externalIps = [ { address: externalIp, family: 'IPv4' } ]; - opts.ifaces = require('./local-ip.js').find({ externals: opts.externalIps }); - opts.externalIfaces = Object.keys(opts.ifaces).reduce(function (all, iname) { - var iface = opts.ifaces[iname]; - - iface.ipv4.forEach(function (addr) { - if (addr.external) { - addr.iface = iname; - all.push(addr); - } - }); - iface.ipv6.forEach(function (addr) { - if (addr.external) { - addr.iface = iname; - all.push(addr); - } - }); - - return all; - }, []).filter(Boolean); - - function resolveIps(hostname) { - var allIps = []; - - return PromiseA.all([ - dns.resolve4Async(hostname).then(function (records) { - records.forEach(function (ip) { - allIps.push({ - address: ip - , family: 'IPv4' - }); - }); - }, function () {}) - , dns.resolve6Async(hostname).then(function (records) { - records.forEach(function (ip) { - allIps.push({ - address: ip - , family: 'IPv6' - }); - }); - }, function () {}) - ]).then(function () { - return allIps; - }); - } - - function resolveIpsAndCnames(hostname) { - return PromiseA.all([ - resolveIps(hostname) - , dns.resolveCnameAsync(hostname).then(function (records) { - return PromiseA.all(records.map(function (hostname) { - return resolveIps(hostname); - })).then(function (allIps) { - return allIps.reduce(function (all, ips) { - return all.concat(ips); - }, []); - }); - }, function () { - return []; - }) - ]).then(function (ips) { - return ips.reduce(function (all, set) { - return all.concat(set); - }, []); - }); - } - - return resolveIpsAndCnames(servername).then(function (allIps) { - var matchingIps = []; - - if (!allIps.length) { - console.warn("Could not resolve '" + servername + "'"); - } - - // { address, family } - allIps.some(function (ip) { - function match(addr) { - if (ip.address === addr.address) { - matchingIps.push(addr); - } - } - - opts.externalIps.forEach(match); - // opts.externalIfaces.forEach(match); - - Object.keys(opts.ifaces).forEach(function (iname) { - var iface = opts.ifaces[iname]; - - iface.ipv4.forEach(match); - iface.ipv6.forEach(match); - }); - - return matchingIps.length; - }); - - matchingIps.externalIps = { - ipv4: [ - { address: externalIp - , family: 'IPv4' - } - ] - , ipv6: [ - ] - }; - matchingIps.matchingIps = matchingIps; - return matchingIps; - }); - }); -}; diff --git a/lib/modules/http.js b/lib/modules/http.js index 048d3ac..636b6bc 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -29,12 +29,14 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { function handleChunk(chunk) { if (!errored) { opts.firstChunk = Buffer.concat([opts.firstChunk, chunk]); - if (opts.firstChunk.includes('\r\n\r\n')) { - resolve(opts.firstChunk.toString()); - conn.removeListener('error', handleErr); - } else { + if (!opts.firstChunk.includes('\r\n\r\n')) { conn.once('data', handleChunk); + return; } + + conn.removeListener('error', handleErr); + conn.pause(); + resolve(opts.firstChunk.toString()); } } conn.once('data', handleChunk); @@ -144,6 +146,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { // data comes in. process.nextTick(function () { conn.unshift(opts.firstChunk); + conn.resume(); }); // Convenience return for all the check* functions. @@ -160,6 +163,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { deps.tunnelServer.handleClientConn(conn); process.nextTick(function () { conn.unshift(opts.firstChunk); + conn.resume(); }); return true; } @@ -211,6 +215,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { deps.tunnelServer.handleAdminConn(conn); process.nextTick(function () { conn.unshift(opts.firstChunk); + conn.resume(); }); return true; } diff --git a/lib/worker.js b/lib/worker.js index 38816e1..56991bc 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -22,6 +22,7 @@ function create(conf) { config = conf; var deps = { messenger: process + , PromiseA: require('bluebird') // Note that if a custom createConnections is used it will be called with different // sets of custom options based on what is actually being proxied. Most notably the // HTTP proxying connection creation is not something we currently control. @@ -31,6 +32,7 @@ function create(conf) { deps.proxy = require('./proxy-conn').create(deps, conf); deps.socks5 = require('./socks5-server').create(deps, conf); deps.loopback = require('./loopback').create(deps, conf); + deps.ddns = require('./ddns').create(deps, conf); require('./goldilocks.js').create(deps, conf); process.removeListener('message', create); diff --git a/package.json b/package.json index 8d7c537..c5e56ee 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,6 @@ "http-proxy": "^1.16.2", "human-readable-ids": "git+https://git.daplie.com/Daplie/human-readable-ids-js#master", "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0", - "ipify": "^1.1.0", "js-yaml": "^3.8.3", "jsonwebtoken": "^7.4.0", "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master",