From 8930a528bc4ab224453b8b0bab85815a0fa86aff Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 25 Sep 2017 18:06:48 -0600 Subject: [PATCH 01/41] moved some things related to DDNS into separate folder --- lib/app.js | 21 +------------ lib/{ddns.js => ddns/index.js} | 31 +++++++++----------- lib/{ => ddns}/loopback.js | 14 ++++----- lib/modules/http.js | 2 +- lib/worker.js | 13 ++++++-- packages/apis/com.daplie.goldilocks/index.js | 23 --------------- 6 files changed, 34 insertions(+), 70 deletions(-) rename lib/{ddns.js => ddns/index.js} (78%) rename lib/{ => ddns}/loopback.js (88%) diff --git a/lib/app.js b/lib/app.js index f633712..356f0da 100644 --- a/lib/app.js +++ b/lib/app.js @@ -15,25 +15,6 @@ module.exports = function (myDeps, conf, overrideHttp) { //var server; var goldilocksApis; var app; - var request; - - function createGoldilocksApis() { - var PromiseA = require('bluebird'); - var OAUTH3 = require('../packages/assets/org.oauth3'); - require('../packages/assets/org.oauth3/oauth3.domains.js'); - require('../packages/assets/org.oauth3/oauth3.dns.js'); - require('../packages/assets/org.oauth3/oauth3.tunnel.js'); - OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); - - request = request || PromiseA.promisify(require('request')); - - myDeps.PromiseA = PromiseA; - myDeps.OAUTH3 = OAUTH3; - myDeps.recase = require('recase').create({}); - myDeps.request = request; - - return require('../packages/apis/com.daplie.goldilocks').create(myDeps, conf); - } app = express(); @@ -116,7 +97,7 @@ module.exports = function (myDeps, conf, overrideHttp) { function _goldApis(req, res, next) { if (!goldilocksApis) { - goldilocksApis = createGoldilocksApis(); + goldilocksApis = require('../packages/apis/com.daplie.goldilocks').create(myDeps, conf); } if (typeof goldilocksApis[req.params.name] === 'function') { diff --git a/lib/ddns.js b/lib/ddns/index.js similarity index 78% rename from lib/ddns.js rename to lib/ddns/index.js index 6985e2a..2637233 100644 --- a/lib/ddns.js +++ b/lib/ddns/index.js @@ -1,11 +1,7 @@ 'use strict'; module.exports.create = function (deps, conf) { - var PromiseA = deps.PromiseA; - var request = PromiseA.promisify(require('request')); - var OAUTH3 = require('../packages/assets/org.oauth3'); - require('../packages/assets/org.oauth3/oauth3.dns.js'); - OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); + var loopback = require('./loopback').create(deps, conf); function dnsType(addr) { if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) { @@ -36,10 +32,10 @@ module.exports.create = function (deps, conf) { async function setDeviceAddress(addr) { var session = await getSession(); - var directives = await OAUTH3.discover(session.token.aud); + var directives = await deps.OAUTH3.discover(session.token.aud); // Set the address of the device to our public address. - await request({ + await deps.request({ url: directives.api+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname , method: 'POST' , headers: { @@ -56,7 +52,7 @@ module.exports.create = function (deps, conf) { // Then update all of the records attached to our hostname, first removing the old records // to remove the reference to the old address, then creating new records for the same domains // using our new address. - var allDns = OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); + var allDns = deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); var ourDomains = allDns.filter(function (record) { return record.device === conf.device.hostname; }).map(function (record) { @@ -73,8 +69,8 @@ module.exports.create = function (deps, conf) { , session: session , device: conf.device.hostname }; - await PromiseA.all(ourDomains.map(function (record) { - return OAUTH3.api(directives.api, Object.assign({}, common, record)); + await deps.PromiseA.all(ourDomains.map(function (record) { + return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); })); common = { @@ -84,16 +80,16 @@ module.exports.create = function (deps, conf) { , ip: addr , ttl: 300 }; - await PromiseA.all(ourDomains.map(function (record) { - return OAUTH3.api(directives.api, Object.assign({}, common, record)); + await deps.PromiseA.all(ourDomains.map(function (record) { + return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); })); } async function getDeviceAddresses() { var session = await getSession(); - var directives = await OAUTH3.discover(session.token.aud); + var directives = await deps.OAUTH3.discover(session.token.aud); - var result = await request({ + var result = await deps.request({ url: directives.api+'/api/org.oauth3.dns/acl/devices' , method: 'GET' , headers: { @@ -123,8 +119,8 @@ module.exports.create = function (deps, conf) { } var session = await getSession(); - var directives = await OAUTH3.discover(session.token.aud); - var addr = await deps.loopback.checkPublicAddr(directives.api); + var directives = await deps.OAUTH3.discover(session.token.aud); + var addr = await loopback.checkPublicAddr(directives.api); if (publicAddress === addr) { return; @@ -142,7 +138,8 @@ module.exports.create = function (deps, conf) { setInterval(recheckPubAddr, 5*60*1000); return { - setDeviceAddress: setDeviceAddress + loopbackServer: loopback.server + , setDeviceAddress: setDeviceAddress , getDeviceAddresses: getDeviceAddresses , recheckPubAddr: recheckPubAddr }; diff --git a/lib/loopback.js b/lib/ddns/loopback.js similarity index 88% rename from lib/loopback.js rename to lib/ddns/loopback.js index 6b4c389..2f7772d 100644 --- a/lib/loopback.js +++ b/lib/ddns/loopback.js @@ -1,12 +1,10 @@ 'use strict'; module.exports.create = function (deps, conf) { - var PromiseA = require('bluebird'); - var request = PromiseA.promisify(require('request')); var pending = {}; async function checkPublicAddr(host) { - var result = await request({ + var result = await deps.request({ method: 'GET' , url: host+'/api/org.oauth3.tunnel/checkip' , json: true @@ -42,7 +40,7 @@ module.exports.create = function (deps, conf) { var result; try { - result = await request(reqObj); + result = await deps.request(reqObj); } catch (err) { delete pending[token]; throw err; @@ -64,10 +62,12 @@ module.exports.create = function (deps, conf) { async function loopback(provider) { var directives = await deps.OAUTH3.discover(provider); var address = await checkPublicAddr(directives.api); - console.log('checking to see if', address, 'gets back to us'); + if (conf.debug) { + console.log('checking to see if', address, 'gets back to us'); + } - var ports = require('./servers').listeners.tcp.list(); - var values = await PromiseA.all(ports.map(function (port) { + var ports = require('../servers').listeners.tcp.list(); + var values = await deps.PromiseA.all(ports.map(function (port) { return checkSinglePort(directives.api, address, port); })); diff --git a/lib/modules/http.js b/lib/modules/http.js index 636b6bc..3b94d46 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -178,7 +178,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { if (headers.url.indexOf('/.well-known/cloud-challenge/') !== 0) { return false; } - return emitConnection(deps.loopback.server, conn, opts); + return emitConnection(deps.ddns.loopbackServer, conn, opts); } var httpsRedirectServer; diff --git a/lib/worker.js b/lib/worker.js index 56991bc..44f4507 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -19,10 +19,20 @@ function update(conf) { } function create(conf) { + var PromiseA = require('bluebird'); + var OAUTH3 = require('../packages/assets/org.oauth3'); + require('../packages/assets/org.oauth3/oauth3.domains.js'); + require('../packages/assets/org.oauth3/oauth3.dns.js'); + require('../packages/assets/org.oauth3/oauth3.tunnel.js'); + OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); + config = conf; var deps = { messenger: process - , PromiseA: require('bluebird') + , PromiseA: PromiseA + , OAUTH3: OAUTH3 + , request: PromiseA.promisify(require('request')) + , recase: require('recase').create({}) // Note that if a custom createConnections is used it will be called with different // sets of custom options based on what is actually being proxied. Most notably the // HTTP proxying connection creation is not something we currently control. @@ -31,7 +41,6 @@ function create(conf) { deps.storage = require('./storage').create(deps, conf); deps.proxy = require('./proxy-conn').create(deps, conf); deps.socks5 = require('./socks5-server').create(deps, conf); - deps.loopback = require('./loopback').create(deps, conf); deps.ddns = require('./ddns').create(deps, conf); require('./goldilocks.js').create(deps, conf); diff --git a/packages/apis/com.daplie.goldilocks/index.js b/packages/apis/com.daplie.goldilocks/index.js index 3c3e54c..8dfbd00 100644 --- a/packages/apis/com.daplie.goldilocks/index.js +++ b/packages/apis/com.daplie.goldilocks/index.js @@ -315,29 +315,6 @@ module.exports.create = function (deps, conf) { }); }); } - , loopback: function (req, res) { - if (handleCors(req, res, 'GET')) { - return; - } - isAuthorized(req, res, function () { - var prom; - var query = require('querystring').parse(require('url').parse(req.url).query); - if (query.provider) { - prom = deps.loopback(query.provider); - } else { - prom = deps.storage.owners.get(req.userId).then(function (session) { - return deps.loopback(session.token.aud); - }); - } - - res.setHeader('Content-Type', 'application/json'); - prom.then(function (result) { - res.end(JSON.stringify(result)); - }, function (err) { - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); - }); - }); - } , paywall_check: function (req, res) { if (handleCors(req, res, 'GET')) { return; From 83f72730a22d014c9685194bd85d483586c61c29 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 27 Sep 2017 10:54:35 -0600 Subject: [PATCH 02/41] moved the DNS API calls to another file --- lib/ddns/dns-ctrl.js | 97 +++++++++++++++++++++++++++++++++++++++++++ lib/ddns/index.js | 98 ++------------------------------------------ 2 files changed, 101 insertions(+), 94 deletions(-) create mode 100644 lib/ddns/dns-ctrl.js diff --git a/lib/ddns/dns-ctrl.js b/lib/ddns/dns-ctrl.js new file mode 100644 index 0000000..500e590 --- /dev/null +++ b/lib/ddns/dns-ctrl.js @@ -0,0 +1,97 @@ +'use strict'; + +module.exports.create = function (deps, conf) { + function dnsType(addr) { + if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) { + return 'A'; + } + if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) { + return 'AAAA'; + } + } + + async function setDeviceAddress(session, addr) { + var directives = await deps.OAUTH3.discover(session.token.aud); + + // Set the address of the device to our public address. + await deps.request({ + url: deps.OAUTH3.url.normalize(directives.api)+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname + , method: 'POST' + , headers: { + 'Authorization': 'Bearer ' + session.refresh_token + , 'Accept': 'application/json; charset=utf-8' + } + , json: { + addresses: [ + { value: addr, type: dnsType(addr) } + ] + } + }); + + // Then update all of the records attached to our hostname, first removing the old records + // to remove the reference to the old address, then creating new records for the same domains + // using our new address. + var allDns = deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); + var ourDomains = allDns.filter(function (record) { + return record.device === conf.device.hostname; + }).map(function (record) { + var zoneSplit = record.zone.split('.'); + return { + tld: zoneSplit.slice(1).join('.') + , sld: zoneSplit[0] + , sub: record.host.slice(0, -(record.zone.length + 1)) + }; + }); + + var common = { + api: 'devices.detach' + , session: session + , device: conf.device.hostname + }; + await deps.PromiseA.all(ourDomains.map(function (record) { + return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); + })); + + common = { + api: 'devices.attach' + , session: session + , device: conf.device.hostname + , ip: addr + , ttl: 300 + }; + await deps.PromiseA.all(ourDomains.map(function (record) { + return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); + })); + } + + async function getDeviceAddresses(session) { + var directives = await deps.OAUTH3.discover(session.token.aud); + + var result = await deps.request({ + url: deps.OAUTH3.url.normalize(directives.api)+'/api/org.oauth3.dns/acl/devices' + , method: 'GET' + , headers: { + 'Authorization': 'Bearer ' + session.refresh_token + , 'Accept': 'application/json; charset=utf-8' + } + , json: true + }); + + if (!result.body) { + throw new Error('No response body in request for device addresses'); + } + if (result.body.error) { + throw Object.assign(new Error('error getting device list'), result.body.error); + } + + var dev = result.body.devices.filter(function (dev) { + return dev.name === conf.device.hostname; + })[0]; + return (dev || {}).addresses || []; + } + + return { + getDeviceAddresses: getDeviceAddresses + , setDeviceAddress: setDeviceAddress +, }; +}; diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 2637233..5163ef6 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -2,15 +2,7 @@ module.exports.create = function (deps, conf) { var loopback = require('./loopback').create(deps, conf); - - function dnsType(addr) { - if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) { - return 'A'; - } - if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) { - return 'AAAA'; - } - } + var dnsCtrl = require('./dns-ctrl').create(deps, conf); async function getSession() { var sessions = await deps.storage.owners.all(); @@ -30,88 +22,6 @@ module.exports.create = function (deps, conf) { return session; } - async function setDeviceAddress(addr) { - var session = await getSession(); - var directives = await deps.OAUTH3.discover(session.token.aud); - - // Set the address of the device to our public address. - await deps.request({ - url: directives.api+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname - , method: 'POST' - , headers: { - 'Authorization': 'Bearer ' + session.refresh_token - , 'Accept': 'application/json; charset=utf-8' - } - , json: { - addresses: [ - { value: addr, type: dnsType(addr) } - ] - } - }); - - // Then update all of the records attached to our hostname, first removing the old records - // to remove the reference to the old address, then creating new records for the same domains - // using our new address. - var allDns = deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); - var ourDomains = allDns.filter(function (record) { - return record.device === conf.device.hostname; - }).map(function (record) { - var zoneSplit = record.zone.split('.'); - return { - tld: zoneSplit.slice(1).join('.') - , sld: zoneSplit[0] - , sub: record.host.slice(0, -(record.zone.length + 1)) - }; - }); - - var common = { - api: 'devices.detach' - , session: session - , device: conf.device.hostname - }; - await deps.PromiseA.all(ourDomains.map(function (record) { - return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); - })); - - common = { - api: 'devices.attach' - , session: session - , device: conf.device.hostname - , ip: addr - , ttl: 300 - }; - await deps.PromiseA.all(ourDomains.map(function (record) { - return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); - })); - } - - async function getDeviceAddresses() { - var session = await getSession(); - var directives = await deps.OAUTH3.discover(session.token.aud); - - var result = await deps.request({ - url: directives.api+'/api/org.oauth3.dns/acl/devices' - , method: 'GET' - , headers: { - 'Authorization': 'Bearer ' + session.refresh_token - , 'Accept': 'application/json; charset=utf-8' - } - , json: true - }); - - if (!result.body) { - throw new Error('No response body in request for device addresses'); - } - if (result.body.error) { - throw Object.assign(new Error('error getting device list'), result.body.error); - } - - var dev = result.body.devices.filter(function (dev) { - return dev.name === conf.device.hostname; - })[0]; - return (dev || {}).addresses || []; - } - var publicAddress; async function recheckPubAddr() { if (!conf.ddns.enabled) { @@ -130,7 +40,7 @@ module.exports.create = function (deps, conf) { console.log('previous public address',publicAddress, 'does not match current public address', addr); } - await setDeviceAddress(addr); + await dnsCtrl.setDeviceAddress(session, addr); publicAddress = addr; } @@ -139,8 +49,8 @@ module.exports.create = function (deps, conf) { return { loopbackServer: loopback.server - , setDeviceAddress: setDeviceAddress - , getDeviceAddresses: getDeviceAddresses + , setDeviceAddress: dnsCtrl.setDeviceAddress + , getDeviceAddresses: dnsCtrl.getDeviceAddresses , recheckPubAddr: recheckPubAddr }; }; From 5cc7e3f187a33b0ebb7aff716383e25015f65f7a Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 27 Sep 2017 14:53:18 -0600 Subject: [PATCH 03/41] added loopback test before setting DNS records to local IP --- lib/ddns/index.js | 36 +++++++++++++++++++++++++++ lib/ddns/loopback.js | 16 ++++++------ package-lock.json | 59 ++++++++++++++++++++++++++++++++++++++------ package.json | 1 + 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 5163ef6..5c71a78 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -1,9 +1,41 @@ 'use strict'; module.exports.create = function (deps, conf) { + var network = deps.PromiseA.promisify(deps.recase.camelCopy(require('network'))); var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); + var localAddr, gateway, accessible; + async function checkNetworkEnv() { + // Since we can't detect the OS level events when a user plugs in an ethernet cable to recheck + // what network environment we are in we check our local network address and the gateway to + // determine if we need to run the loopback check and router configuration again. + var gw = await network.getGatewayIpAsync(); + var addr = await network.getPrivateIpAsync(); + if (localAddr === addr && gateway === gw) { + return; + } + + localAddr = addr; + gateway = gw; + var loopResult = await loopback('seth.daplie.me'); + var notLooped = Object.keys(loopResult.ports).filter(function (port) { + return !loopResult.ports[port]; + }); + + // All ports come back to us, so we are either a public address or the router has already + // been configured to forward these ports to us, so no configuration needs to be done we + // just have to make sure the DNS records stay in sync with our public address. + if (!notLooped.length) { + accessible = true; + return; + } + + // TODO: try to automatically configure router to forward ports to us. + accessible = false; + // TODO: move tunnel client here as fall back. + } + async function getSession() { var sessions = await deps.storage.owners.all(); var session = sessions.filter(function (sess) { @@ -28,6 +60,10 @@ module.exports.create = function (deps, conf) { return; } + await checkNetworkEnv(); + if (!accessible) { + return; + } var session = await getSession(); var directives = await deps.OAUTH3.discover(session.token.aud); var addr = await loopback.checkPublicAddr(directives.api); diff --git a/lib/ddns/loopback.js b/lib/ddns/loopback.js index 2f7772d..c35c742 100644 --- a/lib/ddns/loopback.js +++ b/lib/ddns/loopback.js @@ -6,7 +6,7 @@ module.exports.create = function (deps, conf) { async function checkPublicAddr(host) { var result = await deps.request({ method: 'GET' - , url: host+'/api/org.oauth3.tunnel/checkip' + , url: deps.OAUTH3.url.normalize(host)+'/api/org.oauth3.tunnel/checkip' , json: true }); @@ -28,7 +28,7 @@ module.exports.create = function (deps, conf) { var reqObj = { method: 'POST' - , url: host+'/api/org.oauth3.tunnel/loopback' + , url: deps.OAUTH3.url.normalize(host)+'/api/org.oauth3.tunnel/loopback' , json: { address: address , port: port @@ -75,11 +75,13 @@ module.exports.create = function (deps, conf) { console.log('remaining loopback tokens', pending); } - var result = {error: null, address: address}; - ports.forEach(function (port, ind) { - result[port] = values[ind]; - }); - return result; + return { + address: address + , ports: ports.reduce(function (obj, port, ind) { + obj[port] = values[ind]; + return obj; + }, {}) + }; } loopback.checkPublicAddr = checkPublicAddr; diff --git a/package-lock.json b/package-lock.json index 11aca53..02dd078 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1168,6 +1168,11 @@ "resolved": "https://registry.npmjs.org/localhost.daplie.me-certificates/-/localhost.daplie.me-certificates-1.3.5.tgz", "integrity": "sha1-GjqH5PlX8mn2LP7mCmNpe9JVOpo=" }, + "lodash": { + "version": "4.17.4", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", + "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" + }, "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -1301,11 +1306,38 @@ "integrity": "sha1-5P805slf37WuzAjeZZb0NgWn20U=", "optional": true }, + "needle": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/needle/-/needle-1.1.2.tgz", + "integrity": "sha1-0oQaElv9dP77MMA0QQQ2kGHD4To=", + "requires": { + "debug": "2.6.1", + "iconv-lite": "0.4.15" + } + }, "negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "network": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/network/-/network-0.4.0.tgz", + "integrity": "sha1-ngk+TZzpBjmHJTL6YC/oVf87aSk=", + "requires": { + "async": "1.5.2", + "commander": "2.9.0", + "needle": "1.1.2", + "wmic": "0.1.0" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + } + } + }, "node-forge": { "version": "0.6.49", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.6.49.tgz", @@ -1967,14 +1999,6 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" }, - "stream-pair": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-pair/-/stream-pair-1.0.3.tgz", - "integrity": "sha1-vIdY/jnTgQuva3VMj5BI8PuRNn0=", - "requires": { - "readable-stream": "2.2.11" - } - }, "string_decoder": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.2.tgz", @@ -2233,6 +2257,25 @@ } } }, + "wmic": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/wmic/-/wmic-0.1.0.tgz", + "integrity": "sha1-eLQasR0VTLgSgZ4SkWdNrVXY4dc=", + "requires": { + "async": "2.5.0", + "iconv-lite": "0.4.15" + }, + "dependencies": { + "async": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.5.0.tgz", + "integrity": "sha512-e+lJAJeNWuPCNyxZKOBdaJGyLGHugXVQtrAwtuAe2vhxTYxFTKE73p8JuTmdH0qdQZtDvI4dhJwjZc5zsfIsYw==", + "requires": { + "lodash": "4.17.4" + } + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index c5e56ee..e026a14 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "le-challenge-sni": "^2.0.1", "le-store-certbot": "git+https://git.daplie.com/Daplie/le-store-certbot.git#master", "localhost.daplie.me-certificates": "^1.3.5", + "network": "^0.4.0", "recase": "git+https://git.daplie.com/coolaj86/recase-js.git#v1.0.4", "redirect-https": "^1.1.0", "request": "^2.81.0", From 0dd20e4dfc9017e7b0649eaa97fe215f7604e87a Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 28 Sep 2017 11:18:44 -0600 Subject: [PATCH 04/41] removed tunnel from config and API and made DDNS responsible --- API.md | 18 -- README.md | 14 -- bin/goldilocks.js | 6 +- etc/goldilocks/goldilocks.example.yml | 12 +- lib/ddns/index.js | 33 +-- lib/tunnel-client-manager.js | 205 +++++-------------- packages/apis/com.daplie.goldilocks/index.js | 27 --- 7 files changed, 75 insertions(+), 240 deletions(-) diff --git a/API.md b/API.md index bfb6c54..6495f06 100644 --- a/API.md +++ b/API.md @@ -9,24 +9,6 @@ localhost.admin.daplie.me All requests require an OAuth3 token in the request headers. -## Tunnel - -### Check Status - * **URL** `/api/goldilocks@daplie.com/tunnel` - * **Method** `POST` - * **Reponse**: An object whose keys are the URLs for the tunnels, and whose - properties are arrays of the tunnel tokens. - - This route with return only the sessions started by the same user who is - checking the status. - -### Start Tunnel - * **URL** `/api/goldilocks@daplie.com/tunnel` - * **Method** `POST` - - This route will use the stored token for the user matching the request - header to request a tunnel token from the audience of the stored token. - ## Socks5 Proxy ### Check Status diff --git a/README.md b/README.md index dae6727..4491569 100644 --- a/README.md +++ b/README.md @@ -356,20 +356,6 @@ The tunnel client is meant to be run from behind a firewalls, carrier-grade NAT, or otherwise inaccessible devices to allow them to be accessed publicly on the internet. -It has no options per se, but is rather a list of tokens that can be used to -connect to tunnel servers. If the token does not have an `aud` field it must be -provided in an object with the token provided in the `jwt` field and the tunnel -server url provided in the `tunnelUrl` field. - -Example config: - -```yml -tunnel: - - 'some.jwt_encoded.token' - - jwt: 'other.jwt_encoded.token' - tunnelUrl: 'wss://api.tunnel.example.com/' -``` - ### ddns TODO diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 51d77d2..4e651dc 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -212,8 +212,6 @@ function fillConfig(config, args) { config.addresses = addresses; config.device = { hostname: require('os').hostname() }; - config.tunnel = args.tunnel || config.tunnel; - if (Array.isArray(config.tcp.bind)) { return PromiseA.resolve(config); } @@ -310,8 +308,7 @@ function readEnv(args) { } catch (err) {} var env = { - tunnel: process.env.GOLDILOCKS_TUNNEL_TOKEN || process.env.GOLDILOCKS_TUNNEL && true - , email: process.env.GOLDILOCKS_EMAIL + email: process.env.GOLDILOCKS_EMAIL , cwd: process.env.GOLDILOCKS_HOME || process.cwd() , debug: process.env.GOLDILOCKS_DEBUG && true }; @@ -325,7 +322,6 @@ program .version(require('../package.json').version) .option('--agree-tos [url1,url2]', "agree to all Terms of Service for Daplie, Let's Encrypt, etc (or specific URLs only)") .option('-c --config ', 'Path to config file (Goldilocks.json or Goldilocks.yml) example: --config /etc/goldilocks/Goldilocks.json') - .option('--tunnel [token]', 'Turn tunnel on. This will enter interactive mode for login if no token is specified.') .option('--email ', "(Re)set default email to use for Daplie, Let's Encrypt, ACME, etc.") .option('--debug', "Enable debug output") .parse(process.argv); diff --git a/etc/goldilocks/goldilocks.example.yml b/etc/goldilocks/goldilocks.example.yml index 38ec6b5..86ea64a 100644 --- a/etc/goldilocks/goldilocks.example.yml +++ b/etc/goldilocks/goldilocks.example.yml @@ -9,11 +9,6 @@ tcp: - 22 address: '127.0.0.1:8022' -# tunnel: jwt -# tunnel: -# - jwt1 -# - jwt2 - tunnel_server: secret: abc123 servernames: @@ -91,3 +86,10 @@ mdns: port: 5353 broadcast: '224.0.0.251' ttl: 300 + +ddns: + enabled: true + domains: + - www.example.com + - api.example.com + - test.example.com diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 5c71a78..e63f131 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -5,7 +5,8 @@ module.exports.create = function (deps, conf) { var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); - var localAddr, gateway, accessible; + var localAddr, gateway; + var tunnelActive = false; async function checkNetworkEnv() { // Since we can't detect the OS level events when a user plugs in an ethernet cable to recheck // what network environment we are in we check our local network address and the gateway to @@ -23,17 +24,25 @@ module.exports.create = function (deps, conf) { return !loopResult.ports[port]; }); - // All ports come back to us, so we are either a public address or the router has already - // been configured to forward these ports to us, so no configuration needs to be done we - // just have to make sure the DNS records stay in sync with our public address. - if (!notLooped.length) { - accessible = true; - return; - } + // if (notLooped.length) { + // // TODO: try to automatically configure router to forward ports to us. + // } - // TODO: try to automatically configure router to forward ports to us. - accessible = false; - // TODO: move tunnel client here as fall back. + // If we are on a public accress or all ports we are listening on are forwarded to us then + // we don't need the tunnel and we can set the DNS records for all our domains to our public + // address. Otherwise we need to use the tunnel to accept traffic. + if (!notLooped.length) { + if (tunnelActive) { + deps.tunnelClients.disconnect(); + tunnelActive = false; + } + } else { + if (!tunnelActive) { + var session = await getSession(); + await deps.tunnelClients.start(session, conf.dns.domains); + tunnelActive = true; + } + } } async function getSession() { @@ -61,7 +70,7 @@ module.exports.create = function (deps, conf) { } await checkNetworkEnv(); - if (!accessible) { + if (tunnelActive) { return; } var session = await getSession(); diff --git a/lib/tunnel-client-manager.js b/lib/tunnel-client-manager.js index 021e04f..5355295 100644 --- a/lib/tunnel-client-manager.js +++ b/lib/tunnel-client-manager.js @@ -1,109 +1,15 @@ 'use strict'; module.exports.create = function (deps, config) { - var PromiseA = require('bluebird'); - var fs = PromiseA.promisifyAll(require('fs')); var stunnel = require('stunnel'); var activeTunnels = {}; - var path = require('path'); - var tokensPath = path.join(__dirname, '..', 'var', 'tokens.json'); - var storage = { - _read: function () { - var tokens; - try { - tokens = require(tokensPath); - } catch (err) { - tokens = {}; - } - return tokens; - } - , _write: function (tokens) { - return fs.mkdirAsync(path.dirname(tokensPath)).catch(function (err) { - if (err.code !== 'EEXIST') { - console.error('failed to mkdir', path.dirname(tokensPath), err.toString()); - } - }).then(function () { - return fs.writeFileAsync(tokensPath, JSON.stringify(tokens), 'utf8'); - }); - } - , _makeKey: function (token) { - // We use a stripped down version of the token contents so that if the token is - // re-issued the nonce and the iat and any other less important things are different - // we don't save essentially duplicate tokens multiple times. - var parsed = JSON.parse((new Buffer(token.split('.')[1], 'base64')).toString()); - var stripped = {}; - ['aud', 'iss', 'domains'].forEach(function (key) { - if (parsed[key]) { - stripped[key] = parsed[key]; - } - }); - stripped.domains.sort(); - - var hash = require('crypto').createHash('sha256'); - return hash.update(JSON.stringify(stripped)).digest('hex'); - } - - , all: function () { - var tokens = storage._read(); - return PromiseA.resolve(Object.keys(tokens).map(function (key) { - return tokens[key]; - })); - } - , save: function (token) { - return PromiseA.resolve().then(function () { - var curTokens = storage._read(); - curTokens[storage._makeKey(token.jwt)] = token; - return storage._write(curTokens); - }); - } - , del: function (token) { - return PromiseA.resolve().then(function () { - var curTokens = storage._read(); - delete curTokens[storage._makeKey(token.jwt)]; - return storage._write(curTokens); - }); - } - }; - - function acquireToken(session) { - var OAUTH3 = deps.OAUTH3; - // session seems to be changed by the API call for some reason, so save the - // owner before that happens. - var owner = session.id; - - // The OAUTH3 library stores some things on the root session object that we usually - // just leave inside the token, but we need to pull those out before we use it here - session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss; - session.client_uri = session.client_uri || session.token.azp; - session.scope = session.scope || session.token.scp; - - console.log('asking for tunnel token from', session.token.aud); - return OAUTH3.discover(session.token.aud).then(function (directives) { - var opts = { - api: 'tunnel.token' - , session: session - , data: { - // filter to all domains that are on this device - //domains: Object.keys(domainsMap) - device: { - hostname: config.device.hostname - , id: config.device.uid || config.device.id - } - } - }; - - return OAUTH3.api(directives.api, opts).then(function (result) { - console.log('got a token from the tunnel server?'); - result.owner = owner; - return result; - }); - }); - } - function addToken(data) { + if (typeof data === 'string') { + data = { jwt: data }; + } if (!data.jwt) { - return PromiseA.reject(new Error("missing 'jwt' from tunnel data")); + return deps.PromiseA.reject(new Error("missing 'jwt' from tunnel data")); } if (!data.tunnelUrl) { var decoded; @@ -111,12 +17,12 @@ module.exports.create = function (deps, config) { decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii')); } catch (err) { console.warn('invalid web token given to tunnel manager', err); - return PromiseA.reject(err); + return deps.PromiseA.reject(err); } if (!decoded.aud) { console.warn('tunnel manager given token with no tunnelUrl or audience'); var err = new Error('missing tunnelUrl and audience'); - return PromiseA.reject(err); + return deps.PromiseA.reject(err); } data.tunnelUrl = 'wss://' + decoded.aud + '/'; } @@ -146,19 +52,49 @@ module.exports.create = function (deps, config) { return activeTunnels[data.tunnelUrl].append(data.jwt); } + function acquireToken(session, domains) { + var OAUTH3 = deps.OAUTH3; + + // The OAUTH3 library stores some things on the root session object that we usually + // just leave inside the token, but we need to pull those out before we use it here + session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss; + session.client_uri = session.client_uri || session.token.azp; + session.scope = session.scope || session.token.scp; + + console.log('asking for tunnel token from', session.token.aud); + return OAUTH3.discover(session.token.aud).then(function (directives) { + var opts = { + api: 'tunnel.token' + , session: session + , data: { + domains: domains + , device: { + hostname: config.device.hostname + , id: config.device.uid || config.device.id + } + } + }; + + return OAUTH3.api(directives.api, opts).then(addToken); + }); + } + function removeToken(data) { + if (typeof data === 'string') { + data = { jwt: data }; + } if (!data.tunnelUrl) { var decoded; try { decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii')); } catch (err) { console.warn('invalid web token given to tunnel manager', err); - return PromiseA.reject(err); + return deps.PromiseA.reject(err); } if (!decoded.aud) { console.warn('tunnel manager given token with no tunnelUrl or audience'); var err = new Error('missing tunnelUrl and audience'); - return PromiseA.reject(err); + return deps.PromiseA.reject(err); } data.tunnelUrl = 'wss://' + decoded.aud + '/'; } @@ -166,72 +102,23 @@ module.exports.create = function (deps, config) { // Not sure if we actually want to return an error that the token didn't even belong to a // server that existed, but since it never existed we can consider it as "removed". if (!activeTunnels[data.tunnelUrl]) { - return PromiseA.resolve(); + return deps.PromiseA.resolve(); } console.log('removing token from tunnel at', data.tunnelUrl); return activeTunnels[data.tunnelUrl].clear(data.jwt); } - if (config.tunnel) { - var confTokens = config.tunnel; - if (typeof confTokens === 'string') { - confTokens = confTokens.split(','); - } - confTokens.forEach(function (jwt) { - if (typeof jwt === 'object') { - jwt.owner = 'config'; - addToken(jwt); - } else { - addToken({ jwt: jwt, owner: 'config' }); - } + function disconnectAll() { + Object.keys(activeTunnels).forEach(function (key) { + activeTunnels[key].end(); }); } - - storage.all().then(function (stored) { - stored.forEach(function (result) { - addToken(result); - }); - }); - return { - start: function (session) { - return acquireToken(session).then(function (token) { - return addToken(token).then(function () { - return storage.save(token); - }); - }); - } - , add: function (data) { - return addToken(data).then(function () { - return storage.save(data); - }); - } - , remove: function (data) { - return storage.del(data.jwt).then(function () { - return removeToken(data); - }); - } - , get: function (owner) { - return storage.all().then(function (tokens) { - var result = {}; - tokens.forEach(function (data) { - if (!result[data.owner]) { - result[data.owner] = {}; - } - if (!result[data.owner][data.tunnelUrl]) { - result[data.owner][data.tunnelUrl] = []; - } - data.decoded = JSON.parse(new Buffer(data.jwt.split('.')[0], 'base64')); - result[data.owner][data.tunnelUrl].push(data); - }); - - if (owner) { - return result[owner] || {}; - } - return result; - }); - } + start: acquireToken + , startDirect: addToken + , remove: removeToken + , disconnect: disconnectAll }; }; diff --git a/packages/apis/com.daplie.goldilocks/index.js b/packages/apis/com.daplie.goldilocks/index.js index 8dfbd00..fb7f0fc 100644 --- a/packages/apis/com.daplie.goldilocks/index.js +++ b/packages/apis/com.daplie.goldilocks/index.js @@ -235,33 +235,6 @@ module.exports.create = function (deps, conf) { }); } - , tunnel: function (req, res) { - if (handleCors(req, res)) { - return; - } - isAuthorized(req, res, function () { - if ('POST' !== req.method) { - res.setHeader('Content-Type', 'application/json'); - return deps.tunnelClients.get(req.userId).then(function (result) { - res.end(JSON.stringify(result)); - }, function (err) { - res.statusCode = 500; - res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } })); - }); - } - - return deps.storage.owners.get(req.userId).then(function (session) { - return deps.tunnelClients.start(session).then(function () { - res.setHeader('Content-Type', 'application/json;'); - res.end(JSON.stringify({ success: true })); - }, function (err) { - res.setHeader('Content-Type', 'application/json;'); - res.statusCode = 500; - res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } })); - }); - }); - }); - } , config: function (req, res) { if (handleCors(req, res)) { return; From 9e9b5ca9ad055ba1c16aad0381991bdd7ae91c43 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Fri, 29 Sep 2017 15:29:47 -0600 Subject: [PATCH 05/41] update DDNS to also use the specified list of domains --- lib/ddns/dns-ctrl.js | 90 +++++++++++++++++++++++++++++++++++++------- lib/ddns/index.js | 2 +- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/lib/ddns/dns-ctrl.js b/lib/ddns/dns-ctrl.js index 500e590..ed290ac 100644 --- a/lib/ddns/dns-ctrl.js +++ b/lib/ddns/dns-ctrl.js @@ -10,7 +10,63 @@ module.exports.create = function (deps, conf) { } } - async function setDeviceAddress(session, addr) { + var tldCache = {}; + async function getTlds(provider) { + async function updateCache() { + var reqObj = { + url: deps.OAUTH3.url.normalize(provider)+'/api/com.daplie.domains/prices' + , method: 'GET' + , json: true + }; + + var resp = await deps.OAUTH3.request(reqObj); + var tldObj = {}; + resp.data.forEach(function (tldInfo) { + if (tldInfo.enabled) { + tldObj[tldInfo.com] = true; + } + }); + + tldCache[provider] = { + time: Date.now() + , tlds: tldObj + }; + return tldObj; + } + + // If we've never cached the results we need to return the promise that will fetch the recult, + // otherwise we can return the cached value. If the cached value has "expired", we can still + // return the cached value we just want to update the cache in parellel (making sure we only + // update once). + if (!tldCache[provider]) { + return updateCache(); + } + if (!tldCache[provider].updating && Date.now() - tldCache[provider].time > 24*60*60*1000) { + tldCache[provider].updating = true; + updateCache(); + } + + return tldCache[provider].tlds; + } + + async function splitDomains(provider, domains) { + var tlds = await getTlds(provider); + return domains.map(function (domain) { + var split = domain.split('.'); + var tldSegCnt = tlds[split.slice(-2).join('.')] ? 2 : 1; + + // Currently assuming that the sld can't contain dots, and that the tld can have at + // most one dot. Not 100% sure this is a valid assumption, but exceptions should be + // rare even if the assumption isn't valid. + return { + tld: split.slice(-tldSegCnt).join('.') + , sld: split.slice(-tldSegCnt-1, 1) + , sub: split.slice(0, -tldSegCnt-1) + }; + }); + } + + async function setDeviceAddress(session, addr, domains) { var directives = await deps.OAUTH3.discover(session.token.aud); // Set the address of the device to our public address. @@ -31,27 +87,33 @@ module.exports.create = function (deps, conf) { // Then update all of the records attached to our hostname, first removing the old records // to remove the reference to the old address, then creating new records for the same domains // using our new address. - var allDns = deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); - var ourDomains = allDns.filter(function (record) { - return record.device === conf.device.hostname; - }).map(function (record) { - var zoneSplit = record.zone.split('.'); - return { - tld: zoneSplit.slice(1).join('.') - , sld: zoneSplit[0] - , sub: record.host.slice(0, -(record.zone.length + 1)) - }; + var allDns = await deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'}); + var ourDns = allDns.filter(function (record) { + if (record.device !== conf.device.hostname) { + return false; + } + if ([ 'A', 'AAAA' ].indexOf(record.type) < 0) { + return false; + } + return domains.indexOf(record.host) !== -1; }); + var oldDomains = ourDns.filter(function (record) { + return record.value !== addr; + }).map(function (record) { + return record.host; + }); + var oldDns = await splitDomains(directives.api, oldDomains); var common = { api: 'devices.detach' , session: session , device: conf.device.hostname }; - await deps.PromiseA.all(ourDomains.map(function (record) { + await deps.PromiseA.all(oldDns.map(function (record) { return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); })); + var newDns = await splitDomains(directives.api, domains); common = { api: 'devices.attach' , session: session @@ -59,7 +121,7 @@ module.exports.create = function (deps, conf) { , ip: addr , ttl: 300 }; - await deps.PromiseA.all(ourDomains.map(function (record) { + await deps.PromiseA.all(newDns.map(function (record) { return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); })); } @@ -93,5 +155,5 @@ module.exports.create = function (deps, conf) { return { getDeviceAddresses: getDeviceAddresses , setDeviceAddress: setDeviceAddress -, }; + }; }; diff --git a/lib/ddns/index.js b/lib/ddns/index.js index e63f131..8221306 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -85,7 +85,7 @@ module.exports.create = function (deps, conf) { console.log('previous public address',publicAddress, 'does not match current public address', addr); } - await dnsCtrl.setDeviceAddress(session, addr); + await dnsCtrl.setDeviceAddress(session, addr, conf.ddns.domains); publicAddress = addr; } From b8f282db79ce711bb5363ef02dbd13bd1540fd78 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 2 Oct 2017 15:37:58 -0600 Subject: [PATCH 06/41] fixed bug in promisifying network package --- lib/ddns/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 8221306..27cbfe5 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -1,7 +1,7 @@ 'use strict'; module.exports.create = function (deps, conf) { - var network = deps.PromiseA.promisify(deps.recase.camelCopy(require('network'))); + var network = deps.PromiseA.promisifyAll(deps.recase.camelCopy(require('network'))); var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); From 3d3fac508703ad424ce96cd27765704cc4333f02 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 3 Oct 2017 17:26:44 -0600 Subject: [PATCH 07/41] simplified how the admin routes are handled --- .../index.js => lib/admin/apis.js | 0 lib/admin/index.js | 38 ++ lib/app.js | 325 ------------------ lib/modules/admin.js | 67 ---- lib/modules/http.js | 4 +- lib/worker.js | 12 +- packages/apis/com.daplie.goldilocks/test.js | 23 -- 7 files changed, 51 insertions(+), 418 deletions(-) rename packages/apis/com.daplie.goldilocks/index.js => lib/admin/apis.js (100%) create mode 100644 lib/admin/index.js delete mode 100644 lib/app.js delete mode 100644 lib/modules/admin.js delete mode 100644 packages/apis/com.daplie.goldilocks/test.js diff --git a/packages/apis/com.daplie.goldilocks/index.js b/lib/admin/apis.js similarity index 100% rename from packages/apis/com.daplie.goldilocks/index.js rename to lib/admin/apis.js diff --git a/lib/admin/index.js b/lib/admin/index.js new file mode 100644 index 0000000..e8c146b --- /dev/null +++ b/lib/admin/index.js @@ -0,0 +1,38 @@ +var adminDomains = [ + 'localhost.alpha.daplie.me' +, 'localhost.admin.daplie.me' +, 'alpha.localhost.daplie.me' +, 'admin.localhost.daplie.me' +, 'localhost.daplie.invalid' +]; +module.exports.adminDomains = adminDomains; + +module.exports.create = function (deps, conf) { + 'use strict'; + + var path = require('path'); + var express = require('express'); + var app = express(); + + var apis = require('./apis').create(deps, conf); + function handleApis(req, res, next) { + if (typeof apis[req.params.name] === 'function') { + apis[req.params.name](req, res); + } else { + next(); + } + } + app.use('/api/goldilocks@daplie.com/:name', handleApis); + app.use('/api/com.daplie.goldilocks/:name', handleApis); + + // Serve the static assets for the UI (even though it probably won't be used very + // often since it only works on localhost domains). Note that we are using the default + // .well-known directory from the oauth3 library even though it indicates we have + // capabilities we don't support because it's simpler and it's unlikely anything will + // actually use it to determine our API (it is needed to log into the web page). + app.use('/.well-known', express.static(path.join(__dirname, '../../packages/assets/well-known'))); + app.use('/assets', express.static(path.join(__dirname, '../../packages/assets'))); + app.use('/', express.static(path.join(__dirname, '../../admin/public'))); + + return require('http').createServer(app); +}; diff --git a/lib/app.js b/lib/app.js deleted file mode 100644 index f633712..0000000 --- a/lib/app.js +++ /dev/null @@ -1,325 +0,0 @@ -'use strict'; - -module.exports = function (myDeps, conf, overrideHttp) { - var express = require('express'); - //var finalhandler = require('finalhandler'); - var serveStatic = require('serve-static'); - var serveIndex = require('serve-index'); - //var assetServer = serveStatic(opts.assetsPath); - var path = require('path'); - //var wellKnownServer = serveStatic(path.join(opts.assetsPath, 'well-known')); - - var serveStaticMap = {}; - var serveIndexMap = {}; - var content = conf.content; - //var server; - var goldilocksApis; - var app; - var request; - - function createGoldilocksApis() { - var PromiseA = require('bluebird'); - var OAUTH3 = require('../packages/assets/org.oauth3'); - require('../packages/assets/org.oauth3/oauth3.domains.js'); - require('../packages/assets/org.oauth3/oauth3.dns.js'); - require('../packages/assets/org.oauth3/oauth3.tunnel.js'); - OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); - - request = request || PromiseA.promisify(require('request')); - - myDeps.PromiseA = PromiseA; - myDeps.OAUTH3 = OAUTH3; - myDeps.recase = require('recase').create({}); - myDeps.request = request; - - return require('../packages/apis/com.daplie.goldilocks').create(myDeps, conf); - } - - app = express(); - - var Sites = { - add: function (sitesMap, site) { - if (!sitesMap[site.$id]) { - sitesMap[site.$id] = site; - } - - if (!site.paths) { - site.paths = []; - } - if (!site.paths._map) { - site.paths._map = {}; - } - site.paths.forEach(function (path) { - - site.paths._map[path.$id] = path; - - if (!path.modules) { - path.modules = []; - } - if (!path.modules._map) { - path.modules._map = {}; - } - path.modules.forEach(function (module) { - - path.modules._map[module.$id] = module; - }); - }); - } - }; - - var opts = overrideHttp || conf.http; - if (!opts.defaults) { - opts.defaults = {}; - } - if (!opts.global) { - opts.global = {}; - } - if (!opts.sites) { - opts.sites = []; - } - opts.sites._map = {}; - opts.sites.forEach(function (site) { - - Sites.add(opts.sites._map, site); - }); - - function mapMap(el, i, arr) { - arr._map[el.$id] = el; - } - opts.global.modules._map = {}; - opts.global.modules.forEach(mapMap); - opts.global.paths._map = {}; - opts.global.paths.forEach(function (path, i, arr) { - mapMap(path, i, arr); - //opts.global.paths._map[path.$id] = path; - path.modules._map = {}; - path.modules.forEach(mapMap); - }); - opts.sites.forEach(function (site) { - site.paths._map = {}; - site.paths.forEach(function (path, i, arr) { - mapMap(path, i, arr); - //site.paths._map[path.$id] = path; - path.modules._map = {}; - path.modules.forEach(mapMap); - }); - }); - opts.defaults.modules._map = {}; - opts.defaults.modules.forEach(mapMap); - opts.defaults.paths._map = {}; - opts.defaults.paths.forEach(function (path, i, arr) { - mapMap(path, i, arr); - //opts.global.paths._map[path.$id] = path; - path.modules._map = {}; - path.modules.forEach(mapMap); - }); - - function _goldApis(req, res, next) { - if (!goldilocksApis) { - goldilocksApis = createGoldilocksApis(); - } - - if (typeof goldilocksApis[req.params.name] === 'function') { - goldilocksApis[req.params.name](req, res); - } else { - next(); - } - } - return app - .use('/api/com.daplie.goldilocks/:name', _goldApis) - .use('/api/goldilocks@daplie.com/:name', _goldApis) - .use('/', function (req, res, next) { - if (!req.headers.host) { - next(new Error('missing HTTP Host header')); - return; - } - - if (content && '/' === req.url) { - // res.setHeader('Content-Type', 'application/octet-stream'); - res.end(content); - return; - } - - //var done = finalhandler(req, res); - var host = req.headers.host; - var hostname = (host||'').split(':')[0].toLowerCase(); - - console.log('opts.global', opts.global); - var sites = [ opts.global || null, opts.sites._map[hostname] || null, opts.defaults || null ]; - var loadables = { - serve: function (config, hostname, pathname, req, res, next) { - var originalUrl = req.url; - var dirpaths = config.paths.slice(0); - - function nextServe() { - var dirname = dirpaths.pop(); - if (!dirname) { - req.url = originalUrl; - next(); - return; - } - - console.log('[serve]', req.url, hostname, pathname, dirname); - dirname = path.resolve(conf.cwd, dirname.replace(/:hostname/, hostname)); - if (!serveStaticMap[dirname]) { - serveStaticMap[dirname] = serveStatic(dirname); - } - - serveStaticMap[dirname](req, res, nextServe); - } - - req.url = req.url.substr(pathname.length - 1); - nextServe(); - } - , indexes: function (config, hostname, pathname, req, res, next) { - var originalUrl = req.url; - var dirpaths = config.paths.slice(0); - - function nextIndex() { - var dirname = dirpaths.pop(); - if (!dirname) { - req.url = originalUrl; - next(); - return; - } - - console.log('[indexes]', req.url, hostname, pathname, dirname); - dirname = path.resolve(conf.cwd, dirname.replace(/:hostname/, hostname)); - if (!serveStaticMap[dirname]) { - serveIndexMap[dirname] = serveIndex(dirname); - } - serveIndexMap[dirname](req, res, nextIndex); - } - - req.url = req.url.substr(pathname.length - 1); - nextIndex(); - } - , app: function (config, hostname, pathname, req, res, next) { - //var appfile = path.resolve(/*process.cwd(), */config.path.replace(/:hostname/, hostname)); - var appfile = config.path.replace(/:hostname/, hostname); - try { - var app = require(appfile); - app(req, res, next); - } catch (err) { - next(); - } - } - }; - - function runModule(module, hostname, pathname, modulename, req, res, next) { - if (!loadables[modulename]) { - next(new Error("no module '" + modulename + "' found")); - return; - } - loadables[modulename](module, hostname, pathname, req, res, next); - } - - function iterModules(modules, hostname, pathname, req, res, next) { - console.log('modules'); - console.log(modules); - var modulenames = Object.keys(modules._map); - - function nextModule() { - var modulename = modulenames.pop(); - if (!modulename) { - next(); - return; - } - - console.log('modules', modules); - runModule(modules._map[modulename], hostname, pathname, modulename, req, res, nextModule); - } - - nextModule(); - } - - function iterPaths(site, hostname, req, res, next) { - console.log('site', hostname); - console.log(site); - var pathnames = Object.keys(site.paths._map); - console.log('pathnames', pathnames); - pathnames = pathnames.filter(function (pathname) { - // TODO ensure that pathname has trailing / - return (0 === req.url.indexOf(pathname)); - //return req.url.match(pathname); - }); - pathnames.sort(function (a, b) { - return b.length - a.length; - }); - console.log('pathnames', pathnames); - - function nextPath() { - var pathname = pathnames.shift(); - if (!pathname) { - next(); - return; - } - - console.log('iterPaths', hostname, pathname, req.url); - iterModules(site.paths._map[pathname].modules, hostname, pathname, req, res, nextPath); - } - - nextPath(); - } - - function nextSite() { - console.log('hostname', hostname, sites); - var site; - if (!sites.length) { - next(); // 404 - return; - } - site = sites.shift(); - if (!site) { - nextSite(); - return; - } - iterPaths(site, hostname, req, res, nextSite); - } - - nextSite(); - - /* - function serveStaticly(server) { - function serveTheStatic() { - server.serve(req, res, function (err) { - if (err) { return done(err); } - server.index(req, res, function (err) { - if (err) { return done(err); } - req.url = req.url.replace(/\/assets/, ''); - assetServer(req, res, function () { - if (err) { return done(err); } - req.url = req.url.replace(/\/\.well-known/, ''); - wellKnownServer(req, res, done); - }); - }); - }); - } - - if (server.expressApp) { - server.expressApp(req, res, serveTheStatic); - return; - } - - serveTheStatic(); - } - - if (opts.livereload) { - res.__my_livereload = ''; - res.__my_addLen = res.__my_livereload.length; - - // TODO modify prototype instead of each instance? - res.__write = res.write; - res.write = _reloadWrite; - } - - console.log('hostname:', hostname, opts.sites[0].paths); - - addServer(hostname); - server = hostsMap[hostname] || hostsMap[opts.sites[0].name]; - serveStaticly(server); - */ - }); -}; diff --git a/lib/modules/admin.js b/lib/modules/admin.js deleted file mode 100644 index d22b168..0000000 --- a/lib/modules/admin.js +++ /dev/null @@ -1,67 +0,0 @@ -var adminDomains = [ - 'localhost.alpha.daplie.me' -, 'localhost.admin.daplie.me' -, 'alpha.localhost.daplie.me' -, 'admin.localhost.daplie.me' -, 'localhost.daplie.invalid' -]; -module.exports.adminDomains = adminDomains; - -module.exports.create = function (deps, conf) { - 'use strict'; - - var path = require('path'); - //var defaultServername = 'localhost.daplie.me'; - //var defaultWebRoot = '.'; - var assetsPath = path.join(__dirname, '..', '..', 'packages', 'assets'); - var opts = {}; - - opts.global = opts.global || {}; - opts.sites = opts.sites || []; - opts.sites._map = {}; - - // argv.sites - - opts.groups = []; - - // 'packages', 'assets', 'com.daplie.goldilocks' - opts.global = { - modules: [ // TODO uh-oh we've got a mixed bag of modules (various types), a true map - { $id: 'greenlock', email: opts.email, tos: opts.tos } - , { $id: 'rvpn', email: opts.email, tos: opts.tos } - //, { $id: 'content', content: content } - , { $id: 'livereload', on: opts.livereload } - , { $id: 'app', path: opts.expressApp } - ] - , paths: [ - { $id: '/assets/', modules: [ { $id: 'serve', paths: [ assetsPath ] } ] } - // TODO figure this b out - , { $id: '/.well-known/', modules: [ - { $id: 'serve', paths: [ path.join(assetsPath, 'well-known') ] } - ] } - ] - }; - opts.defaults = { - modules: [] - , paths: [ - /* - { $id: '/', modules: [ - { $id: 'serve', paths: [ defaultWebRoot ] } - , { $id: 'indexes', paths: [ defaultWebRoot ] } - ] } - */ - ] - }; - adminDomains.forEach(function (id) { - opts.sites.push({ - $id: id - , paths: [ - { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] } - , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] } - ] - }); - }); - - var app = require('../app.js')(deps, conf, opts); - return require('http').createServer(app); -}; diff --git a/lib/modules/http.js b/lib/modules/http.js index 636b6bc..365b09a 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -202,11 +202,11 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { var host = separatePort(headers.host).host; if (!adminDomains) { - adminDomains = require('./admin').adminDomains; + adminDomains = require('../admin').adminDomains; } if (adminDomains.indexOf(host) !== -1) { if (!adminServer) { - adminServer = require('./admin').create(deps, conf); + adminServer = require('../admin').create(deps, conf); } return emitConnection(adminServer, conn, opts); } diff --git a/lib/worker.js b/lib/worker.js index 56991bc..d7ad1c1 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -19,10 +19,20 @@ function update(conf) { } function create(conf) { + var PromiseA = require('bluebird'); + var OAUTH3 = require('../packages/assets/org.oauth3'); + require('../packages/assets/org.oauth3/oauth3.domains.js'); + require('../packages/assets/org.oauth3/oauth3.dns.js'); + require('../packages/assets/org.oauth3/oauth3.tunnel.js'); + OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js'); + config = conf; var deps = { messenger: process - , PromiseA: require('bluebird') + , PromiseA: PromiseA + , OAUTH3: OAUTH3 + , request: PromiseA.promisify(require('request')) + , recase: require('recase').create({}) // Note that if a custom createConnections is used it will be called with different // sets of custom options based on what is actually being proxied. Most notably the // HTTP proxying connection creation is not something we currently control. diff --git a/packages/apis/com.daplie.goldilocks/test.js b/packages/apis/com.daplie.goldilocks/test.js deleted file mode 100644 index 77b55de..0000000 --- a/packages/apis/com.daplie.goldilocks/test.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict'; - -var api = require('./index.js').api; -var OAUTH3 = require('../../assets/org.oauth3/'); -// these all auto-register -require('../../assets/org.oauth3/oauth3.domains.js'); -require('../../assets/org.oauth3/oauth3.dns.js'); -require('../../assets/org.oauth3/oauth3.tunnel.js'); -OAUTH3._hooks = require('../../assets/org.oauth3/oauth3.node.storage.js'); - -api.tunnel( - { - OAUTH3: OAUTH3 - , options: { - device: { - hostname: 'test.local' - , id: '' - } - } - } - // OAUTH3.hooks.session.get('oauth3.org').then(function (result) { console.log(result) }); -, require('./test.session.json') -); From f25a0191bd701eabb316c7059bd753735b8048b7 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 3 Oct 2017 19:11:49 -0600 Subject: [PATCH 08/41] changed config API to use an express router --- lib/admin/apis.js | 68 ++++++++++++++++++++++++++++++---------------- lib/admin/index.js | 11 ++------ 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 3c3e54c..6c22ca0 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -38,6 +38,13 @@ module.exports.create = function (deps, conf) { return true; } } + function makeCorsHandler(methods) { + return function corsHandler(req, res, next) { + if (!handleCors(req, res, methods)) { + next(); + } + }; + } function isAuthorized(req, res, fn) { var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, '')); @@ -139,7 +146,10 @@ module.exports.create = function (deps, conf) { ; } - return { + // This object contains all of the API endpoints written before we changed how + // the API routing is handled. Eventually it will hopefully disappear, but for + // now we're focusing on the things that need changing more. + var oldEndPoints = { init: function (req, res) { if (handleCors(req, res, ['GET', 'POST'])) { return; @@ -262,28 +272,6 @@ module.exports.create = function (deps, conf) { }); }); } - , config: function (req, res) { - if (handleCors(req, res)) { - return; - } - isAuthorized(req, res, function () { - if ('POST' !== req.method) { - res.setHeader('Content-Type', 'application/json;'); - res.end(JSON.stringify(deps.recase.snakeCopy(conf))); - return; - } - - jsonParser(req, res, function () { - console.log('config POST body', req.body); - - // Since we are sending the changes to another process we don't really - // have a good way of seeing if it worked, so always report success - deps.storage.config.save(req.body); - res.setHeader('Content-Type', 'application/json;'); - res.end('{"success":true}'); - }); - }); - } , request: function (req, res) { if (handleCors(req, res, '*')) { return; @@ -381,4 +369,38 @@ module.exports.create = function (deps, conf) { }); } }; + + function handleOldApis(req, res, next) { + if (typeof oldEndPoints[req.params.name] === 'function') { + oldEndPoints[req.params.name](req, res); + } else { + next(); + } + } + + var config = { restful: {} }; + config.restful.readConfig = function (req, res) { + res.send(deps.recase.snakeCopy(conf)); + }; + config.restful.saveConfig = function (req, res) { + console.log('config POST body', req.body); + + // Since we are sending the changes to another process we don't really + // have a good way of seeing if it worked, so always report success + deps.storage.config.save(req.body); + res.send({ success: true }); + }; + + var app = require('express')(); + + // Handle all of the API endpoints using the old definition style, and then we can + // add middleware without worrying too much about the consequences to older code. + app.use('/:name', handleOldApis); + + app.use('/', isAuthorized, jsonParser); + + app.use( '/config', makeCorsHandler()); + app.get( '/config', config.restful.readConfig); + app.post( '/config', config.restful.saveConfig); + return app; }; diff --git a/lib/admin/index.js b/lib/admin/index.js index e8c146b..e317d39 100644 --- a/lib/admin/index.js +++ b/lib/admin/index.js @@ -15,15 +15,8 @@ module.exports.create = function (deps, conf) { var app = express(); var apis = require('./apis').create(deps, conf); - function handleApis(req, res, next) { - if (typeof apis[req.params.name] === 'function') { - apis[req.params.name](req, res); - } else { - next(); - } - } - app.use('/api/goldilocks@daplie.com/:name', handleApis); - app.use('/api/com.daplie.goldilocks/:name', handleApis); + app.use('/api/goldilocks@daplie.com', apis); + app.use('/api/com.daplie.goldilocks', apis); // Serve the static assets for the UI (even though it probably won't be used very // often since it only works on localhost domains). Note that we are using the default From 12e4a47855dc8a0eb0b5eb05022385029cde75de Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 4 Oct 2017 11:49:05 -0600 Subject: [PATCH 09/41] removed addresses and cwd from the config --- bin/goldilocks.js | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 51d77d2..908da13 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -178,38 +178,6 @@ function fillConfig(config, args) { config.email = args.email; config.tls.acme.email = args.email; } - - // maybe this should not go in config... but be ephemeral in some way? - config.cwd = args.cwd || config.cwd || process.cwd(); - - var ipaddr = require('ipaddr.js'); - var addresses = []; - var ifaces = require('../lib/local-ip.js').find(); - - Object.keys(ifaces).forEach(function (ifacename) { - var iface = ifaces[ifacename]; - iface.ipv4.forEach(function (ip) { - addresses.push(ip); - }); - iface.ipv6.forEach(function (ip) { - addresses.push(ip); - }); - }); - - addresses.sort(function (a, b) { - if (a.family !== b.family) { - return 'IPv4' === a.family ? 1 : -1; - } - - return a.address > b.address ? 1 : -1; - }); - - addresses.forEach(function (addr) { - addr.range = ipaddr.parse(addr.address).range(); - }); - - // TODO maybe move to config.state.addresses (?) - config.addresses = addresses; config.device = { hostname: require('os').hostname() }; config.tunnel = args.tunnel || config.tunnel; From cc6b34dd46b3db890002824caea8eb611a0a3c1c Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 4 Oct 2017 14:42:19 -0600 Subject: [PATCH 10/41] made it possible to GET specific parts of the config --- lib/admin/apis.js | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 6c22ca0..0a3a7c9 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -379,8 +379,29 @@ module.exports.create = function (deps, conf) { } var config = { restful: {} }; - config.restful.readConfig = function (req, res) { - res.send(deps.recase.snakeCopy(conf)); + config.restful.readConfig = function (req, res, next) { + var part = conf; + if (req.params.group) { + part = part[req.params.group]; + } + if (part && req.params.name) { + part = part[req.params.name]; + } + if (part && req.params.id) { + part = part.find(function (mod) { return mod.id === req.params.id; }); + } + if (part && req.params.name2) { + part = part[req.params.name2]; + } + if (part && req.params.id2) { + part = part.find(function (mod) { return mod.id === req.params.id2; }); + } + + if (part) { + res.send(deps.recase.snakeCopy(part)); + } else { + next(); + } }; config.restful.saveConfig = function (req, res) { console.log('config POST body', req.body); @@ -400,7 +421,12 @@ module.exports.create = function (deps, conf) { app.use('/', isAuthorized, jsonParser); app.use( '/config', makeCorsHandler()); - app.get( '/config', config.restful.readConfig); + app.get( '/config', config.restful.readConfig); + app.get( '/config/:group', config.restful.readConfig); + app.get( '/config/:group/:name(modules|domains)', config.restful.readConfig); + app.get( '/config/:group/:name(modules|domains)/:id', config.restful.readConfig); + app.get( '/config/:group/:name(domains)/:id/:name2(modules)', config.restful.readConfig); + app.get( '/config/:group/:name(domains)/:id/:name2(modules)/:id2', config.restful.readConfig); app.post( '/config', config.restful.saveConfig); return app; }; From d04b750f87d8c5cf894e57564e3250753602a504 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 4 Oct 2017 18:26:27 -0600 Subject: [PATCH 11/41] changed the default config --- bin/goldilocks.js | 49 +++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 908da13..6b71c75 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -148,26 +148,42 @@ var tcpProm; function fillConfig(config, args) { config.debug = config.debug || args.debug; - if (!config.dns) { - config.dns = { bind: [ 53 ], modules: [{ name: 'proxy', port: 3053 }] }; - } if (!config.ddns) { config.ddns = { enabled: false }; } - // Use Object.assign to add any properties needed but not defined in the mdns config. - // It will first copy the defaults into an empty object, then copy any real config over that. - var mdnsDefaults = { port: 5353, broadcast: '224.0.0.251', ttl: 300 }; - config.mdns = Object.assign({}, mdnsDefaults, config.mdns); + // Use Object.assign to copy any real config values over the default values so we can + // easily make sure all the fields we need exist . + var mdnsDefaults = { disabled: false, port: 5353, broadcast: '224.0.0.251', ttl: 300 }; + config.mdns = Object.assign(mdnsDefaults, config.mdns); - if (!config.tcp) { - config.tcp = {}; - } - if (!config.http) { - config.http = { modules: [{ name: 'proxy', domains: ['*'], port: 3000 }] }; - } - if (!config.tls) { - config.tls = {}; + function fillComponent(name, fillBind, fillDomains) { + if (!config[name]) { + config[name] = {}; + } + if (!Array.isArray(config[name].modules)) { + config[name].modules = []; + } + + if (fillBind && !Array.isArray(config[name].bind)) { + config[name].bind = []; + } + + if (fillDomains) { + if (!Array.isArray(config[name].domains)) { + config[name].domains = []; + } + config[name].domains.forEach(function (domain) { + if (!Array.isArray(domain.modules)) { + domain.modules = []; + } + }); + } } + fillComponent('dns', true, false); + fillComponent('tcp', true, false); + fillComponent('http', false, true); + fillComponent('tls', false, true); + if (!config.tls.acme && (args.email || args.agreeTos)) { config.tls.acme = {}; } @@ -175,14 +191,13 @@ function fillConfig(config, args) { config.tls.acme.approvedDomains = args.agreeTos.split(','); } if (args.email) { - config.email = args.email; config.tls.acme.email = args.email; } config.device = { hostname: require('os').hostname() }; config.tunnel = args.tunnel || config.tunnel; - if (Array.isArray(config.tcp.bind)) { + if (Array.isArray(config.tcp.bind) && config.tcp.bind.length) { return PromiseA.resolve(config); } From 0380a8087feb932fcc2480463ddd35205038b294 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 4 Oct 2017 18:27:29 -0600 Subject: [PATCH 12/41] automatically add `id` to modules and domains --- bin/goldilocks.js | 172 ++++++++++++++++++++++++++++++---------------- 1 file changed, 111 insertions(+), 61 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 6b71c75..456e9a9 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -8,6 +8,7 @@ if (!cluster.isMaster) { return; } +var crypto = require('crypto'); var PromiseA = require('bluebird'); var fs = PromiseA.promisifyAll(require('fs')); var configStorage; @@ -25,7 +26,45 @@ function mergeSettings(orig, changes) { } }); } -function createStorage(filename, filetype) { + +function fixRawConfig(config) { + var updated = false; + + function updateModules(list) { + if (!Array.isArray(list)) { + return; + } + list.forEach(function (mod) { + if (!mod.id) { + mod.id = crypto.randomBytes(8).toString('hex'); + updated = true; + } + }); + } + function updateDomains(list) { + if (!Array.isArray(list)) { + return; + } + list.forEach(function (mod) { + if (!mod.id) { + mod.id = crypto.randomBytes(8).toString('hex'); + updated = true; + } + updateModules(mod.modules); + }); + } + + [ 'dns', 'tcp', 'http', 'tls' ].forEach(function (key) { + if (!config[key]) { + return; + } + updateModules(config[key].modules); + updateDomains(config[key].domains); + }); + + return updated; +} +async function createStorage(filename, filetype) { var recase = require('recase').create({}); var snakeCopy = recase.snakeCopy.bind(recase); var camelCopy = recase.camelCopy.bind(recase); @@ -40,13 +79,25 @@ function createStorage(filename, filetype) { dump = yaml.safeDump; } - function read() { - return fs.readFileAsync(filename).then(parse).catch(function (err) { + async function read() { + var text; + try { + text = await fs.readFileAsync(filename); + } catch (err) { if (err.code === 'ENOENT') { - return ''; + return {}; } - return PromiseA.reject(err); - }); + throw err; + } + + var rawConfig = parse(text); + if (fixRawConfig(rawConfig)) { + await fs.writeFileAsync(filename, dump(rawConfig)); + text = await fs.readFileAsync(filename); + rawConfig = parse(text); + } + + return rawConfig; } var result = { @@ -76,72 +127,71 @@ function createStorage(filename, filetype) { }; return result; } -function checkConfigLocation(cwd, configFile) { +async function checkConfigLocation(cwd, configFile) { cwd = cwd || process.cwd(); var path = require('path'); - var filename; + var filename, text; - var prom; if (configFile) { filename = path.resolve(cwd, configFile); - prom = fs.readFileAsync(filename) - .catch(function (err) { - if (err.code !== 'ENOENT') { - return PromiseA.reject(err); - } - if (path.extname(filename) === '.json') { - return '{}'; - } - return ''; - }) - ; + try { + text = await fs.readFileAsync(filename); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; + } + if (path.extname(filename) === '.json') { + return { name: filename, type: 'json' }; + } else { + return { name: filename, type: 'yaml' }; + } + } } else { - prom = PromiseA.reject('blah') - .catch(function () { - filename = path.resolve(cwd, 'goldilocks.yml'); - return fs.readFileAsync(filename); - }) - .catch(function () { - filename = path.resolve(cwd, 'goldilocks.json'); - return fs.readFileAsync(filename); - }) - .catch(function () { - filename = path.resolve(cwd, 'etc/goldilocks/goldilocks.yml'); - return fs.readFileAsync(filename); - }) - .catch(function () { - filename = '/etc/goldilocks/goldilocks.yml'; - return fs.readFileAsync(filename); - }) - .catch(function () { - filename = path.resolve(cwd, 'goldilocks.yml'); - return ''; - }) - ; + // Note that `path.resolve` can handle both relative and absolute paths. + var defLocations = [ + path.resolve(cwd, 'goldilocks.yml') + , path.resolve(cwd, 'goldilocks.json') + , path.resolve(cwd, 'etc/goldilocks/goldilocks.yml') + , '/etc/goldilocks/goldilocks.yml' + , path.resolve(cwd, 'goldilocks.yml') + ]; + + var ind; + for (ind = 0; ind < defLocations.length; ind += 1) { + try { + text = await fs.readFileAsync(defLocations[ind]); + filename = defLocations[ind]; + break; + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; + } + } + } + + if (!filename) { + filename = defLocations[0]; + text = ''; + } } - return prom.then(function (text) { - try { - JSON.parse(text); - return { name: filename, type: 'json' }; - } catch (err) {} + try { + JSON.parse(text); + return { name: filename, type: 'json' }; + } catch (err) {} - try { - require('js-yaml').safeLoad(text); - return { name: filename, type: 'yaml' }; - } catch (err) {} + try { + require('js-yaml').safeLoad(text); + return { name: filename, type: 'yaml' }; + } catch (err) {} - throw new Error('Could not load "' + filename + '" as JSON nor YAML'); - }); + throw new Error('Could not load "' + filename + '" as JSON nor YAML'); } -function createConfigStorage(args) { - return checkConfigLocation(args.cwd, args.config) - .then(function (result) { - console.log('config file', result.name, 'is of type', result.type); - configStorage = createStorage(result.name, result.type); - return configStorage.read(); - }) - ; +async function createConfigStorage(args) { + var result = await checkConfigLocation(args.cwd, args.config); + console.log('config file', result.name, 'is of type', result.type); + configStorage = await createStorage(result.name, result.type); + return configStorage.read(); } var tcpProm; From ded53cf45cc8ead718f7dc1917cacdd9df6a86cc Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 5 Oct 2017 18:10:59 -0600 Subject: [PATCH 13/41] reduced a few lines of code --- bin/goldilocks.js | 3 +-- lib/admin/apis.js | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 456e9a9..ea936b6 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -36,7 +36,7 @@ function fixRawConfig(config) { } list.forEach(function (mod) { if (!mod.id) { - mod.id = crypto.randomBytes(8).toString('hex'); + mod.id = crypto.randomBytes(4).toString('hex'); updated = true; } }); @@ -153,7 +153,6 @@ async function checkConfigLocation(cwd, configFile) { , path.resolve(cwd, 'goldilocks.json') , path.resolve(cwd, 'etc/goldilocks/goldilocks.yml') , '/etc/goldilocks/goldilocks.yml' - , path.resolve(cwd, 'goldilocks.yml') ]; var ind; diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 0a3a7c9..902ec15 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -421,12 +421,10 @@ module.exports.create = function (deps, conf) { app.use('/', isAuthorized, jsonParser); app.use( '/config', makeCorsHandler()); - app.get( '/config', config.restful.readConfig); - app.get( '/config/:group', config.restful.readConfig); - app.get( '/config/:group/:name(modules|domains)', config.restful.readConfig); - app.get( '/config/:group/:name(modules|domains)/:id', config.restful.readConfig); - app.get( '/config/:group/:name(domains)/:id/:name2(modules)', config.restful.readConfig); - app.get( '/config/:group/:name(domains)/:id/:name2(modules)/:id2', config.restful.readConfig); + app.get( '/config', config.restful.readConfig); + app.get( '/config/:group', config.restful.readConfig); + app.get( '/config/:group/:name(modules|domains)/:id?', config.restful.readConfig); + app.get( '/config/:group/:name(domains)/:id/:name2(modules)/:id2?', config.restful.readConfig); app.post( '/config', config.restful.saveConfig); return app; }; From 8f4a733391e5323dc6b718fb047b75f17d6dad9e Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 5 Oct 2017 18:11:58 -0600 Subject: [PATCH 14/41] changed module config property name --- bin/goldilocks.js | 5 +++++ lib/goldilocks.js | 4 ++-- lib/modules/http.js | 8 ++++---- lib/modules/tls.js | 8 ++++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index ea936b6..f7b59c3 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -39,6 +39,11 @@ function fixRawConfig(config) { mod.id = crypto.randomBytes(4).toString('hex'); updated = true; } + if (mod.name) { + mod.type = mod.type || mod.name; + delete mod.name; + updated = true; + } }); } function updateDomains(list) { diff --git a/lib/goldilocks.js b/lib/goldilocks.js index a0b3fcc..3e3ee28 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -101,7 +101,7 @@ module.exports.create = function (deps, config) { } var socket = require('dgram').createSocket('udp4'); config.dns.modules.forEach(function (mod) { - if (mod.name !== 'proxy') { + if (mod.type !== 'proxy') { console.warn('found bad DNS module', mod); return; } @@ -213,7 +213,7 @@ module.exports.create = function (deps, config) { addPorts(config.tcp.bind); (config.tcp.modules || []).forEach(function (mod) { - if (mod.name === 'forward') { + if (mod.type === 'forward') { var forwarder = createTcpForwarder(mod); mod.ports.forEach(function (port) { if (!tcpPortMap[port]) { diff --git a/lib/modules/http.js b/lib/modules/http.js index 365b09a..ed3c571 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -426,10 +426,10 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { var subProm = PromiseA.resolve(false); dom.modules.forEach(function (mod) { - if (moduleChecks[mod.name]) { + if (moduleChecks[mod.type]) { subProm = subProm.then(function (handled) { if (handled) { return handled; } - return moduleChecks[mod.name](mod, conn, opts, headers); + return moduleChecks[mod.type](mod, conn, opts, headers); }); } else { console.warn('unknown HTTP module under domains', dom.names.join(','), mod); @@ -447,8 +447,8 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { return false; } - if (moduleChecks[mod.name]) { - return moduleChecks[mod.name](mod, conn, opts, headers); + if (moduleChecks[mod.type]) { + return moduleChecks[mod.type](mod, conn, opts, headers); } console.warn('unknown HTTP module found', mod); }); diff --git a/lib/modules/tls.js b/lib/modules/tls.js index 19bb2f1..c8191d2 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -142,7 +142,7 @@ module.exports.create = function (deps, config, netHandler) { } return dom.modules.some(function (mod) { - if (mod.name !== 'acme') { + if (mod.type !== 'acme') { return false; } complete(mod, dom.names); @@ -156,7 +156,7 @@ module.exports.create = function (deps, config, netHandler) { if (Array.isArray(config.tls.modules)) { handled = config.tls.modules.some(function (mod) { - if (mod.name !== 'acme') { + if (mod.type !== 'acme') { return false; } if (!nameMatchesDomains(opts.domain, mod.domains)) { @@ -322,10 +322,10 @@ module.exports.create = function (deps, config, netHandler) { } function checkModule(mod) { - if (mod.name === 'proxy') { + if (mod.type === 'proxy') { return proxy(socket, opts, mod); } - if (mod.name !== 'acme') { + if (mod.type !== 'acme') { console.error('saw unknown TLS module', mod); } } From 5761ab9d620630e2da4df75bbfb943ca7cd19340 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Fri, 6 Oct 2017 17:50:16 -0600 Subject: [PATCH 15/41] added JSON Schema to validate the config --- bin/goldilocks.js | 6 +- lib/admin/config.js | 201 ++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 13 ++- package.json | 1 + 4 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 lib/admin/config.js diff --git a/bin/goldilocks.js b/bin/goldilocks.js index f7b59c3..abca131 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -202,9 +202,9 @@ var tcpProm; function fillConfig(config, args) { config.debug = config.debug || args.debug; - if (!config.ddns) { - config.ddns = { enabled: false }; - } + config.socks5 = config.socks5 || { enabled: false }; + config.ddns = config.ddns || { enabled: false }; + // Use Object.assign to copy any real config values over the default values so we can // easily make sure all the fields we need exist . var mdnsDefaults = { disabled: false, port: 5353, broadcast: '224.0.0.251', ttl: 300 }; diff --git a/lib/admin/config.js b/lib/admin/config.js new file mode 100644 index 0000000..9bcb688 --- /dev/null +++ b/lib/admin/config.js @@ -0,0 +1,201 @@ +'use strict'; + +var validator = new (require('jsonschema').Validator)(); +var recase = require('recase').create({}); + +function deepCopy(obj) { + if (!obj || typeof obj !== 'object') { + return obj; + } + + var result; + if (Array.isArray(obj)) { + result = []; + } else { + result = {}; + } + Object.keys(obj).forEach(function (key) { + result[key] = deepCopy(obj[key]); + }); + return result; +} + +var portSchema = { type: 'number', minimum: 1, maximum: 65535 }; + +var moduleSchemas = { + // the proxy module is common to basically all categories. + proxy: { + type: 'object' + , oneOf: [ + { required: [ 'address' ] } + , { required: [ 'port' ] } + ] + , properties: { + address: { type: 'string' } + , host: { type: 'string' } + , port: portSchema + } + } + + // redirect and static modules are for HTTP +, redirect: { + type: 'object' + , required: [ 'to', 'from' ] + , properties: { + to: { type: 'string'} + , from: { type: 'string'} + , status: { type: 'integer', minimum: 1, maximum: 999 } + , } + } +, static: { + type: 'object' + , required: [ 'root' ] + , properties: { + root: { type: 'string' } + } + } + + // the acme module is for TLS +, acme: { + type: 'object' + , required: [ 'email' ] + , properties: { + email: { type: 'string' } + , server: { type: 'string' } + , challengeType: { type: 'string' } + } + } +}; +// forward is basically the name for the TCP proxy +moduleSchemas.forward = deepCopy(moduleSchemas.proxy); +moduleSchemas.forward.required = [ 'ports' ]; +moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema }; + +Object.keys(moduleSchemas).forEach(function (name) { + var schema = moduleSchemas[name]; + schema.id = '/modules/'+name; + schema.required = ['id', 'type'].concat(schema.required || []); + schema.properties.id = { type: 'string' }; + schema.properties.type = { type: 'string', const: name }; + validator.addSchema(schema, schema.id); +}); + +function addDomainsSchema(base, modList) { + var modSchemas = modList.map(function (name) { + return { '$ref': '/modules/'+name }; + }); + + base.required = [ 'modules', 'domains' ].concat(base.required || []); + base.properties.modules = { + type: 'array' + , items: { + type: 'object' + , required: [ 'domains' ] + , properties: { + domains: { type: 'array', items: { type: 'string' }, minLength: 1} + } + , oneOf: modSchemas + } + }; + base.properties.domains = { + type: 'array' + , items: { + type: 'object' + , required: [ 'id', 'names', ] + , properties: { + id: { type: 'string' } + , names: { type: 'array', items: { type: 'string' }, minLength: 1} + , modules: { type: 'array', items: { oneOf: modSchemas }} + } + } + }; +} + +var httpSchema = { + type: 'object' +, properties: { + // These properties should be snake_case to match the API and config format + primary_domain: { type: 'string' } + , allow_insecure: { type: 'boolean' } + , trust_proxy: { type: 'boolean' } +, } +}; +addDomainsSchema(httpSchema, ['proxy', 'static', 'redirect']); + +var tlsSchema = { + type: 'object' +, properties: { + acme: { + type: 'object' + // These properties should be snake_case to match the API and config format + , required: [ 'email', 'approved_domains' ] + , properties: { + email: { type: 'string' } + , server: { type: 'string' } + , challenge_type: { type: 'string' } + , approved_domains: { type: 'array', items: { type: 'string' }, minLength: 1} + } + } + } +}; +addDomainsSchema(tlsSchema, ['proxy', 'acme']); + +var tcpSchema = { + type: 'object' +, required: [ 'bind' ] +, properties: { + bind: { type: 'array', items: portSchema, minLength: 1 } + , modules: { type: 'array', items: { '$ref': '/modules/forward' }} + } +}; + +var dnsSchema = { + type: 'object' +, properties: { + bind: { type: 'array', items: portSchema } + , modules: { type: 'array', items: { '$ref': '/modules/proxy' }} + } +}; + +var mdnsSchema = { + type: 'object' +, required: [ 'port', 'broadcast', 'ttl' ] +, properties: { + port: portSchema + , broadcast: { type: 'string' } + , ttl: { type: 'integer', minimum: 0, maximum: 2147483647 } + } +}; + +var ddnsSchema = { + type: 'object' +, properties: { + enabled: { type: 'boolean' } + } +}; +var socks5Schema = { + type: 'object' +, properties: { + enabled: { type: 'boolean' } + , port: portSchema + } +}; + +var mainSchema = { + type: 'object' +, required: [ 'http', 'tls', 'tcp', 'dns', 'mdns', 'ddns' ] +, properties: { + http: httpSchema + , tls: tlsSchema + , tcp: tcpSchema + , dns: dnsSchema + , mdns: mdnsSchema + , ddns: ddnsSchema + , socks5: socks5Schema + } +, additionalProperties: false +}; + +module.exports.validate = function (config) { + return validator.validate(recase.snakeCopy(config), mainSchema).errors; +}; diff --git a/package-lock.json b/package-lock.json index 11aca53..a09087b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1029,6 +1029,11 @@ "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" }, + "jsonschema": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/jsonschema/-/jsonschema-1.2.0.tgz", + "integrity": "sha512-XDJApzBauMg0TinJNP4iVcJl99PQ4JbWKK7nwzpOIkAOVveDKgh/2xm41T3x7Spu4PWMhnnQpNJmUSIUgl6sKg==" + }, "jsonwebtoken": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-7.4.1.tgz", @@ -1967,14 +1972,6 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" }, - "stream-pair": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-pair/-/stream-pair-1.0.3.tgz", - "integrity": "sha1-vIdY/jnTgQuva3VMj5BI8PuRNn0=", - "requires": { - "readable-stream": "2.2.11" - } - }, "string_decoder": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.2.tgz", diff --git a/package.json b/package.json index c5e56ee..e764379 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "human-readable-ids": "git+https://git.daplie.com/Daplie/human-readable-ids-js#master", "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0", "js-yaml": "^3.8.3", + "jsonschema": "^1.2.0", "jsonwebtoken": "^7.4.0", "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master", "le-challenge-fs": "git+https://git.daplie.com/Daplie/le-challenge-webroot.git#master", From 485a223f86fe48a0307e74e78feb7587f87d9460 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 10 Oct 2017 11:08:19 -0600 Subject: [PATCH 16/41] implemented better management of arrays in the config --- lib/admin/apis.js | 115 +++++++++++++++++++++++++++++-- lib/admin/config.js | 162 ++++++++++++++++++++++++++++++++++++++------ lib/storage.js | 20 ++++++ lib/worker.js | 23 +++++-- 4 files changed, 287 insertions(+), 33 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 902ec15..2d817f9 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -403,13 +403,108 @@ module.exports.create = function (deps, conf) { next(); } }; - config.restful.saveConfig = function (req, res) { - console.log('config POST body', req.body); - // Since we are sending the changes to another process we don't really - // have a good way of seeing if it worked, so always report success - deps.storage.config.save(req.body); - res.send({ success: true }); + config.restful.saveBaseConfig = function (req, res) { + console.log('config POST body', JSON.stringify(req.body)); + + deps.PromiseA.resolve().then(function () { + var update; + if (req.params.group) { + update = {}; + update[req.params.group] = req.body; + } else { + update = req.body; + } + + var changer = new (require('./config').ConfigChanger)(conf); + var errors = changer.update(update); + if (errors.length) { + throw Object.assign(new Error(), errors[0], {statusCode: 400}); + } + + return deps.storage.config.save(changer); + }).then(function (config) { + if (req.params.group) { + config = config[req.params.group]; + } + res.send(deps.recase.snakeCopy(config)); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.createModule = function (req, res) { + var group = req.params.group; + var err; + deps.PromiseA.resolve().then(function () { + var changer = new (require('./config').ConfigChanger)(conf); + if (!changer[group] || !changer[group].modules) { + err = new Error("'"+group+"' is not a valid settings group or has not modules"); + err.statusCode = 404; + throw err; + } + + var modList; + if (req.params.id) { + if (changer[group].domains) { + modList = (changer[group].domains.find(req.params.id) || {}).modules; + } + } else { + modList = changer[group].modules; + } + if (!modList) { + err = new Error("'"+group+"' has no domains list or '"+req.params.id+"' does not exist"); + err.statusCode = 404; + throw err; + } + + modList.add(req.body); + var errors = changer.validate(); + if (errors.length) { + throw Object.assign(new Error(), errors[0], {statusCode: 400}); + } + + return deps.storage.config.save(changer); + }).then(function (config) { + var base; + if (!req.params.id) { + base = config[group]; + } else { + base = config[group].domains.find(function (dom) { return dom.id === req.params.id; }); + } + res.send(deps.recase.snakeCopy(base.modules)); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.createDomain = function (req, res) { + var group = req.params.group; + var err; + deps.PromiseA.resolve().then(function () { + var changer = new (require('./config').ConfigChanger)(conf); + if (!changer[group] || !changer[group].domains) { + err = new Error("'"+group+"' is not a valid settings group or has no domains list"); + err.statusCode = 404; + throw err; + } + + changer[group].domains.add(req.body); + var errors = changer.validate(); + if (errors.length) { + throw Object.assign(new Error(), errors[0], {statusCode: 400}); + } + + return deps.storage.config.save(changer); + }).then(function (config) { + res.send(deps.recase.snakeCopy(config[group].domains)); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); }; var app = require('express')(); @@ -425,6 +520,12 @@ module.exports.create = function (deps, conf) { app.get( '/config/:group', config.restful.readConfig); app.get( '/config/:group/:name(modules|domains)/:id?', config.restful.readConfig); app.get( '/config/:group/:name(domains)/:id/:name2(modules)/:id2?', config.restful.readConfig); - app.post( '/config', config.restful.saveConfig); + + app.post( '/config', config.restful.saveBaseConfig); + app.post( '/config/:group', config.restful.saveBaseConfig); + app.post( '/config/:group/modules', config.restful.createModule); + app.post( '/config/:group/domains', config.restful.createDomain); + app.post( '/config/:group/domains/:id/modules', config.restful.createModule); + return app; }; diff --git a/lib/admin/config.js b/lib/admin/config.js index 9bcb688..4651f09 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -3,23 +3,6 @@ var validator = new (require('jsonschema').Validator)(); var recase = require('recase').create({}); -function deepCopy(obj) { - if (!obj || typeof obj !== 'object') { - return obj; - } - - var result; - if (Array.isArray(obj)) { - result = []; - } else { - result = {}; - } - Object.keys(obj).forEach(function (key) { - result[key] = deepCopy(obj[key]); - }); - return result; -} - var portSchema = { type: 'number', minimum: 1, maximum: 65535 }; var moduleSchemas = { @@ -67,7 +50,7 @@ var moduleSchemas = { } }; // forward is basically the name for the TCP proxy -moduleSchemas.forward = deepCopy(moduleSchemas.proxy); +moduleSchemas.forward = JSON.parse(JSON.stringify(moduleSchemas.proxy)); moduleSchemas.forward.required = [ 'ports' ]; moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema }; @@ -180,6 +163,12 @@ var socks5Schema = { , port: portSchema } }; +var deviceSchema = { + type: 'object' +, properties: { + hostname: { type: 'string' } + } +}; var mainSchema = { type: 'object' @@ -192,10 +181,143 @@ var mainSchema = { , mdns: mdnsSchema , ddns: ddnsSchema , socks5: socks5Schema + , device: deviceSchema } , additionalProperties: false }; -module.exports.validate = function (config) { +function validate(config) { return validator.validate(recase.snakeCopy(config), mainSchema).errors; -}; +} +module.exports.validate = validate; + + +class ModuleList extends Array { + constructor(rawList) { + super(); + if (Array.isArray(rawList)) { + Object.assign(this, JSON.parse(JSON.stringify(rawList))); + } + } + + find(id) { + return Array.prototype.find.call(this, function (mod) { + return mod.id === id; + }); + } + add(mod) { + if (!mod.type) { + throw new Error("module must have a 'type' defined"); + } + if (!moduleSchemas[mod.type]) { + throw new Error("invalid module type '"+mod.type+"'"); + } + + mod.id = require('crypto').randomBytes(4).toString('hex'); + this.push(mod); + } +} +class DomainList extends Array { + constructor(rawList) { + super(); + if (Array.isArray(rawList)) { + Object.assign(this, JSON.parse(JSON.stringify(rawList))); + } + this.forEach(function (dom) { + dom.modules = new ModuleList(dom.modules); + }); + } + + find(id) { + return Array.prototype.find.call(this, function (dom) { + return dom.id === id; + }); + } + add(dom) { + if (!Array.isArray(dom.names) || !dom.names.length) { + throw new Error("domains must have a non-empty array for 'names'"); + } + if (dom.names.some(function (name) { return typeof name !== 'string'; })) { + throw new Error("all domain names must be strings"); + } + + var modList = new ModuleList(); + if (Array.isArray(dom.modules)) { + dom.modules.forEach(function (mod) { + modList.add(mod); + }); + } + + dom.id = require('crypto').randomBytes(4).toString('hex'); + dom.modules = modList; + this.push(dom); + } +} + +class ConfigChanger { + constructor(start) { + Object.assign(this, JSON.parse(JSON.stringify(start))); + delete this.device; + + this.http.modules = new ModuleList(this.http.modules); + this.http.domains = new DomainList(this.http.domains); + this.tls.modules = new ModuleList(this.tls.modules); + this.tls.domains = new DomainList(this.tls.domains); + this.tcp.modules = new ModuleList(this.tcp.modules); + this.dns.modules = new ModuleList(this.dns.modules); + } + + update(update) { + var self = this; + + if (update.http && update.http.modules) { + update.http.modules.forEach(self.http.modules.add.bind(self.http.modules)); + delete update.http.modules; + } + if (update.http && update.http.domains) { + update.http.domains.forEach(self.http.domains.add.bind(self.http.domains)); + delete update.http.domains; + } + + if (update.tls && update.tls.modules) { + update.tls.modules.forEach(self.tls.modules.add.bind(self.tls.modules)); + delete update.tls.modules; + } + if (update.tls && update.tls.domains) { + update.tls.domains.forEach(self.tls.domains.add.bind(self.tls.domains)); + delete update.tls.domains; + } + + if (update.tcp && update.tcp.modules) { + update.tcp.modules.forEach(self.tcp.modules.add.bind(self.tcp.modules)); + delete update.tcp.modules; + } + if (update.dns && update.dns.modules) { + update.dns.modules.forEach(self.dns.modules.add.bind(self.dns.modules)); + delete update.dns.modules; + } + + function mergeSettings(orig, changes) { + Object.keys(changes).forEach(function (key) { + // TODO: use an API that can properly handle updating arrays. + if (!changes[key] || (typeof changes[key] !== 'object') || Array.isArray(changes[key])) { + orig[key] = changes[key]; + } + else if (!orig[key] || typeof orig[key] !== 'object') { + orig[key] = changes[key]; + } + else { + mergeSettings(orig[key], changes[key]); + } + }); + } + mergeSettings(this, update); + + return validate(this); + } + + validate() { + return validate(this); + } +} +module.exports.ConfigChanger = ConfigChanger; diff --git a/lib/storage.js b/lib/storage.js index 4651f2d..33fc8f6 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -65,14 +65,33 @@ module.exports.create = function (deps, conf) { } }; + var confCb; var config = { save: function (changes) { deps.messenger.send({ type: 'com.daplie.goldilocks/config' , changes: changes }); + + return new deps.PromiseA(function (resolve, reject) { + var timeoutId = setTimeout(function () { + reject(new Error('Did not receive config update from main process in a reasonable time')); + confCb = null; + }, 15*1000); + + confCb = function (config) { + confCb = null; + clearTimeout(timeoutId); + resolve(config); + }; + }); } }; + function updateConf(config) { + if (confCb) { + confCb(config); + } + } var mdnsId = { _filename: 'mdns-id' @@ -99,6 +118,7 @@ module.exports.create = function (deps, conf) { return { owners: owners , config: config + , updateConf: updateConf , mdnsId: mdnsId }; }; diff --git a/lib/worker.js b/lib/worker.js index d7ad1c1..c219b96 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -1,6 +1,7 @@ 'use strict'; var config; +var modules; // Everything that uses the config should be reading it when relevant rather than // just at the beginning, so we keep the reference for the main object and just @@ -15,7 +16,13 @@ function update(conf) { config[key] = conf[key]; } }); - console.log('config', JSON.stringify(config)); + + console.log('config update', JSON.stringify(config)); + Object.values(modules).forEach(function (mod) { + if (typeof mod.updateConf === 'function') { + mod.updateConf(config); + } + }); } function create(conf) { @@ -38,11 +45,15 @@ function create(conf) { // HTTP proxying connection creation is not something we currently control. , net: require('net') }; - deps.storage = require('./storage').create(deps, conf); - deps.proxy = require('./proxy-conn').create(deps, conf); - deps.socks5 = require('./socks5-server').create(deps, conf); - deps.loopback = require('./loopback').create(deps, conf); - deps.ddns = require('./ddns').create(deps, conf); + + modules = { + storage: require('./storage').create(deps, conf) + , proxy: require('./proxy-conn').create(deps, conf) + , socks5: require('./socks5-server').create(deps, conf) + , loopback: require('./loopback').create(deps, conf) + , ddns: require('./ddns').create(deps, conf) + }; + Object.assign(deps, modules); require('./goldilocks.js').create(deps, conf); process.removeListener('message', create); From 8371170a141e568d3bfdaa0c9cfb6b58206fe5d7 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 10 Oct 2017 11:32:18 -0600 Subject: [PATCH 17/41] renamed `dns` settings to `udp` --- bin/goldilocks.js | 6 ++++++ lib/admin/config.js | 14 +++++++------- lib/goldilocks.js | 12 ++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index abca131..eacacd4 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -67,6 +67,12 @@ function fixRawConfig(config) { updateDomains(config[key].domains); }); + if (config.dns) { + config.udp = config.dns; + delete config.dns; + updated = true; + } + return updated; } async function createStorage(filename, filetype) { diff --git a/lib/admin/config.js b/lib/admin/config.js index 4651f09..09d7a08 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -132,7 +132,7 @@ var tcpSchema = { } }; -var dnsSchema = { +var udpSchema = { type: 'object' , properties: { bind: { type: 'array', items: portSchema } @@ -172,12 +172,12 @@ var deviceSchema = { var mainSchema = { type: 'object' -, required: [ 'http', 'tls', 'tcp', 'dns', 'mdns', 'ddns' ] +, required: [ 'http', 'tls', 'tcp', 'udp', 'mdns', 'ddns' ] , properties: { http: httpSchema , tls: tlsSchema , tcp: tcpSchema - , dns: dnsSchema + , udp: udpSchema , mdns: mdnsSchema , ddns: ddnsSchema , socks5: socks5Schema @@ -264,7 +264,7 @@ class ConfigChanger { this.tls.modules = new ModuleList(this.tls.modules); this.tls.domains = new DomainList(this.tls.domains); this.tcp.modules = new ModuleList(this.tcp.modules); - this.dns.modules = new ModuleList(this.dns.modules); + this.udp.modules = new ModuleList(this.udp.modules); } update(update) { @@ -292,9 +292,9 @@ class ConfigChanger { update.tcp.modules.forEach(self.tcp.modules.add.bind(self.tcp.modules)); delete update.tcp.modules; } - if (update.dns && update.dns.modules) { - update.dns.modules.forEach(self.dns.modules.add.bind(self.dns.modules)); - delete update.dns.modules; + if (update.udp && update.udp.modules) { + update.udp.modules.forEach(self.udp.modules.add.bind(self.udp.modules)); + delete update.udp.modules; } function mergeSettings(orig, changes) { diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 3e3ee28..14c5f25 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -96,11 +96,11 @@ module.exports.create = function (deps, config) { } function dnsListener(msg) { - if (!Array.isArray(config.dns.modules)) { + if (!Array.isArray(config.udp.modules)) { return; } var socket = require('dgram').createSocket('udp4'); - config.dns.modules.forEach(function (mod) { + config.udp.modules.forEach(function (mod) { if (mod.type !== 'proxy') { console.warn('found bad DNS module', mod); return; @@ -240,13 +240,13 @@ module.exports.create = function (deps, config) { listenPromises.push(listeners.tcp.add(port, netHandler)); }); - if (config.dns.bind) { - if (Array.isArray(config.dns.bind)) { - config.dns.bind.map(function (port) { + if (config.udp.bind) { + if (Array.isArray(config.udp.bind)) { + config.udp.bind.map(function (port) { listenPromises.push(listeners.udp.add(port, dnsListener)); }); } else { - listenPromises.push(listeners.udp.add(config.dns.bind, dnsListener)); + listenPromises.push(listeners.udp.add(config.udp.bind, dnsListener)); } } From ea55d3cc7309dfd8aeb66eccd1748c18fb644dce Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 10 Oct 2017 12:34:32 -0600 Subject: [PATCH 18/41] removed `bind` from the `http` and `tls` settings --- bin/goldilocks.js | 17 +++++++++++++++++ lib/admin/config.js | 6 +++++- lib/goldilocks.js | 23 +++-------------------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index eacacd4..dd2907a 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -67,6 +67,23 @@ function fixRawConfig(config) { updateDomains(config[key].domains); }); + if (config.tcp && config.tcp && !Array.isArray(config.tcp)) { + config.tcp.bind = [ config.tcp.bind ]; + updated = true; + } + if (config.http && config.http.bind) { + config.tcp = config.tcp || { bind: [] }; + config.tcp.bind = (config.tcp.bind || []).concat(config.http.bind); + delete config.http.bind; + updated = true; + } + if (config.tls && config.tls.bind) { + config.tcp = config.tcp || { bind: [] }; + config.tcp.bind = (config.tcp.bind || []).concat(config.tls.bind); + delete config.tls.bind; + updated = true; + } + if (config.dns) { config.udp = config.dns; delete config.dns; diff --git a/lib/admin/config.js b/lib/admin/config.js index 09d7a08..e45a3c2 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -101,7 +101,9 @@ var httpSchema = { primary_domain: { type: 'string' } , allow_insecure: { type: 'boolean' } , trust_proxy: { type: 'boolean' } -, } + + , bind: { not: {} } // this is a forbidden deprecated setting. + } }; addDomainsSchema(httpSchema, ['proxy', 'static', 'redirect']); @@ -117,6 +119,8 @@ var tlsSchema = { , server: { type: 'string' } , challenge_type: { type: 'string' } , approved_domains: { type: 'array', items: { type: 'string' }, minLength: 1} + + , bind: { not: {} } // this is a forbidden deprecated setting. } } } diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 14c5f25..917a5bb 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -197,21 +197,10 @@ module.exports.create = function (deps, config) { var listenPromises = []; var tcpPortMap = {}; - function addPorts(bindList) { - if (!bindList) { - return; - } - if (Array.isArray(bindList)) { - bindList.filter(Number).forEach(function (port) { - tcpPortMap[port] = true; - }); - } - else if (Number(bindList)) { - tcpPortMap[bindList] = true; - } - } + config.tcp.bind.filter(Number).forEach(function (port) { + tcpPortMap[port] = true; + }); - addPorts(config.tcp.bind); (config.tcp.modules || []).forEach(function (mod) { if (mod.type === 'forward') { var forwarder = createTcpForwarder(mod); @@ -229,12 +218,6 @@ module.exports.create = function (deps, config) { } }); - // Even though these ports were specified in different places we treat any TCP - // connections we haven't been told to just forward exactly as is equal so that - // we can potentially use the same ports for different protocols. - addPorts(config.tls.bind); - addPorts(config.http.bind); - var portList = Object.keys(tcpPortMap).map(Number).sort(); portList.forEach(function (port) { listenPromises.push(listeners.tcp.add(port, netHandler)); From 61af4707eeebc33a1cb1ef869096b9dfc083af20 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 11 Oct 2017 12:11:20 -0600 Subject: [PATCH 19/41] moved domains up a level to allow multiple module groups with same domain names --- bin/goldilocks.js | 101 ++++++++++++++++++++++++++++---------------- lib/modules/http.js | 18 +++++--- lib/modules/tls.js | 20 ++++++--- 3 files changed, 89 insertions(+), 50 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index dd2907a..5346ec5 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -30,43 +30,6 @@ function mergeSettings(orig, changes) { function fixRawConfig(config) { var updated = false; - function updateModules(list) { - if (!Array.isArray(list)) { - return; - } - list.forEach(function (mod) { - if (!mod.id) { - mod.id = crypto.randomBytes(4).toString('hex'); - updated = true; - } - if (mod.name) { - mod.type = mod.type || mod.name; - delete mod.name; - updated = true; - } - }); - } - function updateDomains(list) { - if (!Array.isArray(list)) { - return; - } - list.forEach(function (mod) { - if (!mod.id) { - mod.id = crypto.randomBytes(8).toString('hex'); - updated = true; - } - updateModules(mod.modules); - }); - } - - [ 'dns', 'tcp', 'http', 'tls' ].forEach(function (key) { - if (!config[key]) { - return; - } - updateModules(config[key].modules); - updateDomains(config[key].domains); - }); - if (config.tcp && config.tcp && !Array.isArray(config.tcp)) { config.tcp.bind = [ config.tcp.bind ]; updated = true; @@ -90,6 +53,70 @@ function fixRawConfig(config) { updated = true; } + function updateModules(list) { + if (!Array.isArray(list)) { + return; + } + list.forEach(function (mod) { + if (!mod.id) { + mod.id = crypto.randomBytes(4).toString('hex'); + updated = true; + } + if (mod.name) { + mod.type = mod.type || mod.name; + delete mod.name; + updated = true; + } + }); + } + function moveDomains(name) { + if (!config[name].domains) { + return; + } + updated = true; + var domList = config[name].domains; + delete config[name].domains; + + if (!Array.isArray(domList)) { + return; + } + if (!Array.isArray(config.domains)) { + config.domains = []; + } + domList.forEach(function (dom) { + updateModules(dom.modules); + + var strDoms = dom.names.slice().sort().join(','); + var added = config.domain.some(function (existing) { + if (strDoms !== existing.names.slice().sort().join(',')) { + return; + } + existing.modules = existing.modules || {}; + existing.modules[name] = (existing.modules[name] || []).concat(dom.modules); + return true; + }); + if (added) { + return; + } + + var newDom = { + id: crypto.randomBytes(8).toString('hex'), + names: dom.names, + modules: {} + }; + newDom.modules[name] = dom.modules; + config.domains.push(newDom); + }); + } + + [ 'udp', 'tcp', 'http', 'tls' ].forEach(function (key) { + if (!config[key]) { + return; + } + updateModules(config[key].modules); + moveDomains(key); + }); + return updated; } async function createStorage(filename, filetype) { diff --git a/lib/modules/http.js b/lib/modules/http.js index ed3c571..a548260 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -65,18 +65,21 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { }); } - function hostMatchesDomains(req, domains) { + function hostMatchesDomains(req, domainList) { var host = separatePort((req.headers || req).host).host.toLowerCase(); - return domains.some(function (pattern) { + return domainList.some(function (pattern) { return domainMatches(pattern, host); }); } function determinePrimaryHost() { var result; - if (Array.isArray(conf.http.domains)) { - conf.http.domains.some(function (dom) { + if (Array.isArray(conf.domains)) { + conf.domains.some(function (dom) { + if (!dom.modules || !dom.modules.http) { + return false; + } return dom.names.some(function (domain) { if (domain[0] !== '*') { result = domain; @@ -415,17 +418,20 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { if (checkAdmin(conn, opts, headers)) { return; } var prom = PromiseA.resolve(false); - (conf.http.domains || []).forEach(function (dom) { + (conf.domains || []).forEach(function (dom) { prom = prom.then(function (handled) { if (handled) { return handled; } + if (!dom.modules || !dom.modules.http) { + return false; + } if (!hostMatchesDomains(headers, dom.names)) { return false; } var subProm = PromiseA.resolve(false); - dom.modules.forEach(function (mod) { + dom.modules.http.forEach(function (mod) { if (moduleChecks[mod.type]) { subProm = subProm.then(function (handled) { if (handled) { return handled; } diff --git a/lib/modules/tls.js b/lib/modules/tls.js index c8191d2..0c30936 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -27,8 +27,8 @@ module.exports.create = function (deps, config, netHandler) { return value || ''; } - function nameMatchesDomains(name, domains) { - return domains.some(function (pattern) { + function nameMatchesDomains(name, domainList) { + return domainList.some(function (pattern) { return domainMatches(pattern, name); }); } @@ -135,13 +135,16 @@ module.exports.create = function (deps, config, netHandler) { } var handled = false; - if (Array.isArray(config.tls.domains)) { - handled = config.tls.domains.some(function (dom) { + if (Array.isArray(config.domains)) { + handled = config.domains.some(function (dom) { + if (!dom.modules || !dom.modules.tls) { + return false; + } if (!nameMatchesDomains(opts.domain, dom.names)) { return false; } - return dom.modules.some(function (mod) { + return dom.modules.tls.some(function (mod) { if (mod.type !== 'acme') { return false; } @@ -330,12 +333,15 @@ module.exports.create = function (deps, config, netHandler) { } } - var handled = (config.tls.domains || []).some(function (dom) { + var handled = (config.domains || []).some(function (dom) { + if (!dom.modules || !dom.modules.tls) { + return false; + } if (!nameMatchesDomains(opts.servername, dom.names)) { return false; } - return dom.modules.some(checkModule); + return dom.modules.tls.some(checkModule); }); if (handled) { return; From 79ef9694b7fbcdce0891610b8c95001b4ca53b33 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 11 Oct 2017 12:18:01 -0600 Subject: [PATCH 20/41] updated API to reflect moved domains --- lib/admin/apis.js | 91 ++++++++++++++------------- lib/admin/config.js | 147 ++++++++++++++++++++++---------------------- 2 files changed, 123 insertions(+), 115 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 2d817f9..8eb82be 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -380,21 +380,21 @@ module.exports.create = function (deps, conf) { var config = { restful: {} }; config.restful.readConfig = function (req, res, next) { - var part = conf; + var part = new (require('./config').ConfigChanger)(conf); if (req.params.group) { part = part[req.params.group]; } - if (part && req.params.name) { - part = part[req.params.name]; + if (part && req.params.domId) { + part = part.domains.find(req.params.domId); } - if (part && req.params.id) { - part = part.find(function (mod) { return mod.id === req.params.id; }); + if (part && req.params.mod) { + part = part[req.params.mod]; } - if (part && req.params.name2) { - part = part[req.params.name2]; + if (part && req.params.modGrp) { + part = part[req.params.modGrp]; } - if (part && req.params.id2) { - part = part.find(function (mod) { return mod.id === req.params.id2; }); + if (part && req.params.modId) { + part = part.find(req.params.modId); } if (part) { @@ -439,27 +439,35 @@ module.exports.create = function (deps, conf) { var err; deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); - if (!changer[group] || !changer[group].modules) { - err = new Error("'"+group+"' is not a valid settings group or has not modules"); - err.statusCode = 404; - throw err; - } - var modList; - if (req.params.id) { - if (changer[group].domains) { - modList = (changer[group].domains.find(req.params.id) || {}).modules; + if (req.params.domId) { + var dom = changer.domains.find(req.params.domId); + if (!dom) { + err = new Error("no domain with ID '"+req.params.domId+"'"); + } else if (!dom.modules[group]) { + err = new Error("domains don't contain '"+group+"' modules"); + } else { + modList = dom.modules[group]; } } else { - modList = changer[group].modules; + if (!changer[group] || !changer[group].modules) { + err = new Error("'"+group+"' is not a valid settings group or doesn't support modules"); + } else { + modList = changer[group].modules; + } } - if (!modList) { - err = new Error("'"+group+"' has no domains list or '"+req.params.id+"' does not exist"); + + if (err) { err.statusCode = 404; throw err; } - modList.add(req.body); + var update = req.body; + if (!Array.isArray(update)) { + update = [ update ]; + } + update.forEach(modList.add, modList); + var errors = changer.validate(); if (errors.length) { throw Object.assign(new Error(), errors[0], {statusCode: 400}); @@ -467,13 +475,13 @@ module.exports.create = function (deps, conf) { return deps.storage.config.save(changer); }).then(function (config) { - var base; - if (!req.params.id) { - base = config[group]; + var result; + if (!req.params.domId) { + result = config[group].modules; } else { - base = config[group].domains.find(function (dom) { return dom.id === req.params.id; }); + result = config.domains.find(function (dom) { return dom.id === req.params.domId; }).modules[group]; } - res.send(deps.recase.snakeCopy(base.modules)); + res.send(deps.recase.snakeCopy(result)); }, function (err) { res.statusCode = err.statusCode || 500; err.message = err.message || err.toString(); @@ -481,17 +489,15 @@ module.exports.create = function (deps, conf) { }); }; config.restful.createDomain = function (req, res) { - var group = req.params.group; - var err; deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); - if (!changer[group] || !changer[group].domains) { - err = new Error("'"+group+"' is not a valid settings group or has no domains list"); - err.statusCode = 404; - throw err; - } - changer[group].domains.add(req.body); + var update = req.body; + if (!Array.isArray(update)) { + update = [ update ]; + } + update.forEach(changer.domains.add, changer.domains); + var errors = changer.validate(); if (errors.length) { throw Object.assign(new Error(), errors[0], {statusCode: 400}); @@ -499,7 +505,7 @@ module.exports.create = function (deps, conf) { return deps.storage.config.save(changer); }).then(function (config) { - res.send(deps.recase.snakeCopy(config[group].domains)); + res.send(deps.recase.snakeCopy(config.domains)); }, function (err) { res.statusCode = err.statusCode || 500; err.message = err.message || err.toString(); @@ -518,14 +524,15 @@ module.exports.create = function (deps, conf) { app.use( '/config', makeCorsHandler()); app.get( '/config', config.restful.readConfig); app.get( '/config/:group', config.restful.readConfig); - app.get( '/config/:group/:name(modules|domains)/:id?', config.restful.readConfig); - app.get( '/config/:group/:name(domains)/:id/:name2(modules)/:id2?', config.restful.readConfig); + app.get( '/config/:group/:mod(modules)/:modId?', config.restful.readConfig); + app.get( '/config/domains/:domId/:mod(modules)?', config.restful.readConfig); + app.get( '/config/domains/:domId/:mod(modules)/:modGrp/:modId?', config.restful.readConfig); app.post( '/config', config.restful.saveBaseConfig); - app.post( '/config/:group', config.restful.saveBaseConfig); - app.post( '/config/:group/modules', config.restful.createModule); - app.post( '/config/:group/domains', config.restful.createDomain); - app.post( '/config/:group/domains/:id/modules', config.restful.createModule); + app.post( '/config/:group(?!domains)', config.restful.saveBaseConfig); + app.post( '/config/:group(?!domains)/modules', config.restful.createModule); + app.post( '/config/domains', config.restful.createDomain); + app.post( '/config/domains/:domId/modules/:group',config.restful.createModule); return app; }; diff --git a/lib/admin/config.js b/lib/admin/config.js index e45a3c2..0cf8646 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -63,54 +63,64 @@ Object.keys(moduleSchemas).forEach(function (name) { validator.addSchema(schema, schema.id); }); -function addDomainsSchema(base, modList) { - var modSchemas = modList.map(function (name) { - return { '$ref': '/modules/'+name }; - }); - - base.required = [ 'modules', 'domains' ].concat(base.required || []); - base.properties.modules = { - type: 'array' - , items: { - type: 'object' - , required: [ 'domains' ] - , properties: { - domains: { type: 'array', items: { type: 'string' }, minLength: 1} - } - , oneOf: modSchemas - } - }; - base.properties.domains = { - type: 'array' - , items: { - type: 'object' - , required: [ 'id', 'names', ] - , properties: { - id: { type: 'string' } - , names: { type: 'array', items: { type: 'string' }, minLength: 1} - , modules: { type: 'array', items: { oneOf: modSchemas }} - } - } - }; +function toSchemaRef(name) { + return { '$ref': '/modules/'+name }; } +var moduleRefs = { + http: [ 'proxy', 'static', 'redirect' ].map(toSchemaRef) +, tls: [ 'proxy', 'acme' ].map(toSchemaRef) +, tcp: [ 'forward' ].map(toSchemaRef) +, udp: [ 'proxy' ].map(toSchemaRef) +}; + +function addDomainRequirement(itemSchema) { + itemSchema.required = (itemSchema.required || []).concat('domains'); + itemSchema.properties = itemSchema.properties || {}; + itemSchema.domains = { type: 'array', items: { type: 'string' }, minLength: 1}; + return itemSchema; +} + +var domainSchema = { + type: 'array' +, items: { + type: 'object' + , properties: { + id: { type: 'string' } + , names: { type: 'array', items: { type: 'string' }, minLength: 1} + , modules: { + type: 'object' + , properties: { + tls: { type: 'array', items: { oneOf: moduleRefs.tls }} + , http: { type: 'array', items: { oneOf: moduleRefs.http }} + } + , additionalProperties: false + } + } + } +}; var httpSchema = { type: 'object' , properties: { + modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.http }) } + // These properties should be snake_case to match the API and config format - primary_domain: { type: 'string' } + , primary_domain: { type: 'string' } , allow_insecure: { type: 'boolean' } , trust_proxy: { type: 'boolean' } - , bind: { not: {} } // this is a forbidden deprecated setting. + // these are forbidden deprecated settings. + , bind: { not: {} } + , domains: { not: {} } } }; -addDomainsSchema(httpSchema, ['proxy', 'static', 'redirect']); var tlsSchema = { type: 'object' , properties: { - acme: { + modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.tls }) } + + , acme: { type: 'object' // These properties should be snake_case to match the API and config format , required: [ 'email', 'approved_domains' ] @@ -120,19 +130,20 @@ var tlsSchema = { , challenge_type: { type: 'string' } , approved_domains: { type: 'array', items: { type: 'string' }, minLength: 1} - , bind: { not: {} } // this is a forbidden deprecated setting. + // these are forbidden deprecated settings. + , bind: { not: {} } + , domains: { not: {} } } } } }; -addDomainsSchema(tlsSchema, ['proxy', 'acme']); var tcpSchema = { type: 'object' , required: [ 'bind' ] , properties: { bind: { type: 'array', items: portSchema, minLength: 1 } - , modules: { type: 'array', items: { '$ref': '/modules/forward' }} + , modules: { type: 'array', items: { oneOf: moduleRefs.tcp }} } }; @@ -140,7 +151,7 @@ var udpSchema = { type: 'object' , properties: { bind: { type: 'array', items: portSchema } - , modules: { type: 'array', items: { '$ref': '/modules/proxy' }} + , modules: { type: 'array', items: { oneOf: moduleRefs.udp }} } }; @@ -176,9 +187,10 @@ var deviceSchema = { var mainSchema = { type: 'object' -, required: [ 'http', 'tls', 'tcp', 'udp', 'mdns', 'ddns' ] +, required: [ 'domains', 'http', 'tls', 'tcp', 'udp', 'mdns', 'ddns' ] , properties: { - http: httpSchema + domains:domainSchema + , http: httpSchema , tls: tlsSchema , tcp: tcpSchema , udp: udpSchema @@ -228,7 +240,10 @@ class DomainList extends Array { Object.assign(this, JSON.parse(JSON.stringify(rawList))); } this.forEach(function (dom) { - dom.modules = new ModuleList(dom.modules); + dom.modules = { + http: new ModuleList((dom.modules || {}).http), + tls: new ModuleList((dom.modules || {}).tls), + }; }); } @@ -245,15 +260,19 @@ class DomainList extends Array { throw new Error("all domain names must be strings"); } - var modList = new ModuleList(); - if (Array.isArray(dom.modules)) { - dom.modules.forEach(function (mod) { - modList.add(mod); - }); + var modLists = { + http: new ModuleList(), + tls: new ModuleList() + }; + if (dom.modules && Array.isArray(dom.modules.http)) { + dom.modules.http.forEach(modLists.http.add, modLists.http); + } + if (dom.modules && Array.isArray(dom.modules.tls)) { + dom.modules.tls.forEach(modLists.tls.add, modLists.tls); } dom.id = require('crypto').randomBytes(4).toString('hex'); - dom.modules = modList; + dom.modules = modLists; this.push(dom); } } @@ -263,10 +282,9 @@ class ConfigChanger { Object.assign(this, JSON.parse(JSON.stringify(start))); delete this.device; + this.domains = new DomainList(this.domains); this.http.modules = new ModuleList(this.http.modules); - this.http.domains = new DomainList(this.http.domains); this.tls.modules = new ModuleList(this.tls.modules); - this.tls.domains = new DomainList(this.tls.domains); this.tcp.modules = new ModuleList(this.tcp.modules); this.udp.modules = new ModuleList(this.udp.modules); } @@ -274,32 +292,15 @@ class ConfigChanger { update(update) { var self = this; - if (update.http && update.http.modules) { - update.http.modules.forEach(self.http.modules.add.bind(self.http.modules)); - delete update.http.modules; - } - if (update.http && update.http.domains) { - update.http.domains.forEach(self.http.domains.add.bind(self.http.domains)); - delete update.http.domains; - } - - if (update.tls && update.tls.modules) { - update.tls.modules.forEach(self.tls.modules.add.bind(self.tls.modules)); - delete update.tls.modules; - } - if (update.tls && update.tls.domains) { - update.tls.domains.forEach(self.tls.domains.add.bind(self.tls.domains)); - delete update.tls.domains; - } - - if (update.tcp && update.tcp.modules) { - update.tcp.modules.forEach(self.tcp.modules.add.bind(self.tcp.modules)); - delete update.tcp.modules; - } - if (update.udp && update.udp.modules) { - update.udp.modules.forEach(self.udp.modules.add.bind(self.udp.modules)); - delete update.udp.modules; + if (update.domains) { + update.domains.forEach(self.domains.add, self.domains); } + [ 'http', 'tls', 'tcp', 'udp' ].forEach(function (name) { + if (update[name] && update[name].modules) { + update[name].modules.forEach(self[name].modules.add, self[name].modules); + delete update[name].modules; + } + }); function mergeSettings(orig, changes) { Object.keys(changes).forEach(function (key) { From 2a57a1e12cc549685c6e753f929c0c5a0ba3ef60 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 11 Oct 2017 13:06:24 -0600 Subject: [PATCH 21/41] fixed a few misc errors that appeared in testing --- bin/goldilocks.js | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 5346ec5..c03e12b 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -30,7 +30,7 @@ function mergeSettings(orig, changes) { function fixRawConfig(config) { var updated = false; - if (config.tcp && config.tcp && !Array.isArray(config.tcp)) { + if (config.tcp && config.tcp.bind && !Array.isArray(config.tcp.bind)) { config.tcp.bind = [ config.tcp.bind ]; updated = true; } @@ -87,7 +87,7 @@ function fixRawConfig(config) { updateModules(dom.modules); var strDoms = dom.names.slice().sort().join(','); - var added = config.domain.some(function (existing) { + var added = config.domains.some(function (existing) { if (strDoms !== existing.names.slice().sort().join(',')) { return; } @@ -100,7 +100,7 @@ function fixRawConfig(config) { } var newDom = { - id: crypto.randomBytes(8).toString('hex'), + id: crypto.randomBytes(4).toString('hex'), names: dom.names, modules: {} }; @@ -260,7 +260,11 @@ function fillConfig(config, args) { var mdnsDefaults = { disabled: false, port: 5353, broadcast: '224.0.0.251', ttl: 300 }; config.mdns = Object.assign(mdnsDefaults, config.mdns); - function fillComponent(name, fillBind, fillDomains) { + if (!Array.isArray(config.domains)) { + config.domains = []; + } + + function fillComponent(name, fillBind) { if (!config[name]) { config[name] = {}; } @@ -271,22 +275,11 @@ function fillConfig(config, args) { if (fillBind && !Array.isArray(config[name].bind)) { config[name].bind = []; } - - if (fillDomains) { - if (!Array.isArray(config[name].domains)) { - config[name].domains = []; - } - config[name].domains.forEach(function (domain) { - if (!Array.isArray(domain.modules)) { - domain.modules = []; - } - }); - } } - fillComponent('dns', true, false); - fillComponent('tcp', true, false); - fillComponent('http', false, true); - fillComponent('tls', false, true); + fillComponent('udp', true); + fillComponent('tcp', true); + fillComponent('http', false); + fillComponent('tls', false); if (!config.tls.acme && (args.email || args.agreeTos)) { config.tls.acme = {}; @@ -384,6 +377,9 @@ function run(args) { // TODO spin up multiple workers // TODO use greenlock-cluster cluster.fork(); + }).catch(function (err) { + console.error(err); + process.exit(1); }) ; } From 503da9efd0ab8278934113b973897a2ef71320e0 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 11 Oct 2017 17:13:33 -0600 Subject: [PATCH 22/41] implemented routes to edit and delete modules and domains --- lib/admin/apis.js | 194 +++++++++++++++++++++++++++++++------------- lib/admin/config.js | 57 +++++++++---- 2 files changed, 178 insertions(+), 73 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 8eb82be..816c4ac 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -385,7 +385,7 @@ module.exports.create = function (deps, conf) { part = part[req.params.group]; } if (part && req.params.domId) { - part = part.domains.find(req.params.domId); + part = part.domains.findId(req.params.domId); } if (part && req.params.mod) { part = part[req.params.mod]; @@ -394,7 +394,7 @@ module.exports.create = function (deps, conf) { part = part[req.params.modGrp]; } if (part && req.params.modId) { - part = part.find(req.params.modId); + part = part.findId(req.params.modId); } if (part) { @@ -404,8 +404,20 @@ module.exports.create = function (deps, conf) { } }; - config.restful.saveBaseConfig = function (req, res) { + config.save = function (changer) { + var errors = changer.validate(); + if (errors.length) { + throw Object.assign(new Error(), errors[0], {statusCode: 400}); + } + + return deps.storage.config.save(changer); + }; + config.restful.saveBaseConfig = function (req, res, next) { console.log('config POST body', JSON.stringify(req.body)); + if (req.params.group === 'domains') { + next(); + return; + } deps.PromiseA.resolve().then(function () { var update; @@ -417,12 +429,8 @@ module.exports.create = function (deps, conf) { } var changer = new (require('./config').ConfigChanger)(conf); - var errors = changer.update(update); - if (errors.length) { - throw Object.assign(new Error(), errors[0], {statusCode: 400}); - } - - return deps.storage.config.save(changer); + changer.update(update); + return config.save(changer); }).then(function (config) { if (req.params.group) { config = config[req.params.group]; @@ -434,33 +442,41 @@ module.exports.create = function (deps, conf) { res.end(JSON.stringify({error: {message: err.message, code: err.code}})); }); }; - config.restful.createModule = function (req, res) { - var group = req.params.group; + + config.extractModList = function (changer, params) { var err; + if (params.domId) { + var dom = changer.domains.find(function (dom) { + return dom.id === params.domId; + }); + + if (!dom) { + err = new Error("no domain with ID '"+params.domId+"'"); + } else if (!dom.modules[params.group]) { + err = new Error("domains don't contain '"+params.group+"' modules"); + } else { + return dom.modules[params.group]; + } + } else { + if (!changer[params.group] || !changer[params.group].modules) { + err = new Error("'"+params.group+"' is not a valid settings group or doesn't support modules"); + } else { + return changer[params.group].modules; + } + } + + err.statusCode = 404; + throw err; + }; + config.restful.createModule = function (req, res, next) { + if (req.params.group === 'domains') { + next(); + return; + } + deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); - var modList; - if (req.params.domId) { - var dom = changer.domains.find(req.params.domId); - if (!dom) { - err = new Error("no domain with ID '"+req.params.domId+"'"); - } else if (!dom.modules[group]) { - err = new Error("domains don't contain '"+group+"' modules"); - } else { - modList = dom.modules[group]; - } - } else { - if (!changer[group] || !changer[group].modules) { - err = new Error("'"+group+"' is not a valid settings group or doesn't support modules"); - } else { - modList = changer[group].modules; - } - } - - if (err) { - err.statusCode = 404; - throw err; - } + var modList = config.extractModList(changer, req.params); var update = req.body; if (!Array.isArray(update)) { @@ -468,26 +484,54 @@ module.exports.create = function (deps, conf) { } update.forEach(modList.add, modList); - var errors = changer.validate(); - if (errors.length) { - throw Object.assign(new Error(), errors[0], {statusCode: 400}); - } - - return deps.storage.config.save(changer); - }).then(function (config) { - var result; - if (!req.params.domId) { - result = config[group].modules; - } else { - result = config.domains.find(function (dom) { return dom.id === req.params.domId; }).modules[group]; - } - res.send(deps.recase.snakeCopy(result)); + return config.save(changer); + }).then(function (newConf) { + res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); }, function (err) { res.statusCode = err.statusCode || 500; err.message = err.message || err.toString(); res.end(JSON.stringify({error: {message: err.message, code: err.code}})); }); }; + config.restful.updateModule = function (req, res, next) { + if (req.params.group === 'domains') { + next(); + return; + } + + deps.PromiseA.resolve().then(function () { + var changer = new (require('./config').ConfigChanger)(conf); + var modList = config.extractModList(changer, req.params); + modList.update(req.params.modId, req.body); + return config.save(changer); + }).then(function (newConf) { + res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.removeModule = function (req, res, next) { + if (req.params.group === 'domains') { + next(); + return; + } + + deps.PromiseA.resolve().then(function () { + var changer = new (require('./config').ConfigChanger)(conf); + var modList = config.extractModList(changer, req.params); + modList.remove(req.params.modId); + return config.save(changer); + }).then(function (newConf) { + res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.createDomain = function (req, res) { deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); @@ -497,13 +541,37 @@ module.exports.create = function (deps, conf) { update = [ update ]; } update.forEach(changer.domains.add, changer.domains); - - var errors = changer.validate(); - if (errors.length) { - throw Object.assign(new Error(), errors[0], {statusCode: 400}); + return config.save(changer); + }).then(function (config) { + res.send(deps.recase.snakeCopy(config.domains)); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.updateDomain = function (req, res) { + deps.PromiseA.resolve().then(function () { + if (req.body.modules) { + throw Object.assign(new Error('do not add modules with this route'), {statusCode: 400}); } - return deps.storage.config.save(changer); + var changer = new (require('./config').ConfigChanger)(conf); + changer.domains.update(req.params.domId, req.body); + return config.save(changer); + }).then(function (config) { + res.send(deps.recase.snakeCopy(config.domains)); + }, function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + }; + config.restful.removeDomain = function (req, res) { + deps.PromiseA.resolve().then(function () { + var changer = new (require('./config').ConfigChanger)(conf); + changer.domains.remove(req.params.domId); + return config.save(changer); }).then(function (config) { res.send(deps.recase.snakeCopy(config.domains)); }, function (err) { @@ -521,18 +589,28 @@ module.exports.create = function (deps, conf) { app.use('/', isAuthorized, jsonParser); - app.use( '/config', makeCorsHandler()); + // Not all config routes support PUT or DELETE, but not worth making this more specific + app.use( '/config', makeCorsHandler(['GET', 'POST', 'PUT', 'DELETE'])); app.get( '/config', config.restful.readConfig); app.get( '/config/:group', config.restful.readConfig); app.get( '/config/:group/:mod(modules)/:modId?', config.restful.readConfig); app.get( '/config/domains/:domId/:mod(modules)?', config.restful.readConfig); app.get( '/config/domains/:domId/:mod(modules)/:modGrp/:modId?', config.restful.readConfig); - app.post( '/config', config.restful.saveBaseConfig); - app.post( '/config/:group(?!domains)', config.restful.saveBaseConfig); - app.post( '/config/:group(?!domains)/modules', config.restful.createModule); - app.post( '/config/domains', config.restful.createDomain); - app.post( '/config/domains/:domId/modules/:group',config.restful.createModule); + app.post( '/config', config.restful.saveBaseConfig); + app.post( '/config/:group', config.restful.saveBaseConfig); + + app.post( '/config/:group/modules', config.restful.createModule); + app.put( '/config/:group/modules/:modId', config.restful.updateModule); + app.delete('/config/:group/modules/:modId', config.restful.removeModule); + + app.post( '/config/domains/:domId/modules/:group', config.restful.createModule); + app.put( '/config/domains/:domId/modules/:group/:modId', config.restful.updateModule); + app.delete('/config/domains/:domId/modules/:group/:modId', config.restful.removeModule); + + app.post( '/config/domains', config.restful.createDomain); + app.put( '/config/domains/:domId', config.restful.updateDomain); + app.delete('/config/domains/:domId', config.restful.removeDomain); return app; }; diff --git a/lib/admin/config.js b/lib/admin/config.js index 0cf8646..26dcbb7 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -207,20 +207,54 @@ function validate(config) { } module.exports.validate = validate; - -class ModuleList extends Array { +class IdList extends Array { constructor(rawList) { super(); if (Array.isArray(rawList)) { Object.assign(this, JSON.parse(JSON.stringify(rawList))); } + this._itemName = 'item'; } - find(id) { - return Array.prototype.find.call(this, function (mod) { - return mod.id === id; + findId(id) { + return Array.prototype.find.call(this, function (dom) { + return dom.id === id; }); } + + add(item) { + item.id = require('crypto').randomBytes(4).toString('hex'); + this.push(item); + } + + update(id, update) { + var item = this.findId(id); + if (!item) { + var error = new Error("no "+this._itemName+" with ID '"+id+"'"); + error.statusCode = 404; + throw error; + } + Object.assign(this.findId(id), update); + } + + remove(id) { + var index = this.findIndex(function (dom) { + return dom.id === id; + }); + if (index < 0) { + var error = new Error("no "+this._itemName+" with ID '"+id+"'"); + error.statusCode = 404; + throw error; + } + this.splice(index, 1); + } +} +class ModuleList extends IdList { + constructor(rawList) { + super(rawList); + this._itemName = 'module'; + } + add(mod) { if (!mod.type) { throw new Error("module must have a 'type' defined"); @@ -233,12 +267,10 @@ class ModuleList extends Array { this.push(mod); } } -class DomainList extends Array { +class DomainList extends IdList { constructor(rawList) { - super(); - if (Array.isArray(rawList)) { - Object.assign(this, JSON.parse(JSON.stringify(rawList))); - } + super(rawList); + this._itemName = 'domain'; this.forEach(function (dom) { dom.modules = { http: new ModuleList((dom.modules || {}).http), @@ -247,11 +279,6 @@ class DomainList extends Array { }); } - find(id) { - return Array.prototype.find.call(this, function (dom) { - return dom.id === id; - }); - } add(dom) { if (!Array.isArray(dom.names) || !dom.names.length) { throw new Error("domains must have a non-empty array for 'names'"); From 0406d0cd93b3da151d77d4f6e5d41623b65b517b Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 12 Oct 2017 11:57:43 -0600 Subject: [PATCH 23/41] removed the `acme` property from the `tls` config --- bin/goldilocks.js | 43 ++++++++++++++++++++++++++++++++++++++++--- lib/admin/config.js | 33 +++++++++++---------------------- lib/modules/tls.js | 20 -------------------- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index c03e12b..21717de 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -30,6 +30,8 @@ function mergeSettings(orig, changes) { function fixRawConfig(config) { var updated = false; + // First converge all of the `bind` properties for protocols that are on top + // of TCP to `tcp.bind`. if (config.tcp && config.tcp.bind && !Array.isArray(config.tcp.bind)) { config.tcp.bind = [ config.tcp.bind ]; updated = true; @@ -47,12 +49,47 @@ function fixRawConfig(config) { updated = true; } + // Then we rename dns to udp since the only thing we currently do with those + // modules is proxy the packets without inspecting them at all. if (config.dns) { config.udp = config.dns; delete config.dns; updated = true; } + // This we take the old way of defining ACME options and put them into a tls module. + if (config.tls) { + var oldPropMap = { + email: 'email' + , acme_directory_url: 'server' + , challenge_type: 'challenge_type' + , servernames: 'approved_domains' + }; + if (Object.keys(oldPropMap).some(config.tls.hasOwnProperty, config.tls)) { + updated = true; + if (config.tls.acme) { + console.warn('TLS config has `acme` field and old style definitions'); + } else { + config.tls.acme = {}; + Object.keys(oldPropMap).forEach(function (oldKey) { + if (config.tls[oldKey]) { + config.tls.acme[oldPropMap[oldKey]] = config.tls[oldKey]; + } + }); + } + } + if (config.tls.acme) { + updated = true; + config.tls.acme.domains = config.tls.acme.approved_domains; + delete config.tls.acme.approved_domains; + config.tls.modules = config.tls.modules || []; + config.tls.modules.push(Object.assign({}, config.tls.acme, {type: 'acme'})); + delete config.tls.acme; + } + } + + // Then we make sure all modules have an ID and type, and makes sure all domains + // are in the right spot and also have an ID. function updateModules(list) { if (!Array.isArray(list)) { return; @@ -100,9 +137,9 @@ function fixRawConfig(config) { } var newDom = { - id: crypto.randomBytes(4).toString('hex'), - names: dom.names, - modules: {} + id: crypto.randomBytes(4).toString('hex') + , names: dom.names + , modules: {} }; newDom.modules[name] = dom.modules; config.domains.push(newDom); diff --git a/lib/admin/config.js b/lib/admin/config.js index 26dcbb7..9cdb3a9 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -43,9 +43,9 @@ var moduleSchemas = { type: 'object' , required: [ 'email' ] , properties: { - email: { type: 'string' } - , server: { type: 'string' } - , challengeType: { type: 'string' } + email: { type: 'string' } + , server: { type: 'string' } + , challenge_type: { type: 'string' } } } }; @@ -120,21 +120,10 @@ var tlsSchema = { , properties: { modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.tls }) } - , acme: { - type: 'object' - // These properties should be snake_case to match the API and config format - , required: [ 'email', 'approved_domains' ] - , properties: { - email: { type: 'string' } - , server: { type: 'string' } - , challenge_type: { type: 'string' } - , approved_domains: { type: 'array', items: { type: 'string' }, minLength: 1} - - // these are forbidden deprecated settings. - , bind: { not: {} } - , domains: { not: {} } - } - } + // these are forbidden deprecated settings. + , acme: { not: {} } + , bind: { not: {} } + , domains: { not: {} } } }; @@ -273,8 +262,8 @@ class DomainList extends IdList { this._itemName = 'domain'; this.forEach(function (dom) { dom.modules = { - http: new ModuleList((dom.modules || {}).http), - tls: new ModuleList((dom.modules || {}).tls), + http: new ModuleList((dom.modules || {}).http) + , tls: new ModuleList((dom.modules || {}).tls) }; }); } @@ -288,8 +277,8 @@ class DomainList extends IdList { } var modLists = { - http: new ModuleList(), - tls: new ModuleList() + http: new ModuleList() + , tls: new ModuleList() }; if (dom.modules && Array.isArray(dom.modules.http)) { dom.modules.http.forEach(modLists.http.add, modLists.http); diff --git a/lib/modules/tls.js b/lib/modules/tls.js index 0c30936..2b9a614 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -174,26 +174,6 @@ module.exports.create = function (deps, config, netHandler) { return; } - var defAcmeConf; - if (config.tls.acme) { - defAcmeConf = config.tls.acme; - } else { - defAcmeConf = { - email: config.tls.email - , server: config.tls.acmeDirectoryUrl || le.server - , challengeType: config.tls.challengeType || le.challengeType - , approvedDomains: config.tls.servernames - }; - } - - // Check config for domain name - // TODO: if `approvedDomains` isn't defined check all other modules to see if they can - // handle this domain (and what other domains it's grouped with). - if (-1 !== (defAcmeConf.approvedDomains || []).indexOf(opts.domain)) { - complete(defAcmeConf, defAcmeConf.approvedDomains); - return; - } - cb(new Error('domain is not allowed')); } }); From 663fdba446a1cfa02a29cd98a2bffe22912e7fdd Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 12 Oct 2017 14:35:19 -0600 Subject: [PATCH 24/41] changed the valid UDP module from 'proxy' to 'forward' forward is based on incoming port, while proxy is based on domains and we don't have any domain names for raw UDP or TCP --- bin/goldilocks.js | 21 +++++++++++++++++++++ lib/admin/config.js | 5 +++-- lib/goldilocks.js | 18 +++++++++--------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 21717de..06a340b 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -56,6 +56,27 @@ function fixRawConfig(config) { delete config.dns; updated = true; } + // Convert all 'proxy' UDP modules to 'forward' modules that specify which + // incoming ports are relevant. Primarily to make 'proxy' modules consistent + // in needing relevant domain names. + if (config.udp && !Array.isArray(config.udp.bind)) { + config.udp.bind = [].concat(config.udp.bind || []); + updated = true; + } + if (config.udp && config.udp.modules) { + if (!config.udp.bind.length || !Array.isArray(config.udp.modules)) { + delete config.udp.modules; + updated = true; + } else { + config.udp.modules.forEach(function (mod) { + if (mod.type === 'proxy') { + mod.type = 'forward'; + mod.ports = config.udp.bind.slice(); + updated = true; + } + }); + } + } // This we take the old way of defining ACME options and put them into a tls module. if (config.tls) { diff --git a/lib/admin/config.js b/lib/admin/config.js index 9cdb3a9..1ac3dd1 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -49,7 +49,8 @@ var moduleSchemas = { } } }; -// forward is basically the name for the TCP proxy +// forward is basically the same as proxy, but specifies the relevant incoming port(s). +// only allows for the raw transport layers (TCP/UDP) moduleSchemas.forward = JSON.parse(JSON.stringify(moduleSchemas.proxy)); moduleSchemas.forward.required = [ 'ports' ]; moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema }; @@ -70,7 +71,7 @@ var moduleRefs = { http: [ 'proxy', 'static', 'redirect' ].map(toSchemaRef) , tls: [ 'proxy', 'acme' ].map(toSchemaRef) , tcp: [ 'forward' ].map(toSchemaRef) -, udp: [ 'proxy' ].map(toSchemaRef) +, udp: [ 'forward' ].map(toSchemaRef) }; function addDomainRequirement(itemSchema) { diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 917a5bb..8ad86b3 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -95,16 +95,20 @@ module.exports.create = function (deps, config) { }); } - function dnsListener(msg) { + function dnsListener(port, msg) { if (!Array.isArray(config.udp.modules)) { return; } var socket = require('dgram').createSocket('udp4'); config.udp.modules.forEach(function (mod) { - if (mod.type !== 'proxy') { + if (mod.type !== 'forward') { console.warn('found bad DNS module', mod); return; } + if (mod.ports.indexOf(port) < 0) { + return; + } + var dest = require('./domain-utils').separatePort(mod.address || ''); dest.port = dest.port || mod.port; dest.host = dest.host || mod.host || 'localhost'; @@ -224,13 +228,9 @@ module.exports.create = function (deps, config) { }); if (config.udp.bind) { - if (Array.isArray(config.udp.bind)) { - config.udp.bind.map(function (port) { - listenPromises.push(listeners.udp.add(port, dnsListener)); - }); - } else { - listenPromises.push(listeners.udp.add(config.udp.bind, dnsListener)); - } + config.udp.bind.forEach(function (port) { + listenPromises.push(listeners.udp.add(port, dnsListener.bind(port))); + }); } if (!config.mdns.disabled) { From 5e9e2662e0033fb34976440884f20271f88c02e1 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 12 Oct 2017 18:57:17 -0600 Subject: [PATCH 25/41] updated the config documentation in the README --- README.md | 327 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 189 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index dae6727..fb73509 100644 --- a/README.md +++ b/README.md @@ -53,50 +53,90 @@ curl https://git.daplie.com/Daplie/goldilocks.js/raw/master/install.sh | bash Modules & Configuration ----- -Goldilocks has several core systems, which all have their own configuration and some of which have modules: +Goldilocks has several core systems, which all have their own configuration and +some of which have modules: -``` -* http - - static - - redirect - - proxy (reverse proxy) -* tls - - acme - - proxy (reverse proxy) -* tcp - - forward -* tunnel_server -* tunnel_client -* mdns +* [http](#http) + - [proxy (reverse proxy)](#httpproxy-how-to-reverse-proxy-ruby-python-etc) + - [static](#httpstatic-how-to-serve-a-web-page) + - [redirect](#httpredirect-how-to-redirect-urls) +* [tls](#tls) + - [proxy (reverse proxy)](#tlsproxy) + - [acme](#tlsacme) +* [tcp](#tcp) + - [forward](#tcpforward) +* [udp](#udp) + - [forward](#udpforward) +* [domains](#domains) +* [tunnel_server](#tunnel_server) +* [tunnel_client](#tunnel) +* [mdns](#mdns) * api -``` + +All modules require a `type` and an `id`, and any modules not defined inside the +`domains` system also require a `domains` field (with the exception of the `forward` +modules that require the `ports` field). ### http The HTTP system handles plain http (TLS / SSL is handled by the tls system) +Example config: ```yml http: trust_proxy: true # allow localhost, 192.x, 10.x, 172.x, etc to set headers allow_insecure: false # allow non-https even without proxy https headers primary_domain: example.com # attempts to access via IP address will redirect here - # modules can be nested in domains - domains: - - names: - - example.com - modules: - - name: static - root: /srv/www/:hostname - - # The configuration above could also be represented as follows: + # An array of modules that define how to handle incoming HTTP requests modules: - - name: static + - type: static domains: - example.com root: /srv/www/:hostname ``` +### http.proxy - how to reverse proxy (ruby, python, etc) + +The proxy module is for reverse proxying, typically to an application on the same machine. +(Though it can also reverse proxy to other devices on the local network.) + +It has the following options: +``` +address The DNS-resolvable hostname (or IP address) and port connected by `:` to proxy the request to. + Takes priority over host and port if they are also specified. + ex: locahost:3000 + ex: 192.168.1.100:80 + +host The DNS-resolvable hostname (or IP address) of the system to which the request will be proxied. + Defaults to localhost if only the port is specified. + ex: localhost + ex: 192.168.1.100 + +port The port on said system to which the request will be proxied + ex: 3000 + ex: 80 +``` + +Example config: +```yml +http: + modules: + - type: proxy + domains: + - api.example.com + host: 192.168.1.100 + port: 80 + - type: proxy + domains: + - www.example.com + address: 192.168.1.16:80 + - type: proxy + domains: + - '*' + port: 3000 +``` + ### http.static - how to serve a web page The static module is for serving static web pages and assets and has the following options: @@ -109,50 +149,20 @@ root The path to serve as a string. ``` Example config: - ```yml http: modules: - - name: static + - type: static domains: - example.com root: /srv/www/:hostname ``` -### http.proxy - how to reverse proxy (ruby, python, etc) - -The proxy module is for reverse proxying, typically to an application on the same machine. - -It has the following options: - -``` -host The DNS-resolvable hostname (or IP address) of the system to which the request will be proxied - ex: localhost - ex: 192.168.1.100 - -port The port on said system to which the request will be proxied - ex: 3000 - ex: 80 -``` - -Example config: - -```yml -http: - modules: - - name: proxy - domains: - - example.com - host: localhost - port: 3000 -``` - ### http.redirect - how to redirect URLs The redirect module is for, you guessed it, redirecting URLs. It has the following options: - ``` status The HTTP status code to issue (301 is usual permanent redirect, 302 is temporary) ex: 301 @@ -169,11 +179,10 @@ to The new URL path which should be used. ``` Example config: - ```yml http: modules: - - name: proxy + - type: proxy domains: - example.com status: 301 @@ -184,41 +193,14 @@ http: ### tls The tls system handles encrypted connections, including fetching certificates, -and uses ServerName Indication (SNI) to determine if the connection should be handled -by the http system, a tls system module, or rejected. - -It has the following options: - -``` -acme.email The default email address for ACME certificate issuance - ex: john.doe@example.com - -acme.server The default ACME server to use - ex: https://acme-v01.api.letsencrypt.org/directory - ex: https://acme-staging.api.letsencrypt.org/directory - -acme.challenge_type The default ACME challenge to request - ex: http-01, dns-01, tls-01 - -acme.approved_domains The domains for which to request certificates - ex: example.com -``` +and uses ServerName Indication (SNI) to determine if the connection should be +handled by the http system, a tls system module, or rejected. Example config: - ```yml tls: - acme: - email: 'joe.shmoe@example.com' - # IMPORTANT: Switch to in production 'https://acme-v01.api.letsencrypt.org/directory' - server: 'https://acme-staging.api.letsencrypt.org/directory' - challenge_type: 'http-01' - approved_domains: - - example.com - - example.net - modules: - - name: proxy + - type: proxy domains: - example.com - example.net @@ -227,17 +209,44 @@ tls: Certificates are saved to `~/acme`, which may be `/var/www/acme` if Goldilocks is run as the www-data user. -### tls.acme +### tls.proxy -The acme module overrides the acme defaults of the tls system and uses the same options except that `approved_domains` -(in favor of the domains in the scope of the module). +The proxy module routes the traffic based on the ServerName Indication (SNI) **without decrypting** it. + +It has the same options as the [HTTP proxy module](#httpproxy-how-to-reverse-proxy-ruby-python-etc). Example config: - ```yml tls: modules: - - name: acme + - type: proxy + domains: + - example.com + address: '127.0.0.1:5443' +``` + +### tls.acme + +The acme module defines the setting used when getting new certificates. + +It has the following options: +``` +email The email address for ACME certificate issuance + ex: john.doe@example.com + +server The ACME server to use + ex: https://acme-v01.api.letsencrypt.org/directory + ex: https://acme-staging.api.letsencrypt.org/directory + +challenge_type The ACME challenge to request + ex: http-01, dns-01, tls-01 +``` + +Example config: +```yml +tls: + modules: + - type: acme domains: - example.com - example.net @@ -246,41 +255,18 @@ tls: challenge_type: 'http-01' ``` -### tls.proxy - -The proxy module routes the traffic based on the ServerName Indication (SNI) **without decrypting** it. - -It has the following options: - -``` -address The hostname (or IP) and port of the system or application that should receive the traffic -``` - -Example config: - -```yml -tls: - modules: - - name: proxy - domains: - - example.com - address: '127.0.0.1:5443' -``` - ### tcp The tcp system handles all tcp network traffic **before decryption** and may use port numbers or traffic sniffing to determine how the connection should be handled. It has the following options: - ``` bind An array of numeric ports on which to bind ex: 80 ``` -Example Config - +Example Config: ```yml tcp: bind: @@ -288,7 +274,7 @@ tcp: - 80 - 443 modules: - - name: forward + - type: forward ports: - 22 address: '127.0.0.1:2222' @@ -298,18 +284,15 @@ tcp: The forward module routes traffic based on port number **without decrypting** it. -It has the following options: +In addition to the same options as the [HTTP proxy module](#httpproxy-how-to-reverse-proxy-ruby-python-etc), +the TCP forward modules also has the following options: ``` ports A numeric array of source ports ex: 22 - -address The destination hostname and port - ex: 127.0.0.1:2222 ``` -Example Config - +Example Config: ```yml tcp: bind: @@ -317,10 +300,79 @@ tcp: - 80 - 443 modules: - - name: forward + - type: forward ports: - 22 - address: '127.0.0.1:2222' + port: 2222 +``` + +### udp + +The udp system handles all udp network traffic. It currently only supports +forwarding the messages without any examination. + +It has the following options: +``` +bind An array of numeric ports on which to bind + ex: 53 +``` + +Example Config: +```yml +udp: + bind: + - 53 + modules: + - type: forward + ports: + - 53 + address: '127.0.0.1:8053' +``` + +### udp.forward + +The forward module routes traffic based on port number **without decrypting** it. + +It has the same options as the [TCP forward module](#tcpforward). + +Example Config: +```yml +udp: + bind: + - 53 + modules: + - type: forward + ports: + - 53 + address: '127.0.0.1:8053' +``` + +### domains + +To reduce repetition defining multiple modules that operate on the same domain +name the `domains` field can define multiple modules of multiple types for a +single list of names. The modules defined this way do not need to have their +own `domains` field. + +Example Config + +```yml +domains: + names: + - example.com + - www.example.com + - api.example.com + modules: + tls: + - type: acme + email: joe.schmoe@example.com + challenge_type: 'http-01' + http: + - type: redirect + from: /deprecated/path + to: /new/path + - type: proxy + port: 3000 ``` @@ -417,15 +469,14 @@ See [API.md](/API.md) TODO ---- -* http - nowww module -* http - Allow match styles of `www.*`, `*`, and `*.example.com` equally -* http - redirect based on domain name (not just path) -* tcp - bind should be able to specify localhost, uniquelocal, private, or ip -* tcp - if destination host is omitted default to localhost, if dst port is missing, default to src -* sys - handle SIGHUP -* sys - `curl https://daplie.me/goldilocks | bash -s example.com` -* oauth3 - `example.com/.well-known/domains@oauth3.org/directives.json` -* oauth3 - commandline questionnaire -* modules - use consistent conventions (i.e. address vs host + port) - * tls - tls.acme vs tls.modules.acme -* tls - forward should be able to match on source port to reach different destination ports +* [ ] http - nowww module +* [ ] http - Allow match styles of `www.*`, `*`, and `*.example.com` equally +* [ ] http - redirect based on domain name (not just path) +* [ ] tcp - bind should be able to specify localhost, uniquelocal, private, or ip +* [ ] tcp - if destination host is omitted default to localhost, if dst port is missing, default to src +* [ ] sys - `curl https://daplie.me/goldilocks | bash -s example.com` +* [ ] oauth3 - `example.com/.well-known/domains@oauth3.org/directives.json` +* [ ] oauth3 - commandline questionnaire +* [x] modules - use consistent conventions (i.e. address vs host + port) + * [x] tls - tls.acme vs tls.modules.acme +* [ ] tls - forward should be able to match on source port to reach different destination ports From e15d4f830e2721a96742df869c3ad8c84d1af47b Mon Sep 17 00:00:00 2001 From: tigerbot Date: Fri, 13 Oct 2017 12:39:31 -0600 Subject: [PATCH 26/41] updated the example config --- README.md | 1 + etc/goldilocks/goldilocks.example.yml | 151 +++++++++++++------------- 2 files changed, 78 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index fb73509..1d89538 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ some of which have modules: * [tunnel_server](#tunnel_server) * [tunnel_client](#tunnel) * [mdns](#mdns) +* [socks5](#socks5) * api All modules require a `type` and an `id`, and any modules not defined inside the diff --git a/etc/goldilocks/goldilocks.example.yml b/etc/goldilocks/goldilocks.example.yml index 38ec6b5..77bb321 100644 --- a/etc/goldilocks/goldilocks.example.yml +++ b/etc/goldilocks/goldilocks.example.yml @@ -4,11 +4,87 @@ tcp: - 80 - 443 modules: - - name: forward + - type: forward ports: - 22 address: '127.0.0.1:8022' +udp: + bind: + - 53 + modules: + - type: forward + ports: + - 53 + port: 5353 + # default host is localhost + + +tls: + modules: + - type: proxy + domains: + - localhost.bar.daplie.me + - localhost.foo.daplie.me + address: '127.0.0.1:5443' + - type: acme + domains: + - '*.localhost.daplie.me' + email: 'guest@example.com' + challenge_type: 'http-01' + +http: + trust_proxy: true + allow_insecure: false + primary_domain: localhost.daplie.me + + modules: + - type: redirect + domains: + - localhost.beta.daplie.me + status: 301 + from: /old/path/*/other/* + to: /path/new/:2/something/:1 + - type: proxy + domains: + - localhost.daplie.me + host: localhost + port: 4000 + - type: static + domains: + - '*.localhost.daplie.me' + root: '/srv/www/:hostname' + +domains: + - names: + - localhost.gamma.daplie.me + modules: + tls: + - type: proxy + port: 6443 + - names: + - beta.localhost.daplie.me + - baz.localhost.daplie.me + modules: + tls: + - type: acme + email: 'owner@example.com' + challenge_type: 'tls-sni-01' + # default server is 'https://acme-v01.api.letsencrypt.org/directory' + http: + - type: redirect + from: /nowhere/in/particular + to: /just/an/example + - type: proxy + address: '127.0.0.1:3001' + + +mdns: + disabled: false + port: 5353 + broadcast: '224.0.0.251' + ttl: 300 + # tunnel: jwt # tunnel: # - jwt1 @@ -18,76 +94,3 @@ tunnel_server: secret: abc123 servernames: - 'tunnel.localhost.com' - -tls: - acme: - email: 'joe.shmoe@example.com' - server: 'https://acme-staging.api.letsencrypt.org/directory' - challenge_type: 'http-01' - approved_domains: - - localhost.baz.daplie.me - - localhost.beta.daplie.me - domains: - - names: - - localhost.gamma.daplie.me - modules: - - name: proxy - address: '127.0.0.1:6443' - - names: - - beta.localhost.daplie.me - - baz.localhost.daplie.me - modules: - - name: acme - email: 'owner@example.com' - challenge_type: 'tls-sni-01' - # default server is 'https://acme-v01.api.letsencrypt.org/directory' - modules: - - name: proxy - domains: - - localhost.bar.daplie.me - - localhost.foo.daplie.me - address: '127.0.0.1:5443' - - name: acme - email: 'guest@example.com' - challenge_type: 'http-01' - domains: - - foo.localhost.daplie.me - - gamma.localhost.daplie.me - - -http: - trust_proxy: true - allow_insecure: false - primary_domain: localhost.foo.daplie.me - domains: - - names: - - localhost.baz.daplie.me - modules: - - name: redirect - from: /nowhere/in/particular - to: /just/an/example - - name: proxy - port: 3001 - - modules: - - name: redirect - domains: - - localhost.beta.daplie.me - status: 301 - from: /old/path/*/other/* - to: /path/new/:2/something/:1 - - name: proxy - domains: - - localhost.daplie.me - host: localhost - port: 4000 - - name: static - domains: - - '*.localhost.daplie.me' - root: '/srv/www/:hostname' - -mdns: - disabled: false - port: 5353 - broadcast: '224.0.0.251' - ttl: 300 From 72520679d89fc01bf8c2bf4ff810a6d3f635383a Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 16 Oct 2017 12:59:45 -0600 Subject: [PATCH 27/41] updated the documentation for the config API --- API.md | 112 +++++++++++++++++++++++++++++++++++++++++--- lib/admin/config.js | 2 + 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index bfb6c54..84da6fc 100644 --- a/API.md +++ b/API.md @@ -2,13 +2,113 @@ The API system is intended for use with Desktop and Mobile clients. It must be accessed using one of the following domains as the Host header: -``` -admin.invalid -localhost.admin.daplie.me -``` +* localhost.alpha.daplie.me +* localhost.admin.daplie.me +* alpha.localhost.daplie.me +* admin.localhost.daplie.me +* localhost.daplie.invalid All requests require an OAuth3 token in the request headers. +## Config + +### Get All Settings + * **URL** `/api/goldilocks@daplie.com/config` + * **Method** `GET` + * **Reponse**: The JSON representation of the current config. See the [README.md](/README.md) + for the structure of the config. + +### Get Group Setting + * **URL** `/api/goldilocks@daplie.com/config/:group` + * **Method** `GET` + * **Reponse**: The sub-object of the config relevant to the group specified in + the url (ie http, tls, tcp, etc.) + +### Get Group Module List + * **URL** `/api/goldilocks@daplie.com/config/:group/modules` + * **Method** `GET` + * **Reponse**: The list of modules relevant to the group specified in the url + (ie http, tls, tcp, etc.) + +### Get Specific Module + * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId` + * **Method** `GET` + * **Reponse**: The module with the specified module ID. + +### Get Domain Group + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId` + * **Method** `GET` + * **Reponse**: The domains specification with the specified domains ID. + +### Get Domain Group Modules + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules` + * **Method** `GET` + * **Reponse**: An object containing all of the relevant modules for the group + of domains. + +### Get Domain Group Module Category + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group` + * **Method** `GET` + * **Reponse**: A list of the specific category of modules for the group of domains. + +### Get Specific Domain Group Module + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId` + * **Method** `GET` + * **Reponse**: The module with the specified module ID. + + +### Change Settings + * **URL** `/api/goldilocks@daplie.com/config` + * **URL** `/api/goldilocks@daplie.com/config/:group` + * **Method** `POST` + * **Body**: The changes to be applied on top of the current config. See the + [README.md](/README.md) for the settings. If modules or domains are specified + they are added to the current list. + * **Reponse**: The current config. If the group is specified in the URL it will + only be the config relevant to that group. + +### Add Module + * **URL** `/api/goldilocks@daplie.com/config/:group/modules` + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group` + * **Method** `POST` + * **Body**: The module to be added. Can also be provided an array of modules + to add multiple modules in the same request. + * **Reponse**: The current list of modules. + +### Add Domain Group + * **URL** `/api/goldilocks@daplie.com/config/domains` + * **Method** `POST` + * **Body**: The domains names and modules for the new domain group(s). + * **Reponse**: The current list of domain groups. + + +### Edit Module + * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId` + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId` + * **Method** `PUT` + * **Body**: The new parameters for the module. + * **Reponse**: The editted module. + +### Edit Domain Group + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId` + * **Method** `PUT` + * **Body**: The new domains names for the domains group. The module list cannot + be editted through this route. + * **Reponse**: The editted domain group. + + +### Remove Module + * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId` + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId` + * **Method** `DELETE` + * **Reponse**: The list of modules. + +### Remove Domain Group + * **URL** `/api/goldilocks@daplie.com/config/domains/:domId` + * **Method** `DELETE` + * **Reponse**: The list of domain groups. + + ## Tunnel ### Check Status @@ -39,9 +139,9 @@ All requests require an OAuth3 token in the request headers. ### Start Proxy * **URL** `/api/goldilocks@daplie.com/socks5` * **Method** `POST` - * **Response**: Same response as for the `GET` resquest + * **Response**: Same response as for the `GET` request ### Stop Proxy * **URL** `/api/goldilocks@daplie.com/socks5` * **Method** `DELETE` - * **Response**: Same response as for the `GET` resquest + * **Response**: Same response as for the `GET` request diff --git a/lib/admin/config.js b/lib/admin/config.js index 1ac3dd1..682052c 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -281,6 +281,8 @@ class DomainList extends IdList { http: new ModuleList() , tls: new ModuleList() }; + // We add these after instead of in the constructor to run the validation and manipulation + // in the ModList add function since these are all new modules. if (dom.modules && Array.isArray(dom.modules.http)) { dom.modules.http.forEach(modLists.http.add, modLists.http); } From 754ace5cb437122e97981718b4ad01a04f399401 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 17 Oct 2017 12:56:25 -0600 Subject: [PATCH 28/41] removed arguments that populate a deprecated config --- bin/goldilocks.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 06a340b..2278d8f 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -339,15 +339,6 @@ function fillConfig(config, args) { fillComponent('http', false); fillComponent('tls', false); - if (!config.tls.acme && (args.email || args.agreeTos)) { - config.tls.acme = {}; - } - if (typeof args.agreeTos === 'string') { - config.tls.acme.approvedDomains = args.agreeTos.split(','); - } - if (args.email) { - config.tls.acme.email = args.email; - } config.device = { hostname: require('os').hostname() }; config.tunnel = args.tunnel || config.tunnel; @@ -452,7 +443,6 @@ function readEnv(args) { var env = { tunnel: process.env.GOLDILOCKS_TUNNEL_TOKEN || process.env.GOLDILOCKS_TUNNEL && true - , email: process.env.GOLDILOCKS_EMAIL , cwd: process.env.GOLDILOCKS_HOME || process.cwd() , debug: process.env.GOLDILOCKS_DEBUG && true }; @@ -464,10 +454,8 @@ var program = require('commander'); program .version(require('../package.json').version) - .option('--agree-tos [url1,url2]', "agree to all Terms of Service for Daplie, Let's Encrypt, etc (or specific URLs only)") .option('-c --config ', 'Path to config file (Goldilocks.json or Goldilocks.yml) example: --config /etc/goldilocks/Goldilocks.json') .option('--tunnel [token]', 'Turn tunnel on. This will enter interactive mode for login if no token is specified.') - .option('--email ', "(Re)set default email to use for Daplie, Let's Encrypt, ACME, etc.") .option('--debug', "Enable debug output") .parse(process.argv); From 9c7aaa4f984d6f0771549fb1c56d2f9fc9471742 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 17 Oct 2017 16:16:57 -0600 Subject: [PATCH 29/41] reduced some duplication in handling error responses --- lib/admin/apis.js | 87 ++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 47 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index cfef538..19c05b9 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -46,6 +46,16 @@ module.exports.create = function (deps, conf) { }; } + function handlePromise(req, res, prom) { + prom.then(function (result) { + res.send(deps.recase.snakeCopy(result)); + }).catch(function (err) { + res.statusCode = err.statusCode || 500; + err.message = err.message || err.toString(); + res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }); + } + function isAuthorized(req, res, fn) { var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, '')); if (!auth) { @@ -369,7 +379,7 @@ module.exports.create = function (deps, conf) { return; } - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var update; if (req.params.group) { update = {}; @@ -381,16 +391,13 @@ module.exports.create = function (deps, conf) { var changer = new (require('./config').ConfigChanger)(conf); changer.update(update); return config.save(changer); - }).then(function (config) { + }).then(function (newConf) { if (req.params.group) { - config = config[req.params.group]; + return newConf[req.params.group]; } - res.send(deps.recase.snakeCopy(config)); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + return newConf; }); + handlePromise(req, res, promise); }; config.extractModList = function (changer, params) { @@ -424,7 +431,7 @@ module.exports.create = function (deps, conf) { return; } - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); var modList = config.extractModList(changer, req.params); @@ -436,12 +443,9 @@ module.exports.create = function (deps, conf) { return config.save(changer); }).then(function (newConf) { - res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + return config.extractModList(newConf, req.params); }); + handlePromise(req, res, promise); }; config.restful.updateModule = function (req, res, next) { if (req.params.group === 'domains') { @@ -449,18 +453,17 @@ module.exports.create = function (deps, conf) { return; } - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); var modList = config.extractModList(changer, req.params); modList.update(req.params.modId, req.body); return config.save(changer); }).then(function (newConf) { - res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + return config.extractModule(newConf, req.params).find(function (mod) { + return mod.id === req.params.modId; + }); }); + handlePromise(req, res, promise); }; config.restful.removeModule = function (req, res, next) { if (req.params.group === 'domains') { @@ -468,22 +471,19 @@ module.exports.create = function (deps, conf) { return; } - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); var modList = config.extractModList(changer, req.params); modList.remove(req.params.modId); return config.save(changer); }).then(function (newConf) { - res.send(deps.recase.snakeCopy(config.extractModList(newConf, req.params))); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + return config.extractModList(newConf, req.params); }); + handlePromise(req, res, promise); }; config.restful.createDomain = function (req, res) { - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); var update = req.body; @@ -492,16 +492,13 @@ module.exports.create = function (deps, conf) { } update.forEach(changer.domains.add, changer.domains); return config.save(changer); - }).then(function (config) { - res.send(deps.recase.snakeCopy(config.domains)); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }).then(function (newConf) { + return newConf.domains; }); + handlePromise(req, res, promise); }; config.restful.updateDomain = function (req, res) { - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { if (req.body.modules) { throw Object.assign(new Error('do not add modules with this route'), {statusCode: 400}); } @@ -509,26 +506,22 @@ module.exports.create = function (deps, conf) { var changer = new (require('./config').ConfigChanger)(conf); changer.domains.update(req.params.domId, req.body); return config.save(changer); - }).then(function (config) { - res.send(deps.recase.snakeCopy(config.domains)); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }).then(function (newConf) { + return newConf.domains.find(function (dom) { + return dom.id === req.params.domId; + }); }); + handlePromise(req, res, promise); }; config.restful.removeDomain = function (req, res) { - deps.PromiseA.resolve().then(function () { + var promise = deps.PromiseA.resolve().then(function () { var changer = new (require('./config').ConfigChanger)(conf); changer.domains.remove(req.params.domId); return config.save(changer); - }).then(function (config) { - res.send(deps.recase.snakeCopy(config.domains)); - }, function (err) { - res.statusCode = err.statusCode || 500; - err.message = err.message || err.toString(); - res.end(JSON.stringify({error: {message: err.message, code: err.code}})); + }).then(function (newConf) { + return newConf.domains; }); + handlePromise(req, res, promise); }; var app = require('express')(); From cfaa8d495984f0d6efde45cb737858b579620aed Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 17 Oct 2017 18:36:36 -0600 Subject: [PATCH 30/41] added interface to save user tokens --- lib/admin/apis.js | 27 ++++++++++++++++ lib/storage.js | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 19c05b9..5bbe49a 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -50,6 +50,9 @@ module.exports.create = function (deps, conf) { prom.then(function (result) { res.send(deps.recase.snakeCopy(result)); }).catch(function (err) { + if (conf.debug) { + console.log(err); + } res.statusCode = err.statusCode || 500; err.message = err.message || err.toString(); res.end(JSON.stringify({error: {message: err.message, code: err.code}})); @@ -524,6 +527,24 @@ module.exports.create = function (deps, conf) { handlePromise(req, res, promise); }; + var tokens = { restful: {} }; + tokens.restful.getAll = function (req, res) { + handlePromise(req, res, deps.storage.tokens.all()); + }; + tokens.restful.getOne = function (req, res) { + handlePromise(req, res, deps.storage.tokens.get(req.params.id)); + }; + tokens.restful.save = function (req, res) { + handlePromise(req, res, deps.storage.tokens.save(req.body)); + }; + tokens.restful.revoke = function (req, res) { + var promise = deps.storage.tokens.remove(req.params.id).then(function (success) { + return {success: success}; + }); + handlePromise(req, res, promise); + }; + + var app = require('express')(); // Handle all of the API endpoints using the old definition style, and then we can @@ -555,5 +576,11 @@ module.exports.create = function (deps, conf) { app.put( '/config/domains/:domId', config.restful.updateDomain); app.delete('/config/domains/:domId', config.restful.removeDomain); + app.use( '/tokens', makeCorsHandler(['GET', 'POST', 'DELETE'])); + app.get( '/tokens', tokens.restful.getAll); + app.get( '/tokens/:id', tokens.restful.getOne); + app.post( '/tokens', tokens.restful.save); + app.delete('/tokens/:id', tokens.restful.revoke); + return app; }; diff --git a/lib/storage.js b/lib/storage.js index 33fc8f6..81d64d1 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -3,6 +3,8 @@ var PromiseA = require('bluebird'); var path = require('path'); var fs = PromiseA.promisifyAll(require('fs')); +var jwt = require('jsonwebtoken'); +var crypto = require('crypto'); module.exports.create = function (deps, conf) { var hrIds = require('human-readable-ids').humanReadableIds; @@ -93,6 +95,82 @@ module.exports.create = function (deps, conf) { } } + var userTokens = { + _filename: 'user-tokens.json' + , _convertToken(id, token) { + // convert the token into something that looks more like what OAuth3 uses internally + // as sessions so we can use it with OAuth3. We don't use OAuth3's internal session + // storage because it effectively only supports storing tokens based on provider URI. + // We also use the token as the `access_token` instead of `refresh_token` because the + // refresh functionality is closely tied to the storage. + var decoded = jwt.decode(token); + return { + id: id + , access_token: token + , token: decoded + , provider_uri: decoded.iss || decoded.issuer || decoded.provider_uri + , client_uri: decoded.azp + , scope: decoded.scp || decoded.scope || decoded.grants + }; + } + , all: function allUserTokens() { + var self = this; + return read(self._filename).then(function (tokens) { + return Object.keys(tokens).map(function (id) { + return self._convertToken(id, tokens[id]); + }); + }); + } + , get: function getUserToken(id) { + var self = this; + return read(self._filename).then(function (tokens) { + return self._convertToken(id, tokens[id]); + }); + } + , save: function saveUserToken(newToken) { + var self = this; + return read(self._filename).then(function (tokens) { + var rawToken; + if (typeof newToken === 'string') { + rawToken = newToken; + } else { + rawToken = newToken.refresh_token || newToken.access_token; + } + if (typeof rawToken !== 'string') { + throw new Error('cannot save invalid session: missing refresh_token and access_token'); + } + + var decoded = jwt.decode(rawToken); + var idHash = crypto.createHash('sha256'); + idHash.update(decoded.sub || decoded.ppid || decoded.appScopedId || ''); + idHash.update(decoded.iss || decoded.issuer || ''); + idHash.update(decoded.aud || decoded.audience || ''); + + var scope = decoded.scope || decoded.scp || decoded.grants || ''; + idHash.update(scope.split(/[,\s]+/mg).sort().join(',')); + + var id = idHash.digest('hex'); + tokens[id] = rawToken; + return write(self._filename, tokens).then(function () { + return self.get(id); + }); + }); + } + , remove: function removeUserToken(id) { + var self = this; + return read(self._filename).then(function (tokens) { + var present = delete tokens[id]; + if (!present) { + return present; + } + + return write(self._filename, tokens).then(function () { + return true; + }); + }); + } + }; + var mdnsId = { _filename: 'mdns-id' , get: function () { @@ -119,6 +197,7 @@ module.exports.create = function (deps, conf) { owners: owners , config: config , updateConf: updateConf + , tokens: userTokens , mdnsId: mdnsId }; }; From 6b2b91ba26dd39139da8d0db7380b9d3e9dbd717 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 18 Oct 2017 12:06:01 -0600 Subject: [PATCH 31/41] updated the documentation and validation for DDNS settings --- README.md | 47 ++++++++++++++++++++---- bin/goldilocks.js | 2 +- etc/goldilocks/goldilocks.example.yml | 18 +++++++--- lib/admin/config.js | 52 ++++++++++++++++++++++----- 4 files changed, 98 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 54b7931..e78e7da 100644 --- a/README.md +++ b/README.md @@ -403,17 +403,50 @@ tunnel_server: - 'api.tunnel.example.com' ``` -### tunnel +### DDNS -The tunnel client is meant to be run from behind a firewalls, carrier-grade NAT, -or otherwise inaccessible devices to allow them to be accessed publicly on the -internet. +The DDNS module watches the network environment of the unit and makes sure the +device is always accessible on the internet using the domains listed in the +config. If the device has a public address or if it can automatically set up +port forwarding the device will periodically check its public address to ensure +the DNS records always point to it. Otherwise it will to connect to a tunnel +server and set the DNS records to point to that server. -### ddns +The `loopback` setting specifies how the unit will check its public IP address +and whether connections can reach it. Currently only `tunnel@oauth3.org` is +supported. If the loopback setting is not defined it will default to using +`oauth3.org`. -TODO +The `tunnel` setting can be used to specify how to connect to the tunnel. +Currently only `tunnel@oauth3.org` is supported. The token specified in the +`tunnel` setting will be used to acquire the tokens that are used directly with +the tunnel server. If the tunnel setting is not defined it will default to try +using the tokens in the modules for the relevant domains. -### mdns +If a particular DDNS module has been disabled the device will still try to set +up port forwarding (and connect to a tunnel if that doesn't work), but the DNS +records will not be updated to point to the device. This is to allow a setup to +be tested before transitioning services between devices. + +```yaml +ddns: + disabled: false + loopback: + type: 'tunnel@oauth3.org' + domain: oauth3.org + tunnel: + type: 'tunnel@oauth3.org' + token: user_token_id + modules: + - type: 'dns@oauth3.org' + token: user_token_id + domains: + - www.example.com + - api.example.com + - test.example.com +``` + +### mDNS enabled by default diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 516f6d7..c65ede3 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -311,7 +311,6 @@ function fillConfig(config, args) { config.debug = config.debug || args.debug; config.socks5 = config.socks5 || { enabled: false }; - config.ddns = config.ddns || { enabled: false }; // Use Object.assign to copy any real config values over the default values so we can // easily make sure all the fields we need exist . @@ -338,6 +337,7 @@ function fillConfig(config, args) { fillComponent('tcp', true); fillComponent('http', false); fillComponent('tls', false); + fillComponent('ddns', false); config.device = { hostname: require('os').hostname() }; diff --git a/etc/goldilocks/goldilocks.example.yml b/etc/goldilocks/goldilocks.example.yml index 2545f4d..2d1747d 100644 --- a/etc/goldilocks/goldilocks.example.yml +++ b/etc/goldilocks/goldilocks.example.yml @@ -91,8 +91,16 @@ tunnel_server: - 'tunnel.localhost.com' ddns: - enabled: true - domains: - - www.example.com - - api.example.com - - test.example.com + loopback: + type: 'tunnel@oauth3.org' + domain: oauth3.org + tunnel: + type: 'tunnel@oauth3.org' + token: user_token_id + modules: + - type: 'dns@oauth3.org' + token: user_token_id + domains: + - www.example.com + - api.example.com + - test.example.com diff --git a/lib/admin/config.js b/lib/admin/config.js index 682052c..3001f6f 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -48,6 +48,16 @@ var moduleSchemas = { , challenge_type: { type: 'string' } } } + + // the dns control modules for DDNS +, dns_oauth3_org: { + name: 'dns@oauth3.org' + , type: 'object' + , required: [ 'token' ] + , properties: { + token: { type: 'string' } + } + } }; // forward is basically the same as proxy, but specifies the relevant incoming port(s). // only allows for the raw transport layers (TCP/UDP) @@ -57,6 +67,10 @@ moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema }; Object.keys(moduleSchemas).forEach(function (name) { var schema = moduleSchemas[name]; + if (schema.name) { + name = schema.name; + delete schema.name; + } schema.id = '/modules/'+name; schema.required = ['id', 'type'].concat(schema.required || []); schema.properties.id = { type: 'string' }; @@ -72,12 +86,13 @@ var moduleRefs = { , tls: [ 'proxy', 'acme' ].map(toSchemaRef) , tcp: [ 'forward' ].map(toSchemaRef) , udp: [ 'forward' ].map(toSchemaRef) +, ddns: [ 'dns@oauth3.org' ].map(toSchemaRef) }; function addDomainRequirement(itemSchema) { itemSchema.required = (itemSchema.required || []).concat('domains'); itemSchema.properties = itemSchema.properties || {}; - itemSchema.domains = { type: 'array', items: { type: 'string' }, minLength: 1}; + itemSchema.properties.domains = { type: 'array', items: { type: 'string' }, minLength: 1}; return itemSchema; } @@ -93,6 +108,7 @@ var domainSchema = { , properties: { tls: { type: 'array', items: { oneOf: moduleRefs.tls }} , http: { type: 'array', items: { oneOf: moduleRefs.http }} + , ddns: { type: 'array', items: { oneOf: moduleRefs.ddns }} } , additionalProperties: false } @@ -158,7 +174,23 @@ var mdnsSchema = { var ddnsSchema = { type: 'object' , properties: { - enabled: { type: 'boolean' } + loopback: { + type: 'object' + , required: [ 'type', 'domain' ] + , properties: { + type: { type: 'string', const: 'tunnel@oauth3.org' } + , domain: { type: 'string'} + } + } + , tunnel: { + type: 'object' + , required: [ 'type', 'token' ] + , properties: { + type: { type: 'string', const: 'tunnel@oauth3.org' } + , token: { type: 'string'} + } + } + , modules: { type: 'array', items: { oneOf: moduleRefs.ddns }} } }; var socks5Schema = { @@ -265,6 +297,7 @@ class DomainList extends IdList { dom.modules = { http: new ModuleList((dom.modules || {}).http) , tls: new ModuleList((dom.modules || {}).tls) + , ddns: new ModuleList((dom.modules || {}).ddns) }; }); } @@ -280,14 +313,16 @@ class DomainList extends IdList { var modLists = { http: new ModuleList() , tls: new ModuleList() + , ddns: new ModuleList() }; // We add these after instead of in the constructor to run the validation and manipulation // in the ModList add function since these are all new modules. - if (dom.modules && Array.isArray(dom.modules.http)) { - dom.modules.http.forEach(modLists.http.add, modLists.http); - } - if (dom.modules && Array.isArray(dom.modules.tls)) { - dom.modules.tls.forEach(modLists.tls.add, modLists.tls); + if (dom.modules) { + Object.keys(modLists).forEach(function (key) { + if (Array.isArray(dom.modules[key])) { + dom.modules[key].forEach(modLists[key].add, modLists[key]); + } + }); } dom.id = require('crypto').randomBytes(4).toString('hex'); @@ -306,6 +341,7 @@ class ConfigChanger { this.tls.modules = new ModuleList(this.tls.modules); this.tcp.modules = new ModuleList(this.tcp.modules); this.udp.modules = new ModuleList(this.udp.modules); + this.ddns.modules = new ModuleList(this.ddns.modules); } update(update) { @@ -314,7 +350,7 @@ class ConfigChanger { if (update.domains) { update.domains.forEach(self.domains.add, self.domains); } - [ 'http', 'tls', 'tcp', 'udp' ].forEach(function (name) { + [ 'http', 'tls', 'tcp', 'udp', 'ddns' ].forEach(function (name) { if (update[name] && update[name].modules) { update[name].modules.forEach(self[name].modules.add, self[name].modules); delete update[name].modules; From c55c034f11f1cb342f3f45e3b2dbcf0377024b3b Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 18 Oct 2017 13:48:08 -0600 Subject: [PATCH 32/41] started using of the ddns.loopback setting --- lib/ddns/index.js | 19 ++++++++++++++++--- lib/ddns/loopback.js | 8 ++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 27cbfe5..06c39cf 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -5,6 +5,19 @@ module.exports.create = function (deps, conf) { var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); + var loopbackDomain; + function updateConf() { + loopbackDomain = 'oauth3.org'; + if (conf.ddns && conf.ddns.loopback) { + if (conf.ddns.loopback.type === 'tunnel@oauth3.org' && conf.ddns.loopback.domain) { + loopbackDomain = conf.ddns.loopback.domain; + } else { + console.warn('invalid loopback configuration: bad type or missing domain'); + } + } + } + updateConf(); + var localAddr, gateway; var tunnelActive = false; async function checkNetworkEnv() { @@ -19,7 +32,7 @@ module.exports.create = function (deps, conf) { localAddr = addr; gateway = gw; - var loopResult = await loopback('seth.daplie.me'); + var loopResult = await loopback(loopbackDomain); var notLooped = Object.keys(loopResult.ports).filter(function (port) { return !loopResult.ports[port]; }); @@ -74,8 +87,7 @@ module.exports.create = function (deps, conf) { return; } var session = await getSession(); - var directives = await deps.OAUTH3.discover(session.token.aud); - var addr = await loopback.checkPublicAddr(directives.api); + var addr = await loopback.checkPublicAddr(loopbackDomain); if (publicAddress === addr) { return; @@ -97,5 +109,6 @@ module.exports.create = function (deps, conf) { , setDeviceAddress: dnsCtrl.setDeviceAddress , getDeviceAddresses: dnsCtrl.getDeviceAddresses , recheckPubAddr: recheckPubAddr + , updateConf: updateConf }; }; diff --git a/lib/ddns/loopback.js b/lib/ddns/loopback.js index c35c742..3975f97 100644 --- a/lib/ddns/loopback.js +++ b/lib/ddns/loopback.js @@ -3,7 +3,7 @@ module.exports.create = function (deps, conf) { var pending = {}; - async function checkPublicAddr(host) { + async function _checkPublicAddr(host) { var result = await deps.request({ method: 'GET' , url: deps.OAUTH3.url.normalize(host)+'/api/org.oauth3.tunnel/checkip' @@ -19,6 +19,10 @@ module.exports.create = function (deps, conf) { } return result.body.address; } + async function checkPublicAddr(provider) { + var directives = await deps.OAUTH3.discover(provider); + return _checkPublicAddr(directives.api); + } async function checkSinglePort(host, address, port) { var crypto = require('crypto'); @@ -61,7 +65,7 @@ module.exports.create = function (deps, conf) { async function loopback(provider) { var directives = await deps.OAUTH3.discover(provider); - var address = await checkPublicAddr(directives.api); + var address = await _checkPublicAddr(directives.api); if (conf.debug) { console.log('checking to see if', address, 'gets back to us'); } From b9fac21b056e3832c09188fb189db5f1537f44ee Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 18 Oct 2017 15:37:35 -0600 Subject: [PATCH 33/41] switched to using new config format when connecting to tunnel --- README.md | 4 ++-- lib/admin/config.js | 8 +++---- lib/ddns/index.js | 55 ++++++++++++++++++++++++++++++++++++++++----- lib/storage.js | 3 +++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e78e7da..4a18348 100644 --- a/README.md +++ b/README.md @@ -436,10 +436,10 @@ ddns: domain: oauth3.org tunnel: type: 'tunnel@oauth3.org' - token: user_token_id + token_id: user_token_id modules: - type: 'dns@oauth3.org' - token: user_token_id + token_id: user_token_id domains: - www.example.com - api.example.com diff --git a/lib/admin/config.js b/lib/admin/config.js index 3001f6f..607e2a6 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -53,9 +53,9 @@ var moduleSchemas = { , dns_oauth3_org: { name: 'dns@oauth3.org' , type: 'object' - , required: [ 'token' ] + , required: [ 'token_id' ] , properties: { - token: { type: 'string' } + token_id: { type: 'string' } } } }; @@ -184,10 +184,10 @@ var ddnsSchema = { } , tunnel: { type: 'object' - , required: [ 'type', 'token' ] + , required: [ 'type', 'token_id' ] , properties: { type: { type: 'string', const: 'tunnel@oauth3.org' } - , token: { type: 'string'} + , token_id: { type: 'string'} } } , modules: { type: 'array', items: { oneOf: moduleRefs.ddns }} diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 06c39cf..e69d4ed 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -18,8 +18,54 @@ module.exports.create = function (deps, conf) { } updateConf(); - var localAddr, gateway; var tunnelActive = false; + async function connectTunnel() { + var sessionCache = {}; + var sessionOverride; + if (conf.ddns.tunnel) { + sessionOverride = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); + } + async function getSession(id) { + if (sessionOverride) { + return sessionOverride; + } + if (!sessionCache.hasOwnProperty(id)) { + sessionCache[id] = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); + } + if (!sessionCache[id]) { + throw new Error('no user token with ID "'+id+'"'); + } + return sessionCache[id]; + } + + conf.domains.forEach(function(dom) { + if (dom.modules && Array.isArray(dom.modules.ddns) && dom.modules.ddns.length) { + var mod = dom.modules.ddns[0]; + getSession(mod.token_id).then(function (session) { + return deps.tunnelClients.start(session, dom.names); + }).catch(function (err) { + console.log('error starting tunnel for', dom.names.join(', ')); + console.log(err); + }); + } + }); + + conf.ddns.modules.forEach(function (mod) { + getSession(mod.token_id).then(function (session) { + return deps.tunnelClients.start(session, mod.domains); + }).catch(function (err) { + console.log('error starting tunnel for', mod.domains.join(', ')); + console.log(err); + }); + }); + tunnelActive = true; + } + function disconnectTunnel() { + deps.tunnelClients.disconnect(); + tunnelActive = false; + } + + var localAddr, gateway; async function checkNetworkEnv() { // Since we can't detect the OS level events when a user plugs in an ethernet cable to recheck // what network environment we are in we check our local network address and the gateway to @@ -46,14 +92,11 @@ module.exports.create = function (deps, conf) { // address. Otherwise we need to use the tunnel to accept traffic. if (!notLooped.length) { if (tunnelActive) { - deps.tunnelClients.disconnect(); - tunnelActive = false; + disconnectTunnel(); } } else { if (!tunnelActive) { - var session = await getSession(); - await deps.tunnelClients.start(session, conf.dns.domains); - tunnelActive = true; + connectTunnel(); } } } diff --git a/lib/storage.js b/lib/storage.js index 81d64d1..271fed7 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -104,6 +104,9 @@ module.exports.create = function (deps, conf) { // We also use the token as the `access_token` instead of `refresh_token` because the // refresh functionality is closely tied to the storage. var decoded = jwt.decode(token); + if (!decoded) { + return null; + } return { id: id , access_token: token From 3aed276faf49953f5a2bfc79f31adb2c6974b46f Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 18 Oct 2017 16:06:44 -0600 Subject: [PATCH 34/41] switched to newer config structure for setting DNS records --- lib/ddns/index.js | 70 +++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index e69d4ed..5dbcd2b 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -12,7 +12,7 @@ module.exports.create = function (deps, conf) { if (conf.ddns.loopback.type === 'tunnel@oauth3.org' && conf.ddns.loopback.domain) { loopbackDomain = conf.ddns.loopback.domain; } else { - console.warn('invalid loopback configuration: bad type or missing domain'); + console.error('invalid loopback configuration: bad type or missing domain'); } } } @@ -101,37 +101,14 @@ module.exports.create = function (deps, conf) { } } - async function getSession() { - var sessions = await deps.storage.owners.all(); - var session = sessions.filter(function (sess) { - return sess.token.scp.indexOf('dns') >= 0; - })[0]; - - if (!session) { - throw new Error('no sessions with DNS grants'); - } - - // The OAUTH3 library stores some things on the root session object that we usually - // just leave inside the token, but we need to pull those out before we use it here - session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss; - session.client_uri = session.client_uri || session.token.azp; - session.scope = session.scope || session.token.scp; - return session; - } - var publicAddress; async function recheckPubAddr() { - if (!conf.ddns.enabled) { - return; - } - await checkNetworkEnv(); if (tunnelActive) { return; } - var session = await getSession(); - var addr = await loopback.checkPublicAddr(loopbackDomain); + var addr = await loopback.checkPublicAddr(loopbackDomain); if (publicAddress === addr) { return; } @@ -139,9 +116,48 @@ module.exports.create = function (deps, conf) { if (conf.debug) { console.log('previous public address',publicAddress, 'does not match current public address', addr); } - - await dnsCtrl.setDeviceAddress(session, addr, conf.ddns.domains); publicAddress = addr; + + var sessionCache = {}; + async function getSession(id) { + if (!sessionCache.hasOwnProperty(id)) { + sessionCache[id] = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); + } + if (!sessionCache[id]) { + throw new Error('no user token with ID "'+id+'"'); + } + return sessionCache[id]; + } + + conf.domains.forEach(function(dom) { + if (dom.modules && Array.isArray(dom.modules.ddns)) { + dom.modules.ddns.some(function (mod) { + if (mod.type !== 'dns@oauth3.org' || mod.disabled) { + return false; + } + + return getSession(mod.token_id).then(function (session) { + return dnsCtrl.setDeviceAddress(session, addr, dom.names); + }).catch(function (err) { + console.log('error setting DNS records for', dom.names.join(', ')); + console.log(err); + }); + }); + } + }); + + conf.ddns.modules.forEach(function (mod) { + if (mod.type !== 'dns@oauth3.org' || mod.disabled) { + return; + } + + getSession(mod.token_id).then(function (session) { + return dnsCtrl.setDeviceAddress(session, addr, mod.domains); + }).catch(function (err) { + console.log('error setting DNS records for', mod.domains.join(', ')); + console.log(err); + }); + }); } recheckPubAddr(); From 019e4fa0636f39241a8fd77dd6f5a789b388a299 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 19 Oct 2017 12:37:08 -0600 Subject: [PATCH 35/41] made connectTunnel wait for connections to actually start --- lib/ddns/index.js | 80 ++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 5dbcd2b..0de208f 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -18,6 +18,28 @@ module.exports.create = function (deps, conf) { } updateConf(); + function iterateAllModules(action) { + var promises = conf.ddns.modules.map(function (mod) { + return action(mod, mod.domains); + }); + + conf.domains.forEach(function (dom) { + if (!dom.modules || !Array.isArray(dom.modules.ddns) || !dom.modules.ddns.length) { + return null; + } + + // For the time being all of our things should only be tried once (regardless if it succeeded) + // TODO: revisit this behavior when we support multiple ways of setting records, and/or + // if we want to allow later modules to run if early modules fail. + promises.push(dom.modules.ddns.reduce(function (prom, mod) { + if (prom) { return prom; } + return action(mod, dom.names); + }, null)); + }); + + return deps.PromiseA.all(promises.filter(Boolean)); + } + var tunnelActive = false; async function connectTunnel() { var sessionCache = {}; @@ -38,29 +60,20 @@ module.exports.create = function (deps, conf) { return sessionCache[id]; } - conf.domains.forEach(function(dom) { - if (dom.modules && Array.isArray(dom.modules.ddns) && dom.modules.ddns.length) { - var mod = dom.modules.ddns[0]; - getSession(mod.token_id).then(function (session) { - return deps.tunnelClients.start(session, dom.names); - }).catch(function (err) { - console.log('error starting tunnel for', dom.names.join(', ')); - console.log(err); - }); - } - }); + await iterateAllModules(function startTunnel(mod, domainsList) { + if (mod.type !== 'dns@oauth3.org') { return null; } - conf.ddns.modules.forEach(function (mod) { - getSession(mod.token_id).then(function (session) { - return deps.tunnelClients.start(session, mod.domains); + return getSession(mod.token_id).then(function (session) { + return deps.tunnelClients.start(session, domainsList); }).catch(function (err) { - console.log('error starting tunnel for', mod.domains.join(', ')); + console.log('error starting tunnel for', domainsList.join(', ')); console.log(err); }); }); + tunnelActive = true; } - function disconnectTunnel() { + async function disconnectTunnel() { deps.tunnelClients.disconnect(); tunnelActive = false; } @@ -87,16 +100,16 @@ module.exports.create = function (deps, conf) { // // TODO: try to automatically configure router to forward ports to us. // } - // If we are on a public accress or all ports we are listening on are forwarded to us then + // If we are on a public address or all ports we are listening on are forwarded to us then // we don't need the tunnel and we can set the DNS records for all our domains to our public // address. Otherwise we need to use the tunnel to accept traffic. if (!notLooped.length) { if (tunnelActive) { - disconnectTunnel(); + await disconnectTunnel(); } } else { if (!tunnelActive) { - connectTunnel(); + await connectTunnel(); } } } @@ -129,32 +142,13 @@ module.exports.create = function (deps, conf) { return sessionCache[id]; } - conf.domains.forEach(function(dom) { - if (dom.modules && Array.isArray(dom.modules.ddns)) { - dom.modules.ddns.some(function (mod) { - if (mod.type !== 'dns@oauth3.org' || mod.disabled) { - return false; - } + await iterateAllModules(function setModuleDNS(mod, domainsList) { + if (mod.type !== 'dns@oauth3.org' || mod.disabled) { return null; } - return getSession(mod.token_id).then(function (session) { - return dnsCtrl.setDeviceAddress(session, addr, dom.names); - }).catch(function (err) { - console.log('error setting DNS records for', dom.names.join(', ')); - console.log(err); - }); - }); - } - }); - - conf.ddns.modules.forEach(function (mod) { - if (mod.type !== 'dns@oauth3.org' || mod.disabled) { - return; - } - - getSession(mod.token_id).then(function (session) { - return dnsCtrl.setDeviceAddress(session, addr, mod.domains); + return getSession(mod.token_id).then(function (session) { + return dnsCtrl.setDeviceAddress(session, addr, domainsList); }).catch(function (err) { - console.log('error setting DNS records for', mod.domains.join(', ')); + console.log('error setting DNS records for', domainsList.join(', ')); console.log(err); }); }); From c23f5ae25b6fcc3f081eaaa0b7b3d107ce70fa90 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 19 Oct 2017 12:58:04 -0600 Subject: [PATCH 36/41] moved the session cache to be longer lasting --- lib/ddns/index.js | 42 ++++++++++++++---------------------------- lib/storage.js | 24 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 0de208f..4939d68 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -40,31 +40,28 @@ module.exports.create = function (deps, conf) { return deps.PromiseA.all(promises.filter(Boolean)); } + async function getSession(id) { + var session = await deps.storage.tokens.get(id); + if (!session) { + throw new Error('no user token with ID "'+id+'"'); + } + return session; + } + var tunnelActive = false; async function connectTunnel() { - var sessionCache = {}; - var sessionOverride; + var tunnelSession; if (conf.ddns.tunnel) { - sessionOverride = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); - } - async function getSession(id) { - if (sessionOverride) { - return sessionOverride; - } - if (!sessionCache.hasOwnProperty(id)) { - sessionCache[id] = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); - } - if (!sessionCache[id]) { - throw new Error('no user token with ID "'+id+'"'); - } - return sessionCache[id]; + // In the case of a non-existant token, I'm not sure if we want to throw here and prevent + // any tunnel connections, or if we want to ignore it and fall back to the DNS tokens + tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); } await iterateAllModules(function startTunnel(mod, domainsList) { if (mod.type !== 'dns@oauth3.org') { return null; } - return getSession(mod.token_id).then(function (session) { - return deps.tunnelClients.start(session, domainsList); + return getSession(mod.token_id).then(function (dnsSession) { + return deps.tunnelClients.start(tunnelSession || dnsSession, domainsList); }).catch(function (err) { console.log('error starting tunnel for', domainsList.join(', ')); console.log(err); @@ -131,17 +128,6 @@ module.exports.create = function (deps, conf) { } publicAddress = addr; - var sessionCache = {}; - async function getSession(id) { - if (!sessionCache.hasOwnProperty(id)) { - sessionCache[id] = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); - } - if (!sessionCache[id]) { - throw new Error('no user token with ID "'+id+'"'); - } - return sessionCache[id]; - } - await iterateAllModules(function setModuleDNS(mod, domainsList) { if (mod.type !== 'dns@oauth3.org' || mod.disabled) { return null; } diff --git a/lib/storage.js b/lib/storage.js index 271fed7..5c2928f 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -97,7 +97,8 @@ module.exports.create = function (deps, conf) { var userTokens = { _filename: 'user-tokens.json' - , _convertToken(id, token) { + , _cache: {} + , _convertToken: function convertToken(id, token) { // convert the token into something that looks more like what OAuth3 uses internally // as sessions so we can use it with OAuth3. We don't use OAuth3's internal session // storage because it effectively only supports storing tokens based on provider URI. @@ -118,16 +119,31 @@ module.exports.create = function (deps, conf) { } , all: function allUserTokens() { var self = this; + if (self._cacheComplete) { + return deps.PromiseA.resolve(Object.values(self._cache)); + } + return read(self._filename).then(function (tokens) { + // We will read every single token into our cache, so it will be complete once we finish + // creating the result (it's set out of order so we can directly return the result). + self._cacheComplete = true; + return Object.keys(tokens).map(function (id) { - return self._convertToken(id, tokens[id]); + self._cache[id] = self._convertToken(id, tokens[id]); + return self._cache[id]; }); }); } , get: function getUserToken(id) { var self = this; + if (self._cache.hasOwnProperty(id) || self._cacheComplete) { + return deps.PromiseA.resolve(self._cache[id] || null); + } + return read(self._filename).then(function (tokens) { return self._convertToken(id, tokens[id]); + }).then(function (session) { + self._cache[id] = session; }); } , save: function saveUserToken(newToken) { @@ -155,6 +171,9 @@ module.exports.create = function (deps, conf) { var id = idHash.digest('hex'); tokens[id] = rawToken; return write(self._filename, tokens).then(function () { + // Delete the current cache so that if this is an update it will refresh + // the cache once we read the ID. + delete self._cache[id]; return self.get(id); }); }); @@ -168,6 +187,7 @@ module.exports.create = function (deps, conf) { } return write(self._filename, tokens).then(function () { + delete self._cache[id]; return true; }); }); From acf2fd7764cb39dcccb5c22360cc300802bf7c6e Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 19 Oct 2017 17:45:05 -0600 Subject: [PATCH 37/41] looking at active tunnel session on DDNS config update --- .jshintrc | 1 + lib/admin/config.js | 1 + lib/ddns/index.js | 98 +++++++++++++++++++++----- lib/storage.js | 5 +- lib/tunnel-client-manager.js | 131 ++++++++++++++++++++--------------- package-lock.json | 5 ++ package.json | 1 + 7 files changed, 163 insertions(+), 79 deletions(-) diff --git a/.jshintrc b/.jshintrc index 63801ce..7c6a6ae 100644 --- a/.jshintrc +++ b/.jshintrc @@ -13,4 +13,5 @@ , "latedef": true , "curly": true , "trailing": true +, "esversion": 6 } diff --git a/lib/admin/config.js b/lib/admin/config.js index 607e2a6..41f40a4 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -335,6 +335,7 @@ class ConfigChanger { constructor(start) { Object.assign(this, JSON.parse(JSON.stringify(start))); delete this.device; + delete this.debug; this.domains = new DomainList(this.domains); this.http.modules = new ModuleList(this.http.modules); diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 4939d68..036bb02 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -4,19 +4,9 @@ module.exports.create = function (deps, conf) { var network = deps.PromiseA.promisifyAll(deps.recase.camelCopy(require('network'))); var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); + var equal = require('deep-equal'); var loopbackDomain; - function updateConf() { - loopbackDomain = 'oauth3.org'; - if (conf.ddns && conf.ddns.loopback) { - if (conf.ddns.loopback.type === 'tunnel@oauth3.org' && conf.ddns.loopback.domain) { - loopbackDomain = conf.ddns.loopback.domain; - } else { - console.error('invalid loopback configuration: bad type or missing domain'); - } - } - } - updateConf(); function iterateAllModules(action) { var promises = conf.ddns.modules.map(function (mod) { @@ -57,13 +47,13 @@ module.exports.create = function (deps, conf) { tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); } - await iterateAllModules(function startTunnel(mod, domainsList) { + await iterateAllModules(function startTunnel(mod, domainList) { if (mod.type !== 'dns@oauth3.org') { return null; } - return getSession(mod.token_id).then(function (dnsSession) { - return deps.tunnelClients.start(tunnelSession || dnsSession, domainsList); + return getSession(mod.tokenId).then(function (dnsSession) { + return deps.tunnelClients.start(tunnelSession || dnsSession, domainList); }).catch(function (err) { - console.log('error starting tunnel for', domainsList.join(', ')); + console.log('error starting tunnel for', domainList.join(', ')); console.log(err); }); }); @@ -73,6 +63,44 @@ module.exports.create = function (deps, conf) { async function disconnectTunnel() { deps.tunnelClients.disconnect(); tunnelActive = false; + await Promise.resolve(); + } + async function checkTunnelTokens() { + var oldTokens = deps.tunnelClients.current(); + + var newTokens = await iterateAllModules(function checkTokens(mod, domainList) { + if (mod.type !== 'dns@oauth3.org') { return null; } + + var domainStr = domainList.slice().sort().join(','); + // If there is already a token handling exactly the domains this modules + // needs handled remove it from the list of tokens to be removed. Otherwise + // return the module and domain list so we can get new tokens. + if (oldTokens[domainStr]) { + delete oldTokens[domainStr]; + } else { + return Promise.resolve({ mod, domainList }); + } + }); + + await Promise.all(Object.values(oldTokens).map(deps.tunnelClients.remove)); + + if (!newTokens.length) { return; } + + var tunnelSession; + if (conf.ddns.tunnel) { + // In the case of a non-existant token, I'm not sure if we want to throw here and prevent + // any tunnel connections, or if we want to ignore it and fall back to the DNS tokens + tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); + } + + await Promise.all(newTokens.map(function startTunnel({mod, domainList}) { + return getSession(mod.tokenId).then(function (dnsSession) { + return deps.tunnelClients.start(tunnelSession || dnsSession, domainList); + }).catch(function (err) { + console.log('error starting tunnel for', domainList.join(', ')); + console.log(err); + }); + })); } var localAddr, gateway; @@ -128,13 +156,13 @@ module.exports.create = function (deps, conf) { } publicAddress = addr; - await iterateAllModules(function setModuleDNS(mod, domainsList) { + await iterateAllModules(function setModuleDNS(mod, domainList) { if (mod.type !== 'dns@oauth3.org' || mod.disabled) { return null; } - return getSession(mod.token_id).then(function (session) { - return dnsCtrl.setDeviceAddress(session, addr, domainsList); + return getSession(mod.tokenId).then(function (session) { + return dnsCtrl.setDeviceAddress(session, addr, domainList); }).catch(function (err) { - console.log('error setting DNS records for', domainsList.join(', ')); + console.log('error setting DNS records for', domainList.join(', ')); console.log(err); }); }); @@ -143,6 +171,38 @@ module.exports.create = function (deps, conf) { recheckPubAddr(); setInterval(recheckPubAddr, 5*60*1000); + var curConf; + function updateConf() { + if (curConf && equal(curConf.ddns, conf.ddns) && equal(curConf.domains, conf.domains)) { + // We could update curConf, but since everything we care about is the same... + return; + } + + if (!curConf || !equal(curConf.ddns.loopback, conf.ddns.loopback)) { + loopbackDomain = 'oauth3.org'; + if (conf.ddns && conf.ddns.loopback) { + if (conf.ddns.loopback.type === 'tunnel@oauth3.org' && conf.ddns.loopback.domain) { + loopbackDomain = conf.ddns.loopback.domain; + } else { + console.error('invalid loopback configuration: bad type or missing domain'); + } + } + } + + if (tunnelActive) { + if (!curConf || !equal(curConf.ddns.tunnel, conf.ddns.tunnel)) { + disconnectTunnel().then(connectTunnel); + } else { + checkTunnelTokens(); + } + } + + // We need to make a deep copy of the config so we can use it next time to + // compare and see what setup/cleanup is needed to adapt to the changes. + curConf = JSON.parse(JSON.stringify(conf)); + } + updateConf(); + return { loopbackServer: loopback.server , setDeviceAddress: dnsCtrl.setDeviceAddress diff --git a/lib/storage.js b/lib/storage.js index 5c2928f..56d73d4 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -141,9 +141,8 @@ module.exports.create = function (deps, conf) { } return read(self._filename).then(function (tokens) { - return self._convertToken(id, tokens[id]); - }).then(function (session) { - self._cache[id] = session; + self._cache[id] = self._convertToken(id, tokens[id]); + return self._cache[id]; }); } , save: function saveUserToken(newToken) { diff --git a/lib/tunnel-client-manager.js b/lib/tunnel-client-manager.js index 5355295..29105f5 100644 --- a/lib/tunnel-client-manager.js +++ b/lib/tunnel-client-manager.js @@ -2,31 +2,68 @@ module.exports.create = function (deps, config) { var stunnel = require('stunnel'); + var jwt = require('jsonwebtoken'); var activeTunnels = {}; + var activeDomains = {}; - function addToken(data) { + function fillData(data) { if (typeof data === 'string') { data = { jwt: data }; } + if (!data.jwt) { - return deps.PromiseA.reject(new Error("missing 'jwt' from tunnel data")); + throw new Error("missing 'jwt' from tunnel data"); } + var decoded = jwt.decode(data.jwt); + if (!decoded) { + throw new Error('invalid JWT'); + } + if (!data.tunnelUrl) { - var decoded; - try { - decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii')); - } catch (err) { - console.warn('invalid web token given to tunnel manager', err); - return deps.PromiseA.reject(err); - } if (!decoded.aud) { - console.warn('tunnel manager given token with no tunnelUrl or audience'); - var err = new Error('missing tunnelUrl and audience'); - return deps.PromiseA.reject(err); + throw new Error('missing tunnelUrl and audience'); } data.tunnelUrl = 'wss://' + decoded.aud + '/'; } + data.domains = (decoded.domains || []).slice().sort().join(','); + if (!data.domains) { + throw new Error('JWT contains no domains to be forwarded'); + } + + return data; + } + + async function removeToken(data) { + data = fillData(data); + + // Not sure if we might want to throw an error indicating the token didn't + // even belong to a server that existed, but since it never existed we can + // consider it as "removed". + if (!activeTunnels[data.tunnelUrl]) { + return; + } + + console.log('removing token from tunnel at', data.tunnelUrl); + return activeTunnels[data.tunnelUrl].clear(data.jwt).then(function () { + delete activeDomains[data.domains]; + }); + } + + async function addToken(data) { + data = fillData(data); + + if (activeDomains[data.domains]) { + // If already have a token with the exact same domains and to the same tunnel + // server there isn't really a need to add a new one + if (activeDomains[data.domains].tunnelUrl === data.tunnelUrl) { + return; + } + // Otherwise we want to detach from the other tunnel server in favor of the new one + console.warn('added token with the exact same domains as another'); + await removeToken(activeDomains[data.domains]); + } + if (!activeTunnels[data.tunnelUrl]) { console.log('creating new tunnel client for', data.tunnelUrl); // We create the tunnel without an initial token so we can append the token and @@ -48,11 +85,16 @@ module.exports.create = function (deps, config) { }); } - console.log('appending token to tunnel at', data.tunnelUrl); - return activeTunnels[data.tunnelUrl].append(data.jwt); + console.log('appending token to tunnel at', data.tunnelUrl, 'for domains', data.domains); + await activeTunnels[data.tunnelUrl].append(data.jwt); + + // Now that we know the tunnel server accepted our token we can save it + // to keep record of what domains we are handling and what tunnel server + // those domains should go to. + activeDomains[data.domains] = data; } - function acquireToken(session, domains) { + async function acquireToken(session, domains) { var OAUTH3 = deps.OAUTH3; // The OAUTH3 library stores some things on the root session object that we usually @@ -62,51 +104,21 @@ module.exports.create = function (deps, config) { session.scope = session.scope || session.token.scp; console.log('asking for tunnel token from', session.token.aud); - return OAUTH3.discover(session.token.aud).then(function (directives) { - var opts = { - api: 'tunnel.token' - , session: session - , data: { - domains: domains - , device: { - hostname: config.device.hostname - , id: config.device.uid || config.device.id - } + var opts = { + api: 'tunnel.token' + , session: session + , data: { + domains: domains + , device: { + hostname: config.device.hostname + , id: config.device.uid || config.device.id } - }; - - return OAUTH3.api(directives.api, opts).then(addToken); - }); - } - - function removeToken(data) { - if (typeof data === 'string') { - data = { jwt: data }; - } - if (!data.tunnelUrl) { - var decoded; - try { - decoded = JSON.parse(new Buffer(data.jwt.split('.')[1], 'base64').toString('ascii')); - } catch (err) { - console.warn('invalid web token given to tunnel manager', err); - return deps.PromiseA.reject(err); } - if (!decoded.aud) { - console.warn('tunnel manager given token with no tunnelUrl or audience'); - var err = new Error('missing tunnelUrl and audience'); - return deps.PromiseA.reject(err); - } - data.tunnelUrl = 'wss://' + decoded.aud + '/'; - } + }; - // Not sure if we actually want to return an error that the token didn't even belong to a - // server that existed, but since it never existed we can consider it as "removed". - if (!activeTunnels[data.tunnelUrl]) { - return deps.PromiseA.resolve(); - } - - console.log('removing token from tunnel at', data.tunnelUrl); - return activeTunnels[data.tunnelUrl].clear(data.jwt); + var directives = await OAUTH3.discover(session.token.aud); + var tokenData = await OAUTH3.api(directives.api, opts); + await addToken(tokenData); } function disconnectAll() { @@ -115,10 +127,15 @@ module.exports.create = function (deps, config) { }); } + function currentTokens() { + return JSON.parse(JSON.stringify(activeDomains)); + } + return { start: acquireToken , startDirect: addToken , remove: removeToken , disconnect: disconnectAll + , current: currentTokens }; }; diff --git a/package-lock.json b/package-lock.json index 49f699d..08e55c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -427,6 +427,11 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", diff --git a/package.json b/package.json index 1d70e3f..9028745 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "bluebird": "^3.4.6", "body-parser": "git+https://github.com/expressjs/body-parser.git#1.16.1", "commander": "^2.9.0", + "deep-equal": "^1.0.1", "dns-suite": "git+https://git@git.daplie.com/Daplie/dns-suite#v1", "express": "git+https://github.com/expressjs/express.git#4.x", "finalhandler": "^0.4.0", From 82f0b45c56dc36b26602bb065335226a8f6a5225 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Fri, 20 Oct 2017 15:38:10 -0600 Subject: [PATCH 38/41] implemented cleanup/update of DNS records on config change --- lib/ddns/dns-ctrl.js | 25 +++++++-- lib/ddns/index.js | 119 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 127 insertions(+), 17 deletions(-) diff --git a/lib/ddns/dns-ctrl.js b/lib/ddns/dns-ctrl.js index ed290ac..f2421c0 100644 --- a/lib/ddns/dns-ctrl.js +++ b/lib/ddns/dns-ctrl.js @@ -23,7 +23,7 @@ module.exports.create = function (deps, conf) { var tldObj = {}; resp.data.forEach(function (tldInfo) { if (tldInfo.enabled) { - tldObj[tldInfo.com] = true; + tldObj[tldInfo.tld] = true; } }); @@ -60,8 +60,8 @@ module.exports.create = function (deps, conf) { // rare even if the assumption isn't valid. return { tld: split.slice(-tldSegCnt).join('.') - , sld: split.slice(-tldSegCnt-1, 1) - , sub: split.slice(0, -tldSegCnt-1) + , sld: split.slice(-tldSegCnt-1, -tldSegCnt).join('.') + , sub: split.slice(0, -tldSegCnt-1).join('.') }; }); } @@ -152,8 +152,23 @@ module.exports.create = function (deps, conf) { return (dev || {}).addresses || []; } + async function removeDomains(session, domains) { + var directives = await deps.OAUTH3.discover(session.token.aud); + + var oldDns = await splitDomains(directives.api, domains); + var common = { + api: 'devices.detach' + , session: session + , device: conf.device.hostname + }; + await deps.PromiseA.all(oldDns.map(function (record) { + return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); + })); + } + return { - getDeviceAddresses: getDeviceAddresses - , setDeviceAddress: setDeviceAddress + getDeviceAddresses + , setDeviceAddress + , removeDomains }; }; diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 036bb02..74cdc99 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -8,12 +8,13 @@ module.exports.create = function (deps, conf) { var loopbackDomain; - function iterateAllModules(action) { - var promises = conf.ddns.modules.map(function (mod) { + function iterateAllModules(action, curConf) { + curConf = curConf || conf; + var promises = curConf.ddns.modules.map(function (mod) { return action(mod, mod.domains); }); - conf.domains.forEach(function (dom) { + curConf.domains.forEach(function (dom) { if (!dom.modules || !Array.isArray(dom.modules.ddns) || !dom.modules.ddns.length) { return null; } @@ -168,6 +169,87 @@ module.exports.create = function (deps, conf) { }); } + function getModuleDiffs(prevConf) { + var prevMods = {}; + var curMods = {}; + + // this returns a Promise, but since the functions we use are synchronous + // and change our enclosed variables we don't need to wait for the return. + iterateAllModules(function (mod, domainList) { + if (mod.type !== 'dns@oauth3.org') { return; } + + prevMods[mod.id] = { mod, domainList }; + return true; + }, prevConf); + iterateAllModules(function (mod, domainList) { + if (mod.type !== 'dns@oauth3.org') { return; } + + curMods[mod.id] = { mod, domainList }; + return true; + }); + + // Filter out all of the modules that are exactly the same including domainList + // since there is no required action to transition. + Object.keys(prevMods).map(function (id) { + if (equal(prevMods[id], curMods[id])) { + delete prevMods[id]; + delete curMods[id]; + } + }); + + return {prevMods, curMods}; + } + async function cleanOldDns(prevConf) { + var {prevMods, curMods} = getModuleDiffs(prevConf); + + // Then remove DNS records for the domains that we are no longer responsible for. + await Promise.all(Object.values(prevMods).map(function ({mod, domainList}) { + var oldDomains; + if (!curMods[mod.id] || mod.tokenId !== curMods[mod.id].mod.tokenId) { + oldDomains = domainList.slice(); + } else { + oldDomains = domainList.filter(function (domain) { + return curMods[mod.id].domainList.indexOf(domain) < 0; + }); + } + if (conf.debug) { + console.log('removing old domains for module', mod.id, oldDomains.join(', ')); + } + if (!oldDomains.length) { + return; + } + + return getSession(mod.tokenId).then(function (session) { + return dnsCtrl.removeDomains(session, oldDomains); + }); + }).filter(Boolean)); + } + async function setNewDns(prevConf) { + var {prevMods, curMods} = getModuleDiffs(prevConf); + + // And add DNS records for any newly added domains. + await Promise.all(Object.values(curMods).map(function ({mod, domainList}) { + var newDomains; + if (!prevMods[mod.id] || mod.tokenId !== prevMods[mod.id].mod.tokenId) { + newDomains = domainList.slice(); + } else { + newDomains = domainList.filter(function (domain) { + return prevMods[mod.id].domainList.indexOf(domain) < 0; + }); + } + if (conf.debug) { + console.log('adding new domains for module', mod.id, newDomains.join(', ')); + } + if (!newDomains.length) { + return; + } + + return getSession(mod.tokenId).then(function (session) { + return dnsCtrl.setDeviceAddress(session, publicAddress, newDomains); + }); + }).filter(Boolean)); + } + recheckPubAddr(); setInterval(recheckPubAddr, 5*60*1000); @@ -189,17 +271,30 @@ module.exports.create = function (deps, conf) { } } - if (tunnelActive) { - if (!curConf || !equal(curConf.ddns.tunnel, conf.ddns.tunnel)) { - disconnectTunnel().then(connectTunnel); - } else { - checkTunnelTokens(); - } + if (!curConf) { + // We need to make a deep copy of the config so we can use it next time to + // compare and see what setup/cleanup is needed to adapt to the changes. + curConf = JSON.parse(JSON.stringify(conf)); + return; } - // We need to make a deep copy of the config so we can use it next time to - // compare and see what setup/cleanup is needed to adapt to the changes. - curConf = JSON.parse(JSON.stringify(conf)); + cleanOldDns(curConf).then(function () { + if (!tunnelActive) { + return setNewDns(curConf); + } + if (equal(curConf.ddns.tunnel, conf.ddns.tunnel)) { + return checkTunnelTokens(); + } else { + return disconnectTunnel().then(connectTunnel); + } + }).catch(function (err) { + console.error('error transitioning DNS between configurations'); + console.error(err); + }).then(function () { + // We need to make a deep copy of the config so we can use it next time to + // compare and see what setup/cleanup is needed to adapt to the changes. + curConf = JSON.parse(JSON.stringify(conf)); + }); } updateConf(); From 00de23ded7d55c58268d7045a1aaeec667e18439 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Fri, 20 Oct 2017 18:02:55 -0600 Subject: [PATCH 39/41] implemented setting DNS records after tunnel connect currently done automatically by API we get the tunnel token from, but in the near-ish future that will be changed --- lib/ddns/dns-ctrl.js | 18 ++++++++---- lib/ddns/index.js | 53 +++++++++++++++++++++++------------- lib/tunnel-client-manager.js | 6 +++- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/lib/ddns/dns-ctrl.js b/lib/ddns/dns-ctrl.js index f2421c0..2c8912b 100644 --- a/lib/ddns/dns-ctrl.js +++ b/lib/ddns/dns-ctrl.js @@ -98,12 +98,20 @@ module.exports.create = function (deps, conf) { return domains.indexOf(record.host) !== -1; }); - var oldDomains = ourDns.filter(function (record) { + // Of all the DNS records referring to our device and the current list of domains determine + // which domains have records with outdated address, and which ones we can just leave be + // without updating them. + var badAddrDomains = ourDns.filter(function (record) { return record.value !== addr; - }).map(function (record) { - return record.host; + }).map(record => record.host); + var goodAddrDomains = ourDns.filter(function (record) { + return record.value === addr && badAddrDomains.indexOf(record.host) < 0; + }).map(record => record.host); + var requiredUpdates = domains.filter(function (domain) { + return goodAddrDomains.indexOf(domain) !== -1; }); - var oldDns = await splitDomains(directives.api, oldDomains); + + var oldDns = await splitDomains(directives.api, badAddrDomains); var common = { api: 'devices.detach' , session: session @@ -113,7 +121,7 @@ module.exports.create = function (deps, conf) { return deps.OAUTH3.api(directives.api, Object.assign({}, common, record)); })); - var newDns = await splitDomains(directives.api, domains); + var newDns = await splitDomains(directives.api, requiredUpdates); common = { api: 'devices.attach' , session: session diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 74cdc99..60878cf 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -1,6 +1,7 @@ 'use strict'; module.exports.create = function (deps, conf) { + var dns = deps.PromiseA.promisifyAll(require('dns')); var network = deps.PromiseA.promisifyAll(deps.recase.camelCopy(require('network'))); var loopback = require('./loopback').create(deps, conf); var dnsCtrl = require('./dns-ctrl').create(deps, conf); @@ -40,7 +41,31 @@ module.exports.create = function (deps, conf) { } var tunnelActive = false; - async function connectTunnel() { + async function startTunnel(tunnelSession, mod, domainList) { + try { + var dnsSession = await getSession(mod.tokenId); + var tunnelDomain = await deps.tunnelClients.start(tunnelSession || dnsSession, domainList); + + var addrList; + try { + addrList = await dns.resolve4Async(tunnelDomain); + } catch (e) {} + if (!addrList || !addrList.length) { + try { + addrList = await dns.resolve6Async(tunnelDomain); + } catch (e) {} + } + if (!addrList || !addrList.length || !addrList[0]) { + throw new Error('failed to lookup IP for tunnel domain "' + tunnelDomain + '"'); + } + + await dnsCtrl.setDeviceAddress(dnsSession, addrList[0], domainList); + } catch (err) { + console.log('error starting tunnel for', domainList.join(', ')); + console.log(err); + } + } + async function connectAllTunnels() { var tunnelSession; if (conf.ddns.tunnel) { // In the case of a non-existant token, I'm not sure if we want to throw here and prevent @@ -48,20 +73,15 @@ module.exports.create = function (deps, conf) { tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); } - await iterateAllModules(function startTunnel(mod, domainList) { + await iterateAllModules(function (mod, domainList) { if (mod.type !== 'dns@oauth3.org') { return null; } - return getSession(mod.tokenId).then(function (dnsSession) { - return deps.tunnelClients.start(tunnelSession || dnsSession, domainList); - }).catch(function (err) { - console.log('error starting tunnel for', domainList.join(', ')); - console.log(err); - }); + return startTunnel(tunnelSession, mod, domainList); }); tunnelActive = true; } - async function disconnectTunnel() { + async function disconnectTunnels() { deps.tunnelClients.disconnect(); tunnelActive = false; await Promise.resolve(); @@ -94,13 +114,8 @@ module.exports.create = function (deps, conf) { tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId); } - await Promise.all(newTokens.map(function startTunnel({mod, domainList}) { - return getSession(mod.tokenId).then(function (dnsSession) { - return deps.tunnelClients.start(tunnelSession || dnsSession, domainList); - }).catch(function (err) { - console.log('error starting tunnel for', domainList.join(', ')); - console.log(err); - }); + await Promise.all(newTokens.map(function ({mod, domainList}) { + return startTunnel(tunnelSession, mod, domainList); })); } @@ -131,11 +146,11 @@ module.exports.create = function (deps, conf) { // address. Otherwise we need to use the tunnel to accept traffic. if (!notLooped.length) { if (tunnelActive) { - await disconnectTunnel(); + await disconnectTunnels(); } } else { if (!tunnelActive) { - await connectTunnel(); + await connectAllTunnels(); } } } @@ -285,7 +300,7 @@ module.exports.create = function (deps, conf) { if (equal(curConf.ddns.tunnel, conf.ddns.tunnel)) { return checkTunnelTokens(); } else { - return disconnectTunnel().then(connectTunnel); + return disconnectTunnels().then(connectAllTunnels); } }).catch(function (err) { console.error('error transitioning DNS between configurations'); diff --git a/lib/tunnel-client-manager.js b/lib/tunnel-client-manager.js index 29105f5..8153245 100644 --- a/lib/tunnel-client-manager.js +++ b/lib/tunnel-client-manager.js @@ -92,6 +92,10 @@ module.exports.create = function (deps, config) { // to keep record of what domains we are handling and what tunnel server // those domains should go to. activeDomains[data.domains] = data; + + // This is mostly for the start, but return the host for the tunnel server + // we've connected to (after stripping the protocol and path away). + return data.tunnelUrl.replace(/^[a-z]*:\/\//i, '').replace(/\/.*/, ''); } async function acquireToken(session, domains) { @@ -118,7 +122,7 @@ module.exports.create = function (deps, config) { var directives = await OAUTH3.discover(session.token.aud); var tokenData = await OAUTH3.api(directives.api, opts); - await addToken(tokenData); + return addToken(tokenData); } function disconnectAll() { From 72ff65e8339f317ad78f291c22573d2ac77f82ea Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 25 Oct 2017 11:00:06 -0600 Subject: [PATCH 40/41] fix some misc problem found using browser to access API --- lib/admin/apis.js | 7 +++++++ lib/admin/config.js | 9 ++------- lib/ddns/index.js | 10 ++++++++-- lib/modules/tls.js | 5 +---- package-lock.json | 6 +++--- package.json | 2 +- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index 5bbe49a..e4ea68e 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -60,6 +60,13 @@ module.exports.create = function (deps, conf) { } function isAuthorized(req, res, fn) { + // OPTIONS requests are only to determine if a particular request is allowed, and the + // browser won't send the session header with this request, so don't try to authenticate. + if (req.method === 'OPTIONS') { + fn(); + return; + } + var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, '')); if (!auth) { res.statusCode = 401; diff --git a/lib/admin/config.js b/lib/admin/config.js index 41f40a4..2d10423 100644 --- a/lib/admin/config.js +++ b/lib/admin/config.js @@ -50,9 +50,8 @@ var moduleSchemas = { } // the dns control modules for DDNS -, dns_oauth3_org: { - name: 'dns@oauth3.org' - , type: 'object' +, 'dns@oauth3.org': { + type: 'object' , required: [ 'token_id' ] , properties: { token_id: { type: 'string' } @@ -67,10 +66,6 @@ moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema }; Object.keys(moduleSchemas).forEach(function (name) { var schema = moduleSchemas[name]; - if (schema.name) { - name = schema.name; - delete schema.name; - } schema.id = '/modules/'+name; schema.required = ['id', 'type'].concat(schema.required || []); schema.properties.id = { type: 'string' }; diff --git a/lib/ddns/index.js b/lib/ddns/index.js index 60878cf..df70ab9 100644 --- a/lib/ddns/index.js +++ b/lib/ddns/index.js @@ -265,8 +265,14 @@ module.exports.create = function (deps, conf) { }).filter(Boolean)); } - recheckPubAddr(); - setInterval(recheckPubAddr, 5*60*1000); + function check() { + recheckPubAddr().catch(function (err) { + console.error('failed to handle all actions needed for DDNS'); + console.error(err); + }); + } + check(); + setInterval(check, 5*60*1000); var curConf; function updateConf() { diff --git a/lib/modules/tls.js b/lib/modules/tls.js index 2b9a614..63d4d38 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -50,10 +50,7 @@ module.exports.create = function (deps, config, netHandler) { return; } - process.nextTick(function () { - socket.unshift(opts.firstChunk); - }); - + writer.write(opts.firstChunk); socket.pipe(writer); writer.pipe(socket); diff --git a/package-lock.json b/package-lock.json index 08e55c5..67f9108 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1907,9 +1907,9 @@ } }, "socket-pair": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/socket-pair/-/socket-pair-1.0.1.tgz", - "integrity": "sha1-mneFcEv9yOj2NxwodeyjIeMT/po=", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/socket-pair/-/socket-pair-1.0.3.tgz", + "integrity": "sha512-O1WJMNIPAAGCzzJi1Lk9K9adctKM4DukiUO6G6sQSs+CqEAZ5uGX86uIMDKygBZZr62YHDoOGH1rJShOzw6i9Q==", "requires": { "bluebird": "3.5.0" } diff --git a/package.json b/package.json index 9028745..20387c8 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "serve-static": "^1.10.0", "server-destroy": "^1.0.1", "sni": "^1.0.0", - "socket-pair": "^1.0.1", + "socket-pair": "^1.0.3", "socksv5": "0.0.6", "stunnel": "git+https://git.daplie.com/Daplie/node-tunnel-client.git#v1", "stunneld": "git+https://git.daplie.com/Daplie/node-tunnel-server.git#v1", From 20cf66c67dca7f948313d6c0c3c94f11595b1d7a Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 25 Oct 2017 13:35:06 -0600 Subject: [PATCH 41/41] added CORS header needed after recent change to OAuth3 library requests --- lib/admin/apis.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/admin/apis.js b/lib/admin/apis.js index e4ea68e..ab8f589 100644 --- a/lib/admin/apis.js +++ b/lib/admin/apis.js @@ -21,6 +21,7 @@ module.exports.create = function (deps, conf) { res.setHeader('Access-Control-Allow-Origin', req.headers.origin || '*'); res.setHeader('Access-Control-Allow-Methods', methods.join(', ')); res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + res.setHeader('Access-Control-Allow-Credentials', 'true'); if (req.method.toUpperCase() === 'OPTIONS') { res.setHeader('Allow', methods.join(', ')); @@ -60,13 +61,6 @@ module.exports.create = function (deps, conf) { } function isAuthorized(req, res, fn) { - // OPTIONS requests are only to determine if a particular request is allowed, and the - // browser won't send the session header with this request, so don't try to authenticate. - if (req.method === 'OPTIONS') { - fn(); - return; - } - var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, '')); if (!auth) { res.statusCode = 401; @@ -558,10 +552,9 @@ module.exports.create = function (deps, conf) { // add middleware without worrying too much about the consequences to older code. app.use('/:name', handleOldApis); - app.use('/', isAuthorized, jsonParser); + // Not all routes support all of these methods, but not worth making this more specific + app.use('/', makeCorsHandler(['GET', 'POST', 'PUT', 'DELETE']), isAuthorized, jsonParser); - // Not all config routes support PUT or DELETE, but not worth making this more specific - app.use( '/config', makeCorsHandler(['GET', 'POST', 'PUT', 'DELETE'])); app.get( '/config', config.restful.readConfig); app.get( '/config/:group', config.restful.readConfig); app.get( '/config/:group/:mod(modules)/:modId?', config.restful.readConfig); @@ -583,7 +576,6 @@ module.exports.create = function (deps, conf) { app.put( '/config/domains/:domId', config.restful.updateDomain); app.delete('/config/domains/:domId', config.restful.removeDomain); - app.use( '/tokens', makeCorsHandler(['GET', 'POST', 'DELETE'])); app.get( '/tokens', tokens.restful.getAll); app.get( '/tokens/:id', tokens.restful.getOne); app.post( '/tokens', tokens.restful.save);