Merge branch 'master' of git.daplie.com:Daplie/goldilocks.js

This commit is contained in:
AJ ONeal 2017-10-24 12:51:33 -06:00
commit 5e48a2ed5e
9 changed files with 267 additions and 274 deletions

View File

@ -370,6 +370,41 @@ tunnel:
tunnelUrl: 'wss://api.tunnel.example.com/' tunnelUrl: 'wss://api.tunnel.example.com/'
``` ```
**NOTE**: The more common way to use the tunnel with goldilocks is to use the
API to have goldilocks get a token from `oauth3.org`. In order to do this you
will need to have initialized goldilocks with a token that has the `dns` and
`domains` scopes. This is probably easiest to do with the `daplie-desktop-app`,
which will also get the first tunnel token for you.
**If you want to add more domains** to handle on your device while using the tunnel
you will need to manually get a new token that will tell the tunnel server to
deliver the requests to the new domain(s) to your device. The first step in this
is to attach the new domains to your device. To get the name of the device you
can use the `config` API, but it's probably easiest to `ssh` onto the device and
get the hostname. You can also use the daplie cli tool to see what device name
your other domains are routed to.
```bash
# for every new domain you want to route attach the domain to your device
daplie devices:attach -n $new_domain -d $device_name
```
After that step you will need to use the API to get goldilocks to get a new token
that includes the new domains you attached. It is also recommended but not
required to remove the older token with the incomplete list of domains. Run the
following commands from the unit.
```bash
# remove the old token
rm /opt/goldilocks/lib/node_modules/goldilocks/var/tokens.json
# set the "refresh_token" to a bash variable `token`
TOKEN=$(python -mjson.tool /opt/goldilocks/lib/node_modules/goldilocks/var/owners.json | sed -n 's|\s*"refresh_token": "\(.*\)",|\1|p')
# tell goldilocks to get a new tunnel token
curl -H "authorization: bearer $TOKEN" -X POST https://localhost.admin.daplie.me/api/goldilocks@daplie.com/tunnel
```
### ddns ### ddns
TODO TODO

View File

@ -41,15 +41,12 @@ function createStorage(filename, filetype) {
} }
function read() { function read() {
return fs.readFileAsync(filename) return fs.readFileAsync(filename).then(parse).catch(function (err) {
.catch(function (err) {
if (err.code === 'ENOENT') { if (err.code === 'ENOENT') {
return ''; return '';
} }
return PromiseA.reject(err); return PromiseA.reject(err);
}) });
.then(parse)
;
} }
var result = { var result = {
@ -154,6 +151,9 @@ function fillConfig(config, args) {
if (!config.dns) { if (!config.dns) {
config.dns = { bind: [ 53 ], modules: [{ name: 'proxy', port: 3053 }] }; config.dns = { bind: [ 53 ], modules: [{ name: 'proxy', port: 3053 }] };
} }
if (!config.ddns) {
config.ddns = { enabled: false };
}
// Use Object.assign to add any properties needed but not defined in the mdns config. // Use Object.assign to add any properties needed but not defined in the mdns config.
// It will first copy the defaults into an empty object, then copy any real config over that. // It will first copy the defaults into an empty object, then copy any real config over that.
var mdnsDefaults = { port: 5353, broadcast: '224.0.0.251', ttl: 300 }; var mdnsDefaults = { port: 5353, broadcast: '224.0.0.251', ttl: 300 };
@ -250,25 +250,29 @@ function run(args) {
var workers = {}; var workers = {};
var cachedConfig; var cachedConfig;
cluster.on('message', function (worker, message) { function updateConfig(config) {
if (message.type !== 'com.daplie.goldilocks/config') { fillConfig(config, args).then(function (config) {
return;
}
configStorage.save(message.changes)
.then(function (config) {
return fillConfig(config, args);
})
.then(function (config) {
cachedConfig = config; cachedConfig = config;
console.log('changed config', config); console.log('changed config', config);
Object.keys(workers).forEach(function (key) { Object.keys(workers).forEach(function (key) {
workers[key].send(cachedConfig); workers[key].send(cachedConfig);
}); });
}) });
.catch(function (err) { }
process.on('SIGHUP', function () {
configStorage.read().then(updateConfig).catch(function (err) {
console.error('error updating config after SIGHUP', err);
});
});
cluster.on('message', function (worker, message) {
if (message.type !== 'com.daplie.goldilocks/config') {
return;
}
configStorage.save(message.changes).then(updateConfig).catch(function (err) {
console.error('error changing config', err); console.error('error changing config', err);
}) });
;
}); });
cluster.on('online', function (worker) { cluster.on('online', function (worker) {

View File

@ -1,88 +1,149 @@
'use strict'; 'use strict';
module.exports.create = function (opts/*, servers*/) { module.exports.create = function (deps, conf) {
var PromiseA = opts.PromiseA; var PromiseA = deps.PromiseA;
var dns = PromiseA.promisifyAll(require('dns')); var request = PromiseA.promisify(require('request'));
var OAUTH3 = require('../packages/assets/org.oauth3');
require('../packages/assets/org.oauth3/oauth3.dns.js');
OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js');
return PromiseA.all([ function dnsType(addr) {
dns.resolve4Async(opts._old_server_name).then(function (results) { if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) {
return results; return 'A';
}, function () {}) }
, dns.resolve6Async(opts._old_server_name).then(function (results) { if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) {
return results; return 'AAAA';
}, function () {}) }
]).then(function (results) { }
var ipv4 = results[0] || [];
var ipv6 = results[1] || [];
var record;
opts.dnsRecords = { async function getSession() {
A: ipv4 var sessions = await deps.storage.owners.all();
, AAAA: ipv6 var session = sessions.filter(function (sess) {
return sess.token.scp.indexOf('dns') >= 0;
})[0];
if (!session) {
throw new Error('no sessions with DNS grants');
}
// The OAUTH3 library stores some things on the root session object that we usually
// just leave inside the token, but we need to pull those out before we use it here
session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss;
session.client_uri = session.client_uri || session.token.azp;
session.scope = session.scope || session.token.scp;
return session;
}
async function setDeviceAddress(addr) {
var session = await getSession();
var directives = await OAUTH3.discover(session.token.aud);
// Set the address of the device to our public address.
await request({
url: directives.api+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname
, method: 'POST'
, headers: {
'Authorization': 'Bearer ' + session.refresh_token
, 'Accept': 'application/json; charset=utf-8'
}
, json: {
addresses: [
{ value: addr, type: dnsType(addr) }
]
}
});
// Then update all of the records attached to our hostname, first removing the old records
// to remove the reference to the old address, then creating new records for the same domains
// using our new address.
var allDns = OAUTH3.api(directives.api, {session: session, api: 'dns.list'});
var ourDomains = allDns.filter(function (record) {
return record.device === conf.device.hostname;
}).map(function (record) {
var zoneSplit = record.zone.split('.');
return {
tld: zoneSplit.slice(1).join('.')
, sld: zoneSplit[0]
, sub: record.host.slice(0, -(record.zone.length + 1))
}; };
Object.keys(opts.ifaces).some(function (ifacename) {
var iface = opts.ifaces[ifacename];
return iface.ipv4.some(function (localIp) {
return ipv4.some(function (remoteIp) {
if (localIp.address === remoteIp) {
record = localIp;
return record;
}
});
}) || iface.ipv6.some(function (localIp) {
return ipv6.forEach(function (remoteIp) {
if (localIp.address === remoteIp) {
record = localIp;
return record;
}
});
});
}); });
if (!record) { var common = {
console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address."); api: 'devices.detach'
console.info("Use --ddns to allow the people of the Internet to access your server."); , session: session
, device: conf.device.hostname
};
await PromiseA.all(ourDomains.map(function (record) {
return OAUTH3.api(directives.api, Object.assign({}, common, record));
}));
common = {
api: 'devices.attach'
, session: session
, device: conf.device.hostname
, ip: addr
, ttl: 300
};
await PromiseA.all(ourDomains.map(function (record) {
return OAUTH3.api(directives.api, Object.assign({}, common, record));
}));
} }
opts.externalIps.ipv4.some(function (localIp) { async function getDeviceAddresses() {
return ipv4.some(function (remoteIp) { var session = await getSession();
if (localIp.address === remoteIp) { var directives = await OAUTH3.discover(session.token.aud);
record = localIp;
return record; var result = await request({
url: directives.api+'/api/org.oauth3.dns/acl/devices'
, method: 'GET'
, headers: {
'Authorization': 'Bearer ' + session.refresh_token
, 'Accept': 'application/json; charset=utf-8'
} }
}); , json: true
}); });
opts.externalIps.ipv6.some(function (localIp) { if (!result.body) {
return ipv6.some(function (remoteIp) { throw new Error('No response body in request for device addresses');
if (localIp.address === remoteIp) { }
record = localIp; if (result.body.error) {
return record; throw Object.assign(new Error('error getting device list'), result.body.error);
} }
});
});
if (!record) { var dev = result.body.devices.filter(function (dev) {
console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address."); return dev.name === conf.device.hostname;
console.info("Use --ddns to allow the people of the Internet to access your server."); })[0];
return (dev || {}).addresses || [];
} }
});
var publicAddress;
async function recheckPubAddr() {
if (!conf.ddns.enabled) {
return;
}
var session = await getSession();
var directives = await OAUTH3.discover(session.token.aud);
var addr = await deps.loopback.checkPublicAddr(directives.api);
if (publicAddress === addr) {
return;
}
if (conf.debug) {
console.log('previous public address',publicAddress, 'does not match current public address', addr);
}
await setDeviceAddress(addr);
publicAddress = addr;
}
recheckPubAddr();
setInterval(recheckPubAddr, 5*60*1000);
return {
setDeviceAddress: setDeviceAddress
, getDeviceAddresses: getDeviceAddresses
, recheckPubAddr: recheckPubAddr
};
}; };
if (require.main === module) {
var opts = {
_old_server_name: 'aj.daplie.me'
, PromiseA: require('bluebird')
};
// ifaces
opts.ifaces = require('./local-ip.js').find();
console.log('opts.ifaces');
console.log(opts.ifaces);
require('./match-ips.js').match(opts._old_server_name, opts).then(function (ips) {
opts.matchingIps = ips.matchingIps || [];
opts.externalIps = ips.externalIps;
module.exports.create(opts);
});
}

View File

@ -50,7 +50,7 @@ module.exports.create = function (deps, config) {
} }
console.warn('failed to identify protocol from first chunk', firstChunk); console.warn('failed to identify protocol from first chunk', firstChunk);
conn.close(); conn.destroy();
} }
function netHandler(conn, opts) { function netHandler(conn, opts) {
function getProp(name) { function getProp(name) {

View File

@ -1,84 +1,88 @@
'use strict'; 'use strict';
module.exports.create = function (deps) { module.exports.create = function (deps, conf) {
var PromiseA = require('bluebird'); var PromiseA = require('bluebird');
var request = PromiseA.promisify(require('request')); var request = PromiseA.promisify(require('request'));
var pending = {}; var pending = {};
function checkPublicAddr(host) { async function checkPublicAddr(host) {
return request({ var result = await request({
method: 'GET' method: 'GET'
, url: host+'/api/org.oauth3.tunnel/checkip' , url: host+'/api/org.oauth3.tunnel/checkip'
, json: true , json: true
}).then(function (result) { });
if (!result.body) { if (!result.body) {
return PromiseA.reject(new Error('No response body in request for public address')); throw new Error('No response body in request for public address');
} }
if (result.body.error) { if (result.body.error) {
var err = new Error(result.body.error.message); // Note that the error on the body will probably have a message that overwrites the default
return PromiseA.reject(Object.assign(err, result.body.error)); throw Object.assign(new Error('error in check IP response'), result.body.error);
} }
return result.body.address; return result.body.address;
});
} }
function checkSinglePort(host, address, port) { async function checkSinglePort(host, address, port) {
var crypto = require('crypto'); var crypto = require('crypto');
var token = crypto.randomBytes(8).toString('hex'); var token = crypto.randomBytes(8).toString('hex');
var keyAuth = crypto.randomBytes(32).toString('hex'); var keyAuth = crypto.randomBytes(32).toString('hex');
pending[token] = keyAuth; pending[token] = keyAuth;
var opts = { var reqObj = {
method: 'POST'
, url: host+'/api/org.oauth3.tunnel/loopback'
, json: {
address: address address: address
, port: port , port: port
, token: token , token: token
, keyAuthorization: keyAuth , keyAuthorization: keyAuth
, iat: Date.now() , iat: Date.now()
}
}; };
return request({ var result;
method: 'POST' try {
, url: host+'/api/org.oauth3.tunnel/loopback' result = await request(reqObj);
, json: opts } catch (err) {
}) delete pending[token];
.then(function (result) { throw err;
}
delete pending[token]; delete pending[token];
if (!result.body) { if (!result.body) {
return PromiseA.reject(new Error('No response body in loopback request for port '+port)); throw new Error('No response body in loopback request for port '+port);
} }
// If the loopback requests don't go to us then there are all kinds of ways it could // If the loopback requests don't go to us then there are all kinds of ways it could
// error, but none of them really provide much extra information so we don't do // error, but none of them really provide much extra information so we don't do
// anything that will break the PromiseA.all out and mask the other results. // anything that will break the PromiseA.all out and mask the other results.
if (result.body.error) { if (conf.debug && result.body.error) {
console.log('error on remote side of port '+port+' loopback', result.body.error); console.log('error on remote side of port '+port+' loopback', result.body.error);
} }
return !!result.body.success; return !!result.body.success;
}, function (err) {
delete pending[token];
throw err;
});
} }
function loopback(provider) { async function loopback(provider) {
return deps.OAUTH3.discover(provider).then(function (directives) { var directives = await deps.OAUTH3.discover(provider);
return checkPublicAddr(directives.api).then(function (address) { var address = await checkPublicAddr(directives.api);
console.log('checking to see if', address, 'gets back to us'); console.log('checking to see if', address, 'gets back to us');
var ports = require('./servers').listeners.tcp.list(); var ports = require('./servers').listeners.tcp.list();
return PromiseA.all(ports.map(function (port) { var values = await PromiseA.all(ports.map(function (port) {
return checkSinglePort(directives.api, address, port); return checkSinglePort(directives.api, address, port);
})) }));
.then(function (values) {
console.log(pending); if (conf.debug) {
console.log('remaining loopback tokens', pending);
}
var result = {error: null, address: address}; var result = {error: null, address: address};
ports.forEach(function (port, ind) { ports.forEach(function (port, ind) {
result[port] = values[ind]; result[port] = values[ind];
}); });
return result; return result;
});
});
});
} }
loopback.checkPublicAddr = checkPublicAddr;
loopback.server = require('http').createServer(function (req, res) { loopback.server = require('http').createServer(function (req, res) {
var parsed = require('url').parse(req.url); var parsed = require('url').parse(req.url);
var token = parsed.pathname.replace('/.well-known/cloud-challenge/', ''); var token = parsed.pathname.replace('/.well-known/cloud-challenge/', '');

View File

@ -1,117 +0,0 @@
'use strict';
var PromiseA = require('bluebird');
module.exports.match = function (servername, opts) {
return PromiseA.promisify(require('ipify'))().then(function (externalIp) {
var dns = PromiseA.promisifyAll(require('dns'));
opts.externalIps = [ { address: externalIp, family: 'IPv4' } ];
opts.ifaces = require('./local-ip.js').find({ externals: opts.externalIps });
opts.externalIfaces = Object.keys(opts.ifaces).reduce(function (all, iname) {
var iface = opts.ifaces[iname];
iface.ipv4.forEach(function (addr) {
if (addr.external) {
addr.iface = iname;
all.push(addr);
}
});
iface.ipv6.forEach(function (addr) {
if (addr.external) {
addr.iface = iname;
all.push(addr);
}
});
return all;
}, []).filter(Boolean);
function resolveIps(hostname) {
var allIps = [];
return PromiseA.all([
dns.resolve4Async(hostname).then(function (records) {
records.forEach(function (ip) {
allIps.push({
address: ip
, family: 'IPv4'
});
});
}, function () {})
, dns.resolve6Async(hostname).then(function (records) {
records.forEach(function (ip) {
allIps.push({
address: ip
, family: 'IPv6'
});
});
}, function () {})
]).then(function () {
return allIps;
});
}
function resolveIpsAndCnames(hostname) {
return PromiseA.all([
resolveIps(hostname)
, dns.resolveCnameAsync(hostname).then(function (records) {
return PromiseA.all(records.map(function (hostname) {
return resolveIps(hostname);
})).then(function (allIps) {
return allIps.reduce(function (all, ips) {
return all.concat(ips);
}, []);
});
}, function () {
return [];
})
]).then(function (ips) {
return ips.reduce(function (all, set) {
return all.concat(set);
}, []);
});
}
return resolveIpsAndCnames(servername).then(function (allIps) {
var matchingIps = [];
if (!allIps.length) {
console.warn("Could not resolve '" + servername + "'");
}
// { address, family }
allIps.some(function (ip) {
function match(addr) {
if (ip.address === addr.address) {
matchingIps.push(addr);
}
}
opts.externalIps.forEach(match);
// opts.externalIfaces.forEach(match);
Object.keys(opts.ifaces).forEach(function (iname) {
var iface = opts.ifaces[iname];
iface.ipv4.forEach(match);
iface.ipv6.forEach(match);
});
return matchingIps.length;
});
matchingIps.externalIps = {
ipv4: [
{ address: externalIp
, family: 'IPv4'
}
]
, ipv6: [
]
};
matchingIps.matchingIps = matchingIps;
return matchingIps;
});
});
};

View File

@ -29,12 +29,14 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
function handleChunk(chunk) { function handleChunk(chunk) {
if (!errored) { if (!errored) {
opts.firstChunk = Buffer.concat([opts.firstChunk, chunk]); opts.firstChunk = Buffer.concat([opts.firstChunk, chunk]);
if (opts.firstChunk.includes('\r\n\r\n')) { if (!opts.firstChunk.includes('\r\n\r\n')) {
resolve(opts.firstChunk.toString());
conn.removeListener('error', handleErr);
} else {
conn.once('data', handleChunk); conn.once('data', handleChunk);
return;
} }
conn.removeListener('error', handleErr);
conn.pause();
resolve(opts.firstChunk.toString());
} }
} }
conn.once('data', handleChunk); conn.once('data', handleChunk);
@ -144,6 +146,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
// data comes in. // data comes in.
process.nextTick(function () { process.nextTick(function () {
conn.unshift(opts.firstChunk); conn.unshift(opts.firstChunk);
conn.resume();
}); });
// Convenience return for all the check* functions. // Convenience return for all the check* functions.
@ -160,6 +163,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
deps.tunnelServer.handleClientConn(conn); deps.tunnelServer.handleClientConn(conn);
process.nextTick(function () { process.nextTick(function () {
conn.unshift(opts.firstChunk); conn.unshift(opts.firstChunk);
conn.resume();
}); });
return true; return true;
} }
@ -211,6 +215,7 @@ module.exports.create = function (deps, conf, greenlockMiddleware) {
deps.tunnelServer.handleAdminConn(conn); deps.tunnelServer.handleAdminConn(conn);
process.nextTick(function () { process.nextTick(function () {
conn.unshift(opts.firstChunk); conn.unshift(opts.firstChunk);
conn.resume();
}); });
return true; return true;
} }

View File

@ -22,6 +22,7 @@ function create(conf) {
config = conf; config = conf;
var deps = { var deps = {
messenger: process messenger: process
, PromiseA: require('bluebird')
// Note that if a custom createConnections is used it will be called with different // Note that if a custom createConnections is used it will be called with different
// sets of custom options based on what is actually being proxied. Most notably the // sets of custom options based on what is actually being proxied. Most notably the
// HTTP proxying connection creation is not something we currently control. // HTTP proxying connection creation is not something we currently control.
@ -31,6 +32,7 @@ function create(conf) {
deps.proxy = require('./proxy-conn').create(deps, conf); deps.proxy = require('./proxy-conn').create(deps, conf);
deps.socks5 = require('./socks5-server').create(deps, conf); deps.socks5 = require('./socks5-server').create(deps, conf);
deps.loopback = require('./loopback').create(deps, conf); deps.loopback = require('./loopback').create(deps, conf);
deps.ddns = require('./ddns').create(deps, conf);
require('./goldilocks.js').create(deps, conf); require('./goldilocks.js').create(deps, conf);
process.removeListener('message', create); process.removeListener('message', create);

View File

@ -48,7 +48,6 @@
"http-proxy": "^1.16.2", "http-proxy": "^1.16.2",
"human-readable-ids": "git+https://git.daplie.com/Daplie/human-readable-ids-js#master", "human-readable-ids": "git+https://git.daplie.com/Daplie/human-readable-ids-js#master",
"ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0", "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0",
"ipify": "^1.1.0",
"js-yaml": "^3.8.3", "js-yaml": "^3.8.3",
"jsonwebtoken": "^7.4.0", "jsonwebtoken": "^7.4.0",
"le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master", "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master",