From 513e6e8bddb698f5718d52587fbe2dd70378647c Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 8 May 2017 16:52:37 -0600 Subject: [PATCH 01/13] implemented forwarding of TCP based on incoming port --- bin/goldilocks.js | 8 ++--- lib/goldilocks.js | 74 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 43f6627..d11f1a1 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -19,7 +19,7 @@ function run(config) { worker.send(config); }); } - console.log('config.tcp.ports', config.tcp.ports); + console.log('config.tcp.bind', config.tcp.bind); work(); } @@ -123,13 +123,13 @@ function readConfigAndRun(args) { var PromiseA = require('bluebird'); var tcpProm, dnsProm; - if (config.tcp.ports) { + if (config.tcp.bind) { tcpProm = PromiseA.resolve(); } else { tcpProm = new PromiseA(function (resolve, reject) { require('../lib/check-ports').checkTcpPorts(function (failed, bound) { - config.tcp.ports = Object.keys(bound); - if (config.tcp.ports.length) { + config.tcp.bind = Object.keys(bound); + if (config.tcp.bind.length) { resolve(); } else { reject(failed); diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 6264845..bc5e6f1 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -242,6 +242,26 @@ module.exports.create = function (deps, config) { socket.send(msg, config.dns.proxy.port, config.dns.proxy.address || '127.0.0.1'); } + function createTcpForwarder(mod) { + var destination = mod.address.split(':'); + + return function (conn) { + var newConn = deps.net.createConnection({ + port: destination[1] + , host: destination[0] || '127.0.0.1' + + , remoteFamily: conn.remoteFamily + , remoteAddress: conn.remoteAddress + , remotePort: conn.remotePort + }, function () { + + }); + + newConn.pipe(conn); + conn.pipe(newConn); + }; + } + function approveDomains(opts, certs, cb) { // This is where you check your database and associated // email addresses with domains and agreements and such @@ -454,15 +474,59 @@ module.exports.create = function (deps, config) { }); }); - var listenPromises = config.tcp.ports.map(function (port) { - return listeners.tcp.add(port, netHandler); + var listenPromises = []; + var tcpPortMap = {}; + + if (config.tcp.bind) { + config.tcp.bind.forEach(function (port) { + tcpPortMap[port] = true; + }); + } + config.tcp.modules.forEach(function (mod) { + if (mod.name === 'forward') { + var forwarder = createTcpForwarder(mod); + mod.ports.forEach(function (port) { + if (!tcpPortMap[port]) { + console.log("forwarding port", port, "that wasn't specified in bind"); + } else { + delete tcpPortMap[port]; + } + listenPromises.push(listeners.tcp.add(port, forwarder)); + }); + } + else { + console.warn('unknown TCP module specified', mod); + } + }); + + // Even though these ports were specified in different places we treat any TCP + // connections we haven't been told to just forward exactly as is equal so that + // we can potentially use the same ports for different protocols. + function addPorts(bindList) { + if (!bindList) { + return; + } + if (Array.isArray(bindList)) { + bindList.forEach(function (port) { + tcpPortMap[port] = true; + }); + } + else { + tcpPortMap[bindList] = true; + } + } + addPorts(config.tls.bind); + addPorts(config.http.bind); + + Object.keys(tcpPortMap).forEach(function (port) { + listenPromises.push(listeners.tcp.add(port, netHandler)); }); if (config.dns.bind) { if (Array.isArray(config.dns.bind)) { - listenPromises = listenPromises.concat(config.dns.bind.map(function (port) { - return listeners.udp.add(port, dnsListener); - })); + config.dns.bind.map(function (port) { + listenPromises.push(listeners.udp.add(port, dnsListener)); + }); } else { listenPromises.push(listeners.udp.add(config.dns.bind, dnsListener)); } From f32db19b520e66fda036daddcf0cf664010222ac Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 8 May 2017 17:47:51 -0600 Subject: [PATCH 02/13] handled case where no TCP modules exist --- lib/goldilocks.js | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/goldilocks.js b/lib/goldilocks.js index bc5e6f1..d92d8d9 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -476,13 +476,22 @@ module.exports.create = function (deps, config) { var listenPromises = []; var tcpPortMap = {}; - - if (config.tcp.bind) { - config.tcp.bind.forEach(function (port) { - tcpPortMap[port] = true; - }); + function addPorts(bindList) { + if (!bindList) { + return; + } + if (Array.isArray(bindList)) { + bindList.forEach(function (port) { + tcpPortMap[port] = true; + }); + } + else { + tcpPortMap[bindList] = true; + } } - config.tcp.modules.forEach(function (mod) { + + addPorts(config.tcp.bind); + (config.tcp.modules || []).forEach(function (mod) { if (mod.name === 'forward') { var forwarder = createTcpForwarder(mod); mod.ports.forEach(function (port) { @@ -502,19 +511,6 @@ module.exports.create = function (deps, config) { // Even though these ports were specified in different places we treat any TCP // connections we haven't been told to just forward exactly as is equal so that // we can potentially use the same ports for different protocols. - function addPorts(bindList) { - if (!bindList) { - return; - } - if (Array.isArray(bindList)) { - bindList.forEach(function (port) { - tcpPortMap[port] = true; - }); - } - else { - tcpPortMap[bindList] = true; - } - } addPorts(config.tls.bind); addPorts(config.http.bind); From 99a3de649615330a553ac23bd1a2f09503bb58e9 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Mon, 8 May 2017 17:59:45 -0600 Subject: [PATCH 03/13] implemented ability to proxy TLS based on SNI --- lib/goldilocks.js | 161 ++++++++++++++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 55 deletions(-) diff --git a/lib/goldilocks.js b/lib/goldilocks.js index d92d8d9..0c2248f 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -20,6 +20,22 @@ module.exports.create = function (deps, config) { var tunnelAdminTlsOpts = {}; var tls = require('tls'); + function domainMatches(pattern, servername) { + // Everything matches '*' + if (pattern === '*') { + return true; + } + + if (/^\*./.test(pattern)) { + // get rid of the leading "*." to more easily check the servername against it + pattern = pattern.slice(2); + return pattern === servername.slice(-pattern.length); + } + + // pattern doesn't contains any wildcards, so exact match is required + return pattern === servername; + } + var tcpRouter = { _map: { } , _create: function (address, port) { @@ -123,68 +139,103 @@ module.exports.create = function (deps, config) { } }; var tlsRouter = { - _map: { } - , _create: function (address, port/*, nextServer*/) { - // port provides hinting for https, smtps, etc - return function (socket, firstChunk, opts) { - if (opts.hyperPeek) { - // See "PEEK COMMENT" for more info - // This was peeked at properly, so we don't have to re-wrap it - // in order to get the connection to not hang. - // The real first 'data' and 'readable' events will occur as they should - program.tlsTunnelServer.emit('connection', socket); - return; + proxy: function (socket, opts, mod) { + var newConn = deps.net.createConnection({ + port: mod.port + , host: mod.address || '127.0.0.1' + + , servername: opts.servername + , data: opts.data + , remoteFamily: opts.family || socket.remoteFamily || socket._remoteFamily || socket._handle._parent.owner.stream.remoteFamily + , remoteAddress: opts.address || socket.remoteAddress || socket._remoteAddress || socket._handle._parent.owner.stream.remoteAddress + , remotePort: opts.port || socket.remotePort || socket._remotePort || socket._handle._parent.owner.stream.remotePort + }, function () { + // this will happen before 'data' is triggered + }); + + newConn.pipe(socket); + socket.pipe(newConn); + } + , terminate: function (socket) { + // We terminate the TLS by emitting the connections of the TLS server and it should handle + // everything we need to do for us. + program.tlsTunnelServer.emit('connection', socket); + } + + , handleModules: function (socket, opts) { + // needs to wind up in one of 2 states: + // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket) + // 2. Terminated (goes on to a particular module or route, including the admin interface) + + var handled = (config.tls.modules || []).some(function (mod) { + var relevant = mod.domains.some(function (pattern) { + return domainMatches(pattern, opts.servername); + }); + if (!relevant) { + return false; } - var servername = opts.servername; - var packerStream = require('tunnel-packer').Stream; - var myDuplex = packerStream.create(socket); + if (mod.name === 'proxy') { + tlsRouter.proxy(socket, opts, mod); + } + else if (mod.name === 'terminate') { + tlsRouter.terminate(socket); + } + else { + console.error('saw unknown TLS module', mod); + return false; + } - myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; - myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; - console.log('[tlsRouter] ' + address + ':' + port + ' servername', servername, myDuplex.remoteAddress); + return true; + }); - // needs to wind up in one of 3 states: - // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket) - // 2. Admin Interface (skips the proxying) - // 3. Terminated (goes on to a particular module or route) - //myDuplex.__tlsTerminated = true; - - process.nextTick(function () { - // this must happen after the socket is emitted to the next in the chain, - // but before any more data comes in via the network - socket.unshift(firstChunk); - }); - - // nextServer.emit could be used here - program.tlsTunnelServer.emit('connection', myDuplex); - - // Why all this wacky-do with the myDuplex? - // because https://github.com/nodejs/node/issues/8854, that's why - // (because node's internal networking layer == 💩 sometimes) - socket.on('data', function (chunk) { - console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); - myDuplex.push(chunk); - }); - socket.on('error', function (err) { - console.error('[error] httpsTunnel (Admin) TODO close'); - console.error(err); - myDuplex.emit('error', err); - }); - socket.on('close', function () { - myDuplex.end(); - }); - }; + // We gotta do something, so when in doubt terminate the TLS since we don't really have + // any good place to default to when proxying. + if (!handled) { + tlsRouter.terminate(socket); + } } - , get: function getTcpRouter(address, port) { - address = address || '0.0.0.0'; - var id = address + ':' + port; - if (!tlsRouter._map[id]) { - tlsRouter._map[id] = tlsRouter._create(address, port); + , processSocket: function (socket, firstChunk, opts) { + if (opts.hyperPeek) { + // See "PEEK COMMENT" for more info + // This was already peeked at by the tunneler and this connection has been created + // in a way that should work with node's TLS server, so we don't need to do any + // of the myDuplex stuff that we need to do with non-tunnel connections. + tlsRouter.handleModules(socket, opts); + return; } - return tlsRouter._map[id]; + // Why all this wacky-do with the myDuplex? + // because https://github.com/nodejs/node/issues/8854, that's why + // (because node's internal networking layer == 💩 sometimes) + var myDuplex = require('tunnel-packer').Stream.create(socket); + myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; + myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; + + socket.on('data', function (chunk) { + console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); + myDuplex.push(chunk); + }); + socket.on('error', function (err) { + console.error('[error] httpsTunnel (Admin) TODO close'); + console.error(err); + myDuplex.emit('error', err); + }); + socket.on('close', function () { + myDuplex.end(); + }); + + var address = opts.localAddress || socket.localAddress; + var port = opts.localPort || socket.localPort; + console.log('[tlsRouter] ' + address + ':' + port + ' servername', opts.servername, myDuplex.remoteAddress); + + tlsRouter.handleModules(myDuplex, opts); + process.nextTick(function () { + // this must happen after the socket is emitted to the next in the chain, + // but before any more data comes in via the network + socket.unshift(firstChunk); + }); } }; @@ -197,7 +248,7 @@ module.exports.create = function (deps, config) { if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) { console.log('tryTls'); opts.servername = (parseSni(firstChunk)||'').toLowerCase() || 'localhost.invalid'; - tlsRouter.get(opts.localAddress || conn.localAddress, conn.localPort)(conn, firstChunk, opts); + tlsRouter.processSocket(conn, firstChunk, opts); return; } From ab31bae6ffe4ecd5c0bc4a20869363eaf21d2419 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 9 May 2017 14:16:21 -0600 Subject: [PATCH 04/13] implemented more dynamic HTTP proxying --- lib/goldilocks.js | 17 +-------- lib/match-domain.js | 17 +++++++++ lib/modules/http.js | 84 +++++++++++++++++++++++++++++++-------------- package.json | 1 + 4 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 lib/match-domain.js diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 0c2248f..013a4cb 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -19,22 +19,7 @@ module.exports.create = function (deps, config) { var secureContexts = {}; var tunnelAdminTlsOpts = {}; var tls = require('tls'); - - function domainMatches(pattern, servername) { - // Everything matches '*' - if (pattern === '*') { - return true; - } - - if (/^\*./.test(pattern)) { - // get rid of the leading "*." to more easily check the servername against it - pattern = pattern.slice(2); - return pattern === servername.slice(-pattern.length); - } - - // pattern doesn't contains any wildcards, so exact match is required - return pattern === servername; - } + var domainMatches = require('./match-domain').match; var tcpRouter = { _map: { } diff --git a/lib/match-domain.js b/lib/match-domain.js new file mode 100644 index 0000000..e4a8a1d --- /dev/null +++ b/lib/match-domain.js @@ -0,0 +1,17 @@ +'use strict'; + +module.exports.match = function (pattern, servername) { + // Everything matches '*' + if (pattern === '*') { + return true; + } + + if (/^\*./.test(pattern)) { + // get rid of the leading "*." to more easily check the servername against it + pattern = pattern.slice(2); + return pattern === servername.slice(-pattern.length); + } + + // pattern doesn't contains any wildcards, so exact match is required + return pattern === servername; +}; diff --git a/lib/modules/http.js b/lib/modules/http.js index 3631cd5..1b8207e 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -1,34 +1,66 @@ 'use strict'; module.exports.create = function (deps, conf) { - // This should be able to handle things like default web path (i.e. /srv/www/hostname), - // no-www redirect, and transpilation of static assets (i.e. cached versions of raw html) - // but right now it's a very dumb proxy + var app = require('express')(); + var domainMatches = require('../match-domain').match; - function createConnection(conn) { - var opts = conn.__opts; - var newConn = deps.net.createConnection({ - port: conf.http.proxy.port - , host: '127.0.0.1' - - , servername: opts.servername - , data: opts.data - , remoteFamily: opts.family || conn.remoteFamily - , remoteAddress: opts.address || conn.remoteAddress - , remotePort: opts.port || conn.remotePort - }, function () { - //console.log("[=>] first packet from tunneler to '" + cid + "' as '" + opts.service + "'", opts.data.byteLength); - // this will happen before 'data' is triggered - //newConn.write(opts.data); - }); - - newConn.pipe(conn); - conn.pipe(newConn); + // We handle both HTTPS and HTTP traffic on the same ports, and we want to redirect + // any unencrypted requests to the same port they came from unless it came in on + // the default HTTP port, in which case there wont be a port specified in the host. + var redirecters = {}; + function redirectHttps(req, res, next) { + var port = req.headers.host.split(':')[1]; + var redirecter = redirecters[port]; + if (!redirecter) { + redirecter = redirecters[port] = require('redirect-https')({port: port}); + } + redirecter(req, res, next); } - return { - emit: function (type, conn) { - createConnection(conn); + function respond404(req, res) { + res.writeHead(404); + res.end('Not Found'); + } + + function createProxyRoute(mod) { + // This is the easiest way to override the createConnections function the proxy + // module uses, but take note the since we don't have control over where this is + // called the extra options availabled will be different. + var agent = new require('http').Agent({}); + agent.createConnection = deps.net.createConnection; + + var proxy = require('http-proxy').createProxyServer({ + agent: agent + , target: 'http://' + mod.address + , xfwd: true + , toProxy: true + }); + + return function (req, res, next) { + var hostname = req.headers.host.split(':')[0]; + var relevant = mod.domains.some(function (pattern) { + return domainMatches(pattern, hostname); + }); + + if (relevant) { + proxy.web(req, res); + } else { + next(); + } + }; + } + + app.use(redirectHttps); + + (conf.http.modules || []).forEach(function (mod) { + if (mod.name === 'proxy') { + app.use(createProxyRoute(mod)); } - }; + else { + console.warn('unknown HTTP module', mod); + } + }); + + app.use(respond404); + return require('http').createServer(app); }; diff --git a/package.json b/package.json index 197b0d4..6202868 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "finalhandler": "^0.4.0", "greenlock": "git+https://git.daplie.com/Daplie/node-greenlock.git#master", "greenlock-express": "git+https://git.daplie.com/Daplie/greenlock-express.git#master", + "http-proxy": "^1.16.2", "httpolyglot": "^0.1.1", "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0", "ipify": "^1.1.0", From ab011d1829492c2bef8046688567ddedc591e667 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 9 May 2017 15:46:49 -0600 Subject: [PATCH 05/13] cleaned up all of the custom HTTP handling logic --- lib/goldilocks.js | 135 +++++++------------------------------------ lib/modules/admin.js | 4 +- lib/modules/http.js | 29 ++++++++++ 3 files changed, 52 insertions(+), 116 deletions(-) diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 013a4cb..78905a2 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -21,108 +21,6 @@ module.exports.create = function (deps, config) { var tls = require('tls'); var domainMatches = require('./match-domain').match; - var tcpRouter = { - _map: { } - , _create: function (address, port) { - // port provides hinting for http, smtp, etc - return function (conn, firstChunk, opts) { - console.log('[tcpRouter] ' + address + ':' + port + ' ' + (opts.servername || '')); - - var m; - var str; - var hostname; - var newHeads; - - // TODO test per-module - // Maybe HTTP - if (firstChunk[0] > 32 && firstChunk[0] < 127) { - str = firstChunk.toString(); - m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im); - hostname = (m && m[1].toLowerCase() || '').split(':')[0]; - console.log('[tcpRouter] hostname', hostname); - if (/HTTP\//i.test(str)) { - //conn.__service = 'http'; - } - } - - if (!hostname) { - // TODO allow tcp tunneling - // TODO we need some way of tagging tcp as either terminated tls or insecure - conn.write( - "HTTP/1.1 404 Not Found\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/html\r\n" - + "Content-Length: " + 9 + "\r\n" - + "\r\n" - + "Not Found" - ); - conn.end(); - return; - } - - - // Poor-man's http proxy - // XXX SECURITY XXX: should strip existing X-Forwarded headers - newHeads = - [ "X-Forwarded-Proto: " + (opts.encrypted ? 'https' : 'http') - , "X-Forwarded-For: " + (opts.remoteAddress || conn.remoteAddress) - , "X-Forwarded-Host: " + hostname - ]; - - if (!opts.encrypted) { - // a exists-only header that a bad client could not remove - newHeads.push("X-Not-Encrypted: yes"); - } - if (opts.servername) { - newHeads.push("X-Forwarded-Sni: " + opts.servername); - if (opts.servername !== hostname) { - // an exists-only header that a bad client could not remove - newHeads.push("X-Two-Servernames: yes"); - } - } - - firstChunk = firstChunk.toString('utf8'); - // JSON.stringify("Host: example.com\r\nNext: Header".replace(/(Host: [^\r\n]*)/i, "$1" + "\r\n" + "X: XYZ")) - firstChunk = firstChunk.replace(/(Host: [^\r\n]*)/i, "$1" + "\r\n" + newHeads.join("\r\n")); - - process.nextTick(function () { - conn.unshift(Buffer.from(firstChunk, 'utf8')); - }); - - // - // hard-coded routes for the admin interface - if ( - /\blocalhost\.admin\./.test(hostname) || /\badmin\.localhost\./.test(hostname) - || /\blocalhost\.alpha\./.test(hostname) || /\balpha\.localhost\./.test(hostname) - ) { - if (!modules.admin) { - modules.admin = require('./modules/admin.js').create(deps, config); - } - modules.admin.emit('connection', conn); - return; - } - - // TODO static file handiling and such or whatever - if (!modules.http) { - modules.http = require('./modules/http.js').create(deps, config); - } - opts.hostname = hostname; - conn.__opts = opts; - - modules.http.emit('connection', conn); - }; - } - , get: function getTcpRouter(address, port) { - address = address || '0.0.0.0'; - - var id = address + ':' + port; - if (!tcpRouter._map[id]) { - tcpRouter._map[id] = tcpRouter._create(address, port); - } - - return tcpRouter._map[id]; - } - }; var tlsRouter = { proxy: function (socket, opts, mod) { var newConn = deps.net.createConnection({ @@ -231,25 +129,36 @@ module.exports.create = function (deps, config) { // TLS byte 1 is handshake and byte 6 is client hello if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) { - console.log('tryTls'); opts.servername = (parseSni(firstChunk)||'').toLowerCase() || 'localhost.invalid'; tlsRouter.processSocket(conn, firstChunk, opts); return; } - console.log('tryTcp'); - - if (opts.hyperPeek) { - // even though we've already peeked, this logic is just as well to let be - // since it works properly either way, unlike the tls socket - conn.once('data', function (chunk) { - console.log('hyperPeek re-peek data', chunk.toString('utf8')); - tcpRouter.get(opts.localAddress || conn.localAddress, conn.localPort)(conn, chunk, opts); + // This doesn't work with TLS, but now that we know this isn't a TLS connection we can + // unshift the first chunk back onto the connection for future use. The unshift should + // happen after any listeners are attached to it but before any new data comes in. + if (!opts.hyperPeek) { + process.nextTick(function () { + conn.unshift(firstChunk); }); - return; } - tcpRouter.get(opts.localAddress || conn.localAddress, conn.localPort)(conn, firstChunk, opts); + // Connection is not TLS, check for HTTP next. + if (firstChunk[0] > 32 && firstChunk[0] < 127) { + var firstStr = firstChunk.toString(); + if (/HTTP\//i.test(firstStr)) { + if (!modules.http) { + modules.http = require('./modules/http.js').create(deps, config); + } + + conn.__opts = opts; + modules.http.emit('connection', conn); + return; + } + } + + console.warn('failed to identify protocol from first chunk', firstChunk); + conn.close(); } function netHandler(conn, opts) { opts = opts || {}; diff --git a/lib/modules/admin.js b/lib/modules/admin.js index acde6a2..7f5a81a 100644 --- a/lib/modules/admin.js +++ b/lib/modules/admin.js @@ -60,7 +60,5 @@ module.exports.create = function (deps, conf) { }); /* device, addresses, cwd, http */ - var app = require('../app.js')(deps, conf, opts); - var http = require('http'); - return http.createServer(app); + return require('../app.js')(deps, conf, opts); }; diff --git a/lib/modules/http.js b/lib/modules/http.js index 1b8207e..307cda6 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -2,8 +2,16 @@ module.exports.create = function (deps, conf) { var app = require('express')(); + var adminApp = require('./admin').create(deps, conf); var domainMatches = require('../match-domain').match; + var adminDomains = [ + /\blocalhost\.admin\./ + , /\blocalhost\.alpha\./ + , /\badmin\.localhost\./ + , /\balpha\.localhost\./ + ]; + // We handle both HTTPS and HTTP traffic on the same ports, and we want to redirect // any unencrypted requests to the same port they came from unless it came in on // the default HTTP port, in which case there wont be a port specified in the host. @@ -17,6 +25,18 @@ module.exports.create = function (deps, conf) { redirecter(req, res, next); } + function handleAdmin(req, res, next) { + var admin = adminDomains.some(function (re) { + return re.test(req.headers.host); + }); + + if (admin) { + adminApp(req, res); + } else { + next(); + } + } + function respond404(req, res) { res.writeHead(404); res.end('Not Found'); @@ -36,6 +56,14 @@ module.exports.create = function (deps, conf) { , toProxy: true }); + // We want to override the default value for some headers with the extra information we + // have available to us in the opts object attached to the connection. + proxy.on('proxyReq', function (proxyReq, req) { + var conn = req.connection; + var opts = conn.__opts; + proxyReq.setHeader('X-Forwarded-For', opts.remoteAddress || conn.remoteAddress); + }); + return function (req, res, next) { var hostname = req.headers.host.split(':')[0]; var relevant = mod.domains.some(function (pattern) { @@ -51,6 +79,7 @@ module.exports.create = function (deps, conf) { } app.use(redirectHttps); + app.use(handleAdmin); (conf.http.modules || []).forEach(function (mod) { if (mod.name === 'proxy') { From bcba0abddc2a26326365c9517ff127bbd992c109 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 9 May 2017 16:23:30 -0600 Subject: [PATCH 06/13] added error handling when HTTP proxy doesn't connect --- lib/modules/http.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/modules/http.js b/lib/modules/http.js index 307cda6..2277903 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -64,6 +64,18 @@ module.exports.create = function (deps, conf) { proxyReq.setHeader('X-Forwarded-For', opts.remoteAddress || conn.remoteAddress); }); + proxy.on('error', function (err, req, res) { + console.log(err); + res.writeHead(502); + if (err.code === 'ECONNREFUSED') { + res.end('The connection was refused. Most likely the service being connected to ' + + 'has stopped running or the configuration is wrong.'); + } + else { + res.end('Bad Gateway: ' + err.code); + } + }); + return function (req, res, next) { var hostname = req.headers.host.split(':')[0]; var relevant = mod.domains.some(function (pattern) { From 56113cb100d2a274d3ee78f1f3b5c1999441af73 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Tue, 9 May 2017 16:50:07 -0600 Subject: [PATCH 07/13] implemented static file serving HTTP module --- lib/modules/http.js | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/modules/http.js b/lib/modules/http.js index 2277903..5c6247a 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -1,7 +1,8 @@ 'use strict'; module.exports.create = function (deps, conf) { - var app = require('express')(); + var express = require('express'); + var app = express(); var adminApp = require('./admin').create(deps, conf); var domainMatches = require('../match-domain').match; @@ -90,6 +91,38 @@ module.exports.create = function (deps, conf) { }; } + function createStaticRoute(mod) { + var getStaticApp, staticApp; + if (/:hostname/.test(mod.root)) { + staticApp = {}; + getStaticApp = function (hostname) { + if (!staticApp[hostname]) { + staticApp[hostname] = express.static(mod.root.replace(':hostname', hostname)); + } + return staticApp[hostname]; + }; + } + else { + staticApp = express.static(mod.root); + getStaticApp = function () { + return staticApp; + }; + } + + return function (req, res, next) { + var hostname = req.headers.host.split(':')[0]; + var relevant = mod.domains.some(function (pattern) { + return domainMatches(pattern, hostname); + }); + + if (relevant) { + getStaticApp(hostname)(req, res, next); + } else { + next(); + } + }; + } + app.use(redirectHttps); app.use(handleAdmin); @@ -97,6 +130,9 @@ module.exports.create = function (deps, conf) { if (mod.name === 'proxy') { app.use(createProxyRoute(mod)); } + else if (mod.name === 'static') { + app.use(createStaticRoute(mod)); + } else { console.warn('unknown HTTP module', mod); } From afca49feae516409e8a0f48771f7dd7c70ad5a16 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 10 May 2017 12:56:47 -0600 Subject: [PATCH 08/13] moved TLS handling into a separate file --- lib/goldilocks.js | 259 ++------------------------------------------- lib/modules/tls.js | 235 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 252 deletions(-) create mode 100644 lib/modules/tls.js diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 78905a2..d398741 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -5,132 +5,22 @@ module.exports.create = function (deps, config) { //var PromiseA = global.Promise; var PromiseA = require('bluebird'); - var greenlock = require('greenlock'); var listeners = require('./servers').listeners; - var parseSni = require('sni'); var modules = { }; - var program = { - tlsOptions: require('localhost.daplie.me-certificates').merge({}) -// , acmeDirectoryUrl: 'https://acme-v01.api.letsencrypt.org/directory' - , acmeDirectoryUrl: 'https://acme-staging.api.letsencrypt.org/directory' -// , challengeType: 'tls-sni-01' // won't work with a tunnel - , challengeType: 'http-01' - }; - var secureContexts = {}; - var tunnelAdminTlsOpts = {}; - var tls = require('tls'); - var domainMatches = require('./match-domain').match; - - var tlsRouter = { - proxy: function (socket, opts, mod) { - var newConn = deps.net.createConnection({ - port: mod.port - , host: mod.address || '127.0.0.1' - - , servername: opts.servername - , data: opts.data - , remoteFamily: opts.family || socket.remoteFamily || socket._remoteFamily || socket._handle._parent.owner.stream.remoteFamily - , remoteAddress: opts.address || socket.remoteAddress || socket._remoteAddress || socket._handle._parent.owner.stream.remoteAddress - , remotePort: opts.port || socket.remotePort || socket._remotePort || socket._handle._parent.owner.stream.remotePort - }, function () { - // this will happen before 'data' is triggered - }); - - newConn.pipe(socket); - socket.pipe(newConn); - } - , terminate: function (socket) { - // We terminate the TLS by emitting the connections of the TLS server and it should handle - // everything we need to do for us. - program.tlsTunnelServer.emit('connection', socket); - } - - , handleModules: function (socket, opts) { - // needs to wind up in one of 2 states: - // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket) - // 2. Terminated (goes on to a particular module or route, including the admin interface) - - var handled = (config.tls.modules || []).some(function (mod) { - var relevant = mod.domains.some(function (pattern) { - return domainMatches(pattern, opts.servername); - }); - if (!relevant) { - return false; - } - - if (mod.name === 'proxy') { - tlsRouter.proxy(socket, opts, mod); - } - else if (mod.name === 'terminate') { - tlsRouter.terminate(socket); - } - else { - console.error('saw unknown TLS module', mod); - return false; - } - - return true; - }); - - // We gotta do something, so when in doubt terminate the TLS since we don't really have - // any good place to default to when proxying. - if (!handled) { - tlsRouter.terminate(socket); - } - } - - , processSocket: function (socket, firstChunk, opts) { - if (opts.hyperPeek) { - // See "PEEK COMMENT" for more info - // This was already peeked at by the tunneler and this connection has been created - // in a way that should work with node's TLS server, so we don't need to do any - // of the myDuplex stuff that we need to do with non-tunnel connections. - tlsRouter.handleModules(socket, opts); - return; - } - - // Why all this wacky-do with the myDuplex? - // because https://github.com/nodejs/node/issues/8854, that's why - // (because node's internal networking layer == 💩 sometimes) - var myDuplex = require('tunnel-packer').Stream.create(socket); - myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; - myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; - - socket.on('data', function (chunk) { - console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); - myDuplex.push(chunk); - }); - socket.on('error', function (err) { - console.error('[error] httpsTunnel (Admin) TODO close'); - console.error(err); - myDuplex.emit('error', err); - }); - socket.on('close', function () { - myDuplex.end(); - }); - - var address = opts.localAddress || socket.localAddress; - var port = opts.localPort || socket.localPort; - console.log('[tlsRouter] ' + address + ':' + port + ' servername', opts.servername, myDuplex.remoteAddress); - - tlsRouter.handleModules(myDuplex, opts); - process.nextTick(function () { - // this must happen after the socket is emitted to the next in the chain, - // but before any more data comes in via the network - socket.unshift(firstChunk); - }); - } - }; - // opts = { servername, encrypted, peek, data, remoteAddress, remotePort } function peek(conn, firstChunk, opts) { + opts.firstChunk = firstChunk; + conn.__opts = opts; // TODO port/service-based routing can do here // TLS byte 1 is handshake and byte 6 is client hello if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) { - opts.servername = (parseSni(firstChunk)||'').toLowerCase() || 'localhost.invalid'; - tlsRouter.processSocket(conn, firstChunk, opts); + if (!modules.tls) { + modules.tls = require('./modules/tls').create(deps, config, netHandler); + } + + modules.tls.emit('connection', conn); return; } @@ -151,7 +41,6 @@ module.exports.create = function (deps, config) { modules.http = require('./modules/http.js').create(deps, config); } - conn.__opts = opts; modules.http.emit('connection', conn); return; } @@ -207,92 +96,6 @@ module.exports.create = function (deps, config) { }; } - function approveDomains(opts, certs, cb) { - // This is where you check your database and associated - // email addresses with domains and agreements and such - - // The domains being approved for the first time are listed in opts.domains - // Certs being renewed are listed in certs.altnames - - function complete(err, stuff) { - opts.email = stuff.email; - opts.agreeTos = stuff.agreeTos; - opts.server = stuff.server; - opts.challengeType = stuff.challengeType; - - cb(null, { options: opts, certs: certs }); - } - - if (certs) { - // TODO make sure the same options are used for renewal as for registration? - opts.domains = certs.altnames; - - cb(null, { options: opts, certs: certs }); - return; - } - - // check config for domain name - if (-1 !== config.tls.servernames.indexOf(opts.domain)) { - // TODO how to handle SANs? - // TODO fetch domain-specific email - // TODO fetch domain-specific acmeDirectory - // NOTE: you can also change other options such as `challengeType` and `challenge` - // opts.challengeType = 'http-01'; - // opts.challenge = require('le-challenge-fs').create({}); // TODO this doesn't actually work yet - complete(null, { - email: config.tls.email, agreeTos: true, server: program.acmeDirectoryUrl, challengeType: program.challengeType }); - return; - } - // TODO ask http module about the default path (/srv/www/:hostname) - // (if it exists, we can allow and add to config) - if (!modules.http) { - modules.http = require('./modules/http.js').create(deps, config); - } - modules.http.checkServername(opts.domain).then(function (stuff) { - if (!stuff || !stuff.domains) { - // TODO once precheck is implemented we can just let it pass if it passes, yknow? - cb(new Error('domain is not allowed')); - return; - } - - complete(null, { - domain: stuff.domain || stuff.domains[0] - , domains: stuff.domains - , email: stuff.email || program.email - , server: stuff.acmeDirectoryUrl || program.acmeDirectoryUrl - , challengeType: stuff.challengeType || program.challengeType - , challenge: stuff.challenge - }); - return; - }, cb); - } - - function getAcme() { - return greenlock.create({ - - //server: 'staging' - server: 'https://acme-v01.api.letsencrypt.org/directory' - - , challenges: { - // TODO dns-01 - 'http-01': require('le-challenge-fs').create({ webrootPath: '/tmp/acme-challenges', debug: config.debug }) - , 'tls-sni-01': require('le-challenge-sni').create({ debug: config.debug }) - //, 'dns-01': require('le-challenge-ddns').create() - } - - , store: require('le-store-certbot').create({ webrootPath: '/tmp/acme-challenges' }) - - //, email: program.email - - //, agreeTos: program.agreeTos - - , approveDomains: approveDomains - - //, approvedDomains: program.servernames - - }); - } - deps.tunnel = deps.tunnel || {}; deps.tunnel.net = { createConnection: function (opts, cb) { @@ -371,54 +174,6 @@ module.exports.create = function (deps, config) { } }; - Object.keys(program.tlsOptions).forEach(function (key) { - tunnelAdminTlsOpts[key] = program.tlsOptions[key]; - }); - tunnelAdminTlsOpts.SNICallback = function (sni, cb) { - console.log("[tlsOptions.SNICallback] SNI: '" + sni + "'"); - - var tlsOptions; - - // Static Certs - if (/.*localhost.*\.daplie\.me/.test(sni.toLowerCase())) { - // TODO implement - if (!secureContexts[sni]) { - tlsOptions = require('localhost.daplie.me-certificates').mergeTlsOptions(sni, {}); - } - if (tlsOptions) { - secureContexts[sni] = tls.createSecureContext(tlsOptions); - } - if (secureContexts[sni]) { - console.log('Got static secure context:', sni, secureContexts[sni]); - cb(null, secureContexts[sni]); - return; - } - } - - if (!program.greenlock) { - program.greenlock = getAcme(); - } - (program.greenlock.tlsOptions||program.greenlock.httpsOptions).SNICallback(sni, cb); - }; - - program.tlsTunnelServer = tls.createServer(tunnelAdminTlsOpts, function (tlsSocket) { - console.log('(pre-terminated) tls connection, addr:', tlsSocket.remoteAddress); - // things get a little messed up here - //tlsSocket.on('data', function (chunk) { - // console.log('terminated data:', chunk.toString()); - //}); - //(program.httpTunnelServer || program.httpServer).emit('connection', tlsSocket); - //tcpRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, { encrypted: false }); - netHandler(tlsSocket, { - servername: tlsSocket.servername - , encrypted: true - // remoteAddress... ugh... https://github.com/nodejs/node/issues/8854 - , remoteAddress: tlsSocket.remoteAddress || tlsSocket._remoteAddress || tlsSocket._handle._parent.owner.stream.remoteAddress - , remotePort: tlsSocket.remotePort || tlsSocket._remotePort || tlsSocket._handle._parent.owner.stream.remotePort - , remoteFamily: tlsSocket.remoteFamily || tlsSocket._remoteFamily || tlsSocket._handle._parent.owner.stream.remoteFamily - }); - }); - var listenPromises = []; var tcpPortMap = {}; function addPorts(bindList) { diff --git a/lib/modules/tls.js b/lib/modules/tls.js new file mode 100644 index 0000000..054bea5 --- /dev/null +++ b/lib/modules/tls.js @@ -0,0 +1,235 @@ +'use strict'; + +module.exports.create = function (deps, config, netHandler) { + var tls = require('tls'); + var parseSni = require('sni'); + var greenlock = require('greenlock'); + var domainMatches = require('../match-domain').match; + + function extractSocketProp(socket, propName) { + // remoteAddress, remotePort... ugh... https://github.com/nodejs/node/issues/8854 + return socket[propName] + || socket['_' + propName] + || socket._handle._parent.owner.stream[propName] + ; + } + + var le = greenlock.create({ + // server: 'staging' + server: 'https://acme-v01.api.letsencrypt.org/directory' + + , challenges: { + 'http-01': require('le-challenge-fs').create({ webrootPath: '/tmp/acme-challenges', debug: config.debug }) + , 'tls-sni-01': require('le-challenge-sni').create({ debug: config.debug }) + // TODO dns-01 + //, 'dns-01': require('le-challenge-ddns').create() + } + + , store: require('le-store-certbot').create({ webrootPath: '/tmp/acme-challenges' }) + + , approveDomains: function (opts, certs, cb) { + // This is where you check your database and associated + // email addresses with domains and agreements and such + + // The domains being approved for the first time are listed in opts.domains + // Certs being renewed are listed in certs.altnames + if (certs) { + // TODO make sure the same options are used for renewal as for registration? + opts.domains = certs.altnames; + + cb(null, { options: opts, certs: certs }); + return; + } + + function complete(optsOverride) { + Object.keys(optsOverride).forEach(function (key) { + opts[key] = optsOverride[key]; + }); + + cb(null, { options: opts, certs: certs }); + } + + + // check config for domain name + if (-1 !== (config.tls.servernames || []).indexOf(opts.domain)) { + // TODO how to handle SANs? + // TODO fetch domain-specific email + // TODO fetch domain-specific acmeDirectory + // NOTE: you can also change other options such as `challengeType` and `challenge` + // opts.challengeType = 'http-01'; + // opts.challenge = require('le-challenge-fs').create({}); // TODO this doesn't actually work yet + complete({ + email: config.tls.email + , agreeTos: true + , server: config.tls.acmeDirectoryUrl || le.server + , challengeType: config.tls.challengeType || 'http-01' + }); + return; + } + + // TODO ask http module (and potentially all other modules) about what domains it can + // handle. We can allow any domains that other modules will handle after we terminate TLS. + cb(new Error('domain is not allowed')); + // if (!modules.http) { + // modules.http = require('./modules/http.js').create(deps, config); + // } + // modules.http.checkServername(opts.domain).then(function (stuff) { + // if (!stuff || !stuff.domains) { + // // TODO once precheck is implemented we can just let it pass if it passes, yknow? + // cb(new Error('domain is not allowed')); + // return; + // } + + // complete({ + // domain: stuff.domain || stuff.domains[0] + // , domains: stuff.domains + // , email: stuff.email || program.email + // , server: stuff.acmeDirectoryUrl || program.acmeDirectoryUrl + // , challengeType: stuff.challengeType || program.challengeType + // , challenge: stuff.challenge + // }); + // return; + // }, cb); + } + }); + le.tlsOptions = le.tlsOptions || le.httpsOptions; + + var secureContexts = {}; + var terminatorOpts = require('localhost.daplie.me-certificates').merge({}); + terminatorOpts.SNICallback = function (sni, cb) { + console.log("[tlsOptions.SNICallback] SNI: '" + sni + "'"); + + var tlsOptions; + + // Static Certs + if (/.*localhost.*\.daplie\.me/.test(sni.toLowerCase())) { + // TODO implement + if (!secureContexts[sni]) { + tlsOptions = require('localhost.daplie.me-certificates').mergeTlsOptions(sni, {}); + } + if (tlsOptions) { + secureContexts[sni] = tls.createSecureContext(tlsOptions); + } + if (secureContexts[sni]) { + console.log('Got static secure context:', sni, secureContexts[sni]); + cb(null, secureContexts[sni]); + return; + } + } + + le.tlsOptions.SNICallback(sni, cb); + }; + + var terminator = tls.createServer(terminatorOpts, function (socket) { + console.log('(pre-terminated) tls connection, addr:', socket.remoteAddress); + + netHandler(socket, { + servername: socket.servername + , encrypted: true + // remoteAddress... ugh... https://github.com/nodejs/node/issues/8854 + , remoteAddress: extractSocketProp(socket, 'remoteAddress') + , remotePort: extractSocketProp(socket, 'remotePort') + , remoteFamily: extractSocketProp(socket, 'remoteFamily') + }); + }); + + function proxy(socket, opts, mod) { + var destination = mod.address.split(':'); + + var newConn = deps.net.createConnection({ + port: destination[1] + , host: destination[0] || '127.0.0.1' + + , servername: opts.servername + , data: opts.firstChunk + , remoteFamily: opts.family || extractSocketProp(socket, 'remoteFamily') + , remoteAddress: opts.address || extractSocketProp(socket, 'remoteAddress') + , remotePort: opts.port || extractSocketProp(socket, 'remotePort') + }); + + newConn.write(opts.firstChunk); + newConn.pipe(socket); + socket.pipe(newConn); + } + + function terminate(socket, opts) { + console.log('[tls-terminate] ' + opts.localAddress || socket.localAddress + ':' + opts.localPort || socket.localPort + ' servername', opts.servername, socket.remoteAddress); + + if (opts.hyperPeek) { + // This connection was peeked at using a method that doesn't interferre with the TLS + // server's ability to handle it properly. Currently the only way this happens is + // with tunnel connections where we have the first chunk of data before creating the + // new connection (thus removing need to get data off the new connection). + terminator.emit('connection', socket); + return; + } + + // The hyperPeek flag wasn't set, so we had to read data off of this connection, which + // means we can no longer use it directly in the TLS server. + // See https://github.com/nodejs/node/issues/8752 (node's internal networking layer == 💩 sometimes) + var myDuplex = require('tunnel-packer').Stream.create(socket); + myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; + myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; + + socket.on('data', function (chunk) { + console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); + myDuplex.push(chunk); + }); + socket.on('error', function (err) { + console.error('[error] httpsTunnel (Admin) TODO close'); + console.error(err); + myDuplex.emit('error', err); + }); + socket.on('close', function () { + myDuplex.end(); + }); + + terminator.emit('connection', myDuplex); + process.nextTick(function () { + // this must happen after the socket is emitted to the next in the chain, + // but before any more data comes in via the network + socket.unshift(opts.firstChunk); + }); + } + + function handleConn(socket, opts) { + opts.servername = (parseSni(opts.firstChunk)||'').toLowerCase() || 'localhost.invalid'; + // needs to wind up in one of 2 states: + // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket) + // 2. Terminated (goes on to a particular module or route, including the admin interface) + // 3. Closed (we don't recognize the SNI servername as something we actually want to handle) + + var handled = (config.tls.modules || []).some(function (mod) { + var relevant = mod.domains.some(function (pattern) { + return domainMatches(pattern, opts.servername); + }); + if (!relevant) { + return false; + } + + if (mod.name === 'proxy') { + proxy(socket, opts, mod); + } + else { + console.error('saw unknown TLS module', mod); + return false; + } + + return true; + }); + + // TODO: figure out all of the domains that the other modules intend to handle, and only + // terminate those ones, closing connections for all others. + if (!handled) { + terminate(socket, opts); + } + } + + return { + emit: function (type, socket) { + if (type === 'connection') { + handleConn(socket, socket.__opts); + } + } + }; +}; From 70e7d573955712799e5ff52f9b00274ccff6ebc7 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 10 May 2017 16:05:54 -0600 Subject: [PATCH 09/13] added hooks to handle ACME challenges --- lib/goldilocks.js | 21 ++++++++++++--------- lib/modules/http.js | 13 ++++++++----- lib/modules/tls.js | 9 +++++++++ lib/worker.js | 3 +++ 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/goldilocks.js b/lib/goldilocks.js index d398741..fd88055 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -6,20 +6,27 @@ module.exports.create = function (deps, config) { //var PromiseA = global.Promise; var PromiseA = require('bluebird'); var listeners = require('./servers').listeners; - var modules = { }; + var modules; + + function loadModules() { + modules = {}; + + modules.tls = require('./modules/tls').create(deps, config, netHandler); + modules.http = require('./modules/http.js').create(deps, config, modules.tls.middleware); + } // opts = { servername, encrypted, peek, data, remoteAddress, remotePort } function peek(conn, firstChunk, opts) { + if (!modules) { + loadModules(); + } + opts.firstChunk = firstChunk; conn.__opts = opts; // TODO port/service-based routing can do here // TLS byte 1 is handshake and byte 6 is client hello if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) { - if (!modules.tls) { - modules.tls = require('./modules/tls').create(deps, config, netHandler); - } - modules.tls.emit('connection', conn); return; } @@ -37,10 +44,6 @@ module.exports.create = function (deps, config) { if (firstChunk[0] > 32 && firstChunk[0] < 127) { var firstStr = firstChunk.toString(); if (/HTTP\//i.test(firstStr)) { - if (!modules.http) { - modules.http = require('./modules/http.js').create(deps, config); - } - modules.http.emit('connection', conn); return; } diff --git a/lib/modules/http.js b/lib/modules/http.js index 5c6247a..ab41e6c 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -1,6 +1,6 @@ 'use strict'; -module.exports.create = function (deps, conf) { +module.exports.create = function (deps, conf, greenlockMiddleware) { var express = require('express'); var app = express(); var adminApp = require('./admin').create(deps, conf); @@ -19,11 +19,13 @@ module.exports.create = function (deps, conf) { var redirecters = {}; function redirectHttps(req, res, next) { var port = req.headers.host.split(':')[1]; - var redirecter = redirecters[port]; - if (!redirecter) { - redirecter = redirecters[port] = require('redirect-https')({port: port}); + if (!redirecters[port]) { + redirecters[port] = require('redirect-https')({ + port: port + , trustProxy: conf.http.trustProxy + }); } - redirecter(req, res, next); + redirecters[port](req, res, next); } function handleAdmin(req, res, next) { @@ -123,6 +125,7 @@ module.exports.create = function (deps, conf) { }; } + app.use(greenlockMiddleware); app.use(redirectHttps); app.use(handleAdmin); diff --git a/lib/modules/tls.js b/lib/modules/tls.js index 054bea5..c3570ce 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -199,6 +199,14 @@ module.exports.create = function (deps, config, netHandler) { // 2. Terminated (goes on to a particular module or route, including the admin interface) // 3. Closed (we don't recognize the SNI servername as something we actually want to handle) + // We always want to terminate is the SNI matches the challenge pattern, unless a client + // on the south side has temporarily claimed a particular challenge. For the time being + // we don't have a way for the south-side to communicate with us, so that part isn't done. + if (domainMatches('*.acme-challenge.invalid', opts.servername)) { + terminate(socket, opts); + return; + } + var handled = (config.tls.modules || []).some(function (mod) { var relevant = mod.domains.some(function (pattern) { return domainMatches(pattern, opts.servername); @@ -231,5 +239,6 @@ module.exports.create = function (deps, config, netHandler) { handleConn(socket, socket.__opts); } } + , middleware: le.middleware() }; }; diff --git a/lib/worker.js b/lib/worker.js index 84569e1..79db594 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -4,6 +4,9 @@ process.on('message', function (conf) { var deps = { messenger: process + // Note that if a custom createConnections is used it will be called with different + // sets of custom options based on what is actually being proxied. Most notably the + // HTTP proxying connection creation is not something we currently control. , net: require('net') }; require('./goldilocks.js').create(deps, conf); From 158c363c8868d96a599c663c5b665f8311b8a444 Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 10 May 2017 16:56:08 -0600 Subject: [PATCH 10/13] added example config to show what can currently be done --- goldilocks.example.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 goldilocks.example.yml diff --git a/goldilocks.example.yml b/goldilocks.example.yml new file mode 100644 index 0000000..1646a19 --- /dev/null +++ b/goldilocks.example.yml @@ -0,0 +1,30 @@ +tcp: + bind: + - 22 + - 80 + - 443 + modules: + - name: forward + ports: + - 22 + address: '127.0.0.1:8022' + +tls: + modules: + - name: proxy + domains: + - localhost.bar.daplie.me + - localhost.foo.daplie.me + address: '127.0.0.1:5443' + +http: + trustProxy: true + modules: + - name: proxy + domains: + - localhost.daplie.me + address: '127.0.0.1:4000' + - name: static + domains: + - '*.localhost.daplie.me' + root: '/srv/www/:hostname' From e24f9412dd743fdd77ac4e0603cac1f2daf2a13d Mon Sep 17 00:00:00 2001 From: tigerbot Date: Wed, 10 May 2017 17:21:03 -0600 Subject: [PATCH 11/13] improved error handling for TLS/TCP proxying --- lib/goldilocks.js | 11 +++++++++++ lib/modules/tls.js | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/goldilocks.js b/lib/goldilocks.js index fd88055..f0f269a 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -94,6 +94,17 @@ module.exports.create = function (deps, config) { }); + // Not sure how to effectively report this to the user or client, but we need to listen + // for the event to prevent it from crashing us. + newConn.on('error', function (err) { + console.error('TCP forward connection error', err); + conn.end(); + }); + conn.on('error', function (err) { + console.error('TCP forward client error', err); + newConn.end(); + }); + newConn.pipe(conn); conn.pipe(newConn); }; diff --git a/lib/modules/tls.js b/lib/modules/tls.js index c3570ce..16f551f 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -147,6 +147,17 @@ module.exports.create = function (deps, config, netHandler) { , remotePort: opts.port || extractSocketProp(socket, 'remotePort') }); + // Not sure how to effectively report this to the user or client, but we need to listen + // for the event to prevent it from crashing us. + newConn.on('error', function (err) { + console.error('TLS proxy connection error', err); + socket.end(); + }); + socket.on('error', function (err) { + console.error('TLS proxy client error', err); + newConn.end(); + }); + newConn.write(opts.firstChunk); newConn.pipe(socket); socket.pipe(newConn); From 5777a885a4841a9f7bfb74138bc96f4f0b9061ba Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 11 May 2017 16:42:14 -0600 Subject: [PATCH 12/13] improved feedback for bad TLS/TCP gateways --- lib/goldilocks.js | 15 ++++-- lib/modules/http.js | 12 ++--- lib/modules/tls.js | 104 +++++++++++++++++++++++++++--------------- lib/proxy-err-resp.js | 32 +++++++++++++ 4 files changed, 113 insertions(+), 50 deletions(-) create mode 100644 lib/proxy-err-resp.js diff --git a/lib/goldilocks.js b/lib/goldilocks.js index f0f269a..023a44a 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -81,6 +81,7 @@ module.exports.create = function (deps, config) { function createTcpForwarder(mod) { var destination = mod.address.split(':'); + var connected = false; return function (conn) { var newConn = deps.net.createConnection({ @@ -91,22 +92,28 @@ module.exports.create = function (deps, config) { , remoteAddress: conn.remoteAddress , remotePort: conn.remotePort }, function () { + connected = true; + newConn.pipe(conn); + conn.pipe(newConn); }); // Not sure how to effectively report this to the user or client, but we need to listen // for the event to prevent it from crashing us. newConn.on('error', function (err) { - console.error('TCP forward connection error', err); - conn.end(); + if (connected) { + console.error('TCP forward remote error', err); + conn.end(); + } else { + console.log('TCP forward connection error', err); + require('./proxy-err-resp').sendBadGateway(conn, err, config.debug); + } }); conn.on('error', function (err) { console.error('TCP forward client error', err); newConn.end(); }); - newConn.pipe(conn); - conn.pipe(newConn); }; } diff --git a/lib/modules/http.js b/lib/modules/http.js index ab41e6c..7643862 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -69,14 +69,10 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { proxy.on('error', function (err, req, res) { console.log(err); - res.writeHead(502); - if (err.code === 'ECONNREFUSED') { - res.end('The connection was refused. Most likely the service being connected to ' - + 'has stopped running or the configuration is wrong.'); - } - else { - res.end('Bad Gateway: ' + err.code); - } + res.statusCode = 502; + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Connection', 'close'); + res.end(require('../proxy-err-resp').getRespBody(err, conf.debug)); }); return function (req, res, next) { diff --git a/lib/modules/tls.js b/lib/modules/tls.js index 16f551f..1b6b606 100644 --- a/lib/modules/tls.js +++ b/lib/modules/tls.js @@ -4,6 +4,7 @@ module.exports.create = function (deps, config, netHandler) { var tls = require('tls'); var parseSni = require('sni'); var greenlock = require('greenlock'); + var localhostCerts = require('localhost.daplie.me-certificates'); var domainMatches = require('../match-domain').match; function extractSocketProp(socket, propName) { @@ -14,6 +15,34 @@ module.exports.create = function (deps, config, netHandler) { ; } + function wrapSocket(socket, opts) { + var myDuplex = require('tunnel-packer').Stream.create(socket); + myDuplex.remoteFamily = opts.remoteFamily || myDuplex.remoteFamily; + myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; + myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; + + socket.on('data', function (chunk) { + console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); + myDuplex.push(chunk); + }); + socket.on('error', function (err) { + console.error('[error] httpsTunnel (Admin) TODO close'); + console.error(err); + myDuplex.emit('error', err); + }); + socket.on('close', function () { + myDuplex.end(); + }); + + process.nextTick(function () { + // this must happen after the socket is emitted to the next in the chain, + // but before any more data comes in via the network + socket.unshift(opts.firstChunk); + }); + + return myDuplex; + } + var le = greenlock.create({ // server: 'staging' server: 'https://acme-v01.api.letsencrypt.org/directory' @@ -105,7 +134,7 @@ module.exports.create = function (deps, config, netHandler) { if (/.*localhost.*\.daplie\.me/.test(sni.toLowerCase())) { // TODO implement if (!secureContexts[sni]) { - tlsOptions = require('localhost.daplie.me-certificates').mergeTlsOptions(sni, {}); + tlsOptions = localhostCerts.mergeTlsOptions(sni, {}); } if (tlsOptions) { secureContexts[sni] = tls.createSecureContext(tlsOptions); @@ -120,7 +149,7 @@ module.exports.create = function (deps, config, netHandler) { le.tlsOptions.SNICallback(sni, cb); }; - var terminator = tls.createServer(terminatorOpts, function (socket) { + var terminateServer = tls.createServer(terminatorOpts, function (socket) { console.log('(pre-terminated) tls connection, addr:', socket.remoteAddress); netHandler(socket, { @@ -135,6 +164,7 @@ module.exports.create = function (deps, config, netHandler) { function proxy(socket, opts, mod) { var destination = mod.address.split(':'); + var connected = false; var newConn = deps.net.createConnection({ port: destination[1] @@ -145,62 +175,60 @@ module.exports.create = function (deps, config, netHandler) { , remoteFamily: opts.family || extractSocketProp(socket, 'remoteFamily') , remoteAddress: opts.address || extractSocketProp(socket, 'remoteAddress') , remotePort: opts.port || extractSocketProp(socket, 'remotePort') + }, function () { + connected = true; + if (!opts.hyperPeek) { + newConn.write(opts.firstChunk); + } + newConn.pipe(socket); + socket.pipe(newConn); }); // Not sure how to effectively report this to the user or client, but we need to listen // for the event to prevent it from crashing us. newConn.on('error', function (err) { - console.error('TLS proxy connection error', err); - socket.end(); + if (connected) { + console.error('TLS proxy remote error', err); + socket.end(); + } else { + console.log('TLS proxy connection error', err); + var tlsOpts = localhostCerts.mergeTlsOptions('localhost.daplie.me', {isServer: true}); + var decrypted; + if (opts.hyperPeek) { + decrypted = new tls.TLSSocket(socket, tlsOpts); + } else { + decrypted = new tls.TLSSocket(wrapSocket(socket, opts), tlsOpts); + } + require('../proxy-err-resp').sendBadGateway(decrypted, err, config.debug); + } }); socket.on('error', function (err) { console.error('TLS proxy client error', err); newConn.end(); }); - - newConn.write(opts.firstChunk); - newConn.pipe(socket); - socket.pipe(newConn); } function terminate(socket, opts) { - console.log('[tls-terminate] ' + opts.localAddress || socket.localAddress + ':' + opts.localPort || socket.localPort + ' servername', opts.servername, socket.remoteAddress); + console.log( + '[tls-terminate]' + , opts.localAddress || socket.localAddress +':'+ opts.localPort || socket.localPort + , 'servername=' + opts.servername + , opts.remoteAddress || socket.remoteAddress + ); if (opts.hyperPeek) { // This connection was peeked at using a method that doesn't interferre with the TLS // server's ability to handle it properly. Currently the only way this happens is // with tunnel connections where we have the first chunk of data before creating the // new connection (thus removing need to get data off the new connection). - terminator.emit('connection', socket); - return; + terminateServer.emit('connection', socket); + } + else { + // The hyperPeek flag wasn't set, so we had to read data off of this connection, which + // means we can no longer use it directly in the TLS server. + // See https://github.com/nodejs/node/issues/8752 (node's internal networking layer == 💩 sometimes) + terminateServer.emit('connection', wrapSocket(socket, opts)); } - - // The hyperPeek flag wasn't set, so we had to read data off of this connection, which - // means we can no longer use it directly in the TLS server. - // See https://github.com/nodejs/node/issues/8752 (node's internal networking layer == 💩 sometimes) - var myDuplex = require('tunnel-packer').Stream.create(socket); - myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress; - myDuplex.remotePort = opts.remotePort || myDuplex.remotePort; - - socket.on('data', function (chunk) { - console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); - myDuplex.push(chunk); - }); - socket.on('error', function (err) { - console.error('[error] httpsTunnel (Admin) TODO close'); - console.error(err); - myDuplex.emit('error', err); - }); - socket.on('close', function () { - myDuplex.end(); - }); - - terminator.emit('connection', myDuplex); - process.nextTick(function () { - // this must happen after the socket is emitted to the next in the chain, - // but before any more data comes in via the network - socket.unshift(opts.firstChunk); - }); } function handleConn(socket, opts) { diff --git a/lib/proxy-err-resp.js b/lib/proxy-err-resp.js new file mode 100644 index 0000000..c2a35a8 --- /dev/null +++ b/lib/proxy-err-resp.js @@ -0,0 +1,32 @@ +'use strict'; + +function getRespBody(err, debug) { + if (debug) { + return err.toString(); + } + + if (err.code === 'ECONNREFUSED') { + return 'The connection was refused. Most likely the service being connected to ' + + 'has stopped running or the configuration is wrong.'; + } + + return 'Bad Gateway: ' + err.code; +} + +function sendBadGateway(conn, err, debug) { + var body = getRespBody(err, debug); + + conn.write([ + 'HTTP/1.1 502 Bad Gateway' + , 'Date: ' + (new Date()).toUTCString() + , 'Connection: close' + , 'Content-Type: text/html' + , 'Content-Length: ' + body.length + , '' + , body + ].join('\r\n')); + conn.end(); +} + +module.exports.getRespBody = getRespBody; +module.exports.sendBadGateway = sendBadGateway; From 87de2c65add9fb99e3f3098f36319409e451a6ef Mon Sep 17 00:00:00 2001 From: tigerbot Date: Thu, 11 May 2017 19:16:23 -0600 Subject: [PATCH 13/13] redirect localhost and IP addresses to real domains --- goldilocks.example.yml | 2 ++ lib/goldilocks.js | 2 +- lib/modules/http.js | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/goldilocks.example.yml b/goldilocks.example.yml index 1646a19..ef5ac96 100644 --- a/goldilocks.example.yml +++ b/goldilocks.example.yml @@ -19,6 +19,8 @@ tls: http: trustProxy: true + allowInsecure: false + primaryDomain: localhost.foo.daplie.me modules: - name: proxy domains: diff --git a/lib/goldilocks.js b/lib/goldilocks.js index 023a44a..709ade1 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -54,7 +54,7 @@ module.exports.create = function (deps, config) { } function netHandler(conn, opts) { opts = opts || {}; - console.log('[netHandler]', conn.localAddres, conn.localPort, opts.encrypted); + console.log('[netHandler]', conn.localAddress, conn.localPort, opts.encrypted); // XXX PEEK COMMENT XXX // TODO we can have our cake and eat it too diff --git a/lib/modules/http.js b/lib/modules/http.js index 7643862..8ace19d 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -13,18 +13,55 @@ module.exports.create = function (deps, conf, greenlockMiddleware) { , /\balpha\.localhost\./ ]; + function verifyHost(fullHost) { + var host = /^(.*?)(:\d+)?$/.exec(fullHost)[1]; + + if (host === 'localhost') { + return fullHost.replace(host, 'localhost.daplie.me'); + } + + // Test for IPv4 and IPv6 addresses. These patterns will match some invalid addresses, + // but since those still won't be valid domains that won't really be a problem. + if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host) || /^\[[0-9a-fA-F:]+\]$/.test(host)) { + if (!conf.http.primaryDomain) { + (conf.http.modules || []).some(function (mod) { + return mod.domains.some(function (domain) { + if (domain[0] !== '*') { + conf.http.primaryDomain = domain; + return true; + } + }); + }); + } + return fullHost.replace(host, conf.http.primaryDomain || host); + } + + return fullHost; + } + // We handle both HTTPS and HTTP traffic on the same ports, and we want to redirect // any unencrypted requests to the same port they came from unless it came in on // the default HTTP port, in which case there wont be a port specified in the host. var redirecters = {}; function redirectHttps(req, res, next) { - var port = req.headers.host.split(':')[1]; + if (conf.http.allowInsecure) { + next(); + return; + } + + var port = (/:(\d+)$/.exec(req.headers.host) || [])[1]; if (!redirecters[port]) { redirecters[port] = require('redirect-https')({ port: port , trustProxy: conf.http.trustProxy }); } + + // localhost and IP addresses cannot have real SSL certs (and don't contain any useful + // info for redirection either), so we direct some hosts to either localhost.daplie.me + // or the "primary domain" ie the first manually specified domain. + req.headers.host = verifyHost(req.headers.host); + redirecters[port](req, res, next); }