From dc5516941530acd45b313ba0a18b5e62c1b91208 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 27 Apr 2017 16:05:34 -0600 Subject: [PATCH] proxy mostly works --- bin/goldilocks.js | 2 +- lib/goldilocks.js | 140 +++++++++------ lib/modules/http.js | 416 +++----------------------------------------- lib/worker.js | 1 + 4 files changed, 114 insertions(+), 445 deletions(-) diff --git a/bin/goldilocks.js b/bin/goldilocks.js index 4125450..ed618ce 100755 --- a/bin/goldilocks.js +++ b/bin/goldilocks.js @@ -65,7 +65,7 @@ function readConfigAndRun(args) { config.tcp = {}; } if (!config.http) { - config.http = {}; + config.http = { proxy: { port: 3000 } }; } if (!config.tls) { config.tls = { diff --git a/lib/goldilocks.js b/lib/goldilocks.js index ad7e86c..f714e28 100644 --- a/lib/goldilocks.js +++ b/lib/goldilocks.js @@ -22,29 +22,27 @@ module.exports.create = function (deps, config) { _map: { } , _create: function (address, port) { // port provides hinting for http, smtp, etc - return function (conn, firstChunk) { - console.log('[tcpRouter] ' + address + ':' + port + ' servername'); + return function (conn, firstChunk, opts) { + console.log('[tcpRouter] ' + address + ':' + port + ' ' + (opts.servername || '')); - // At this point we cannot necessarily trace which port or address the socket came from - // (because node's netowrking layer == 💩 ) var m; var str; - var servername; + var hostname; + var newHeads; + // TODO test per-module // Maybe HTTP if (firstChunk[0] > 32 && firstChunk[0] < 127) { str = firstChunk.toString(); m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im); - servername = (m && m[1].toLowerCase() || '').split(':')[0]; - //conn.__servername = servername; - console.log('[tcpRouter] hostname', servername); + hostname = (m && m[1].toLowerCase() || '').split(':')[0]; + console.log('[tcpRouter] hostname', hostname); if (/HTTP\//i.test(str)) { //conn.__service = 'http'; } } - console.log('1010'); - if (!servername) { + if (!hostname) { // TODO allow tcp tunneling // TODO we need some way of tagging tcp as either terminated tls or insecure conn.write( @@ -59,25 +57,54 @@ module.exports.create = function (deps, config) { return; } - console.log('1020'); - if (/\blocalhost\.admin\./.test(servername) || /\badmin\.localhost\./.test(servername) - || /\blocalhost\.alpha\./.test(servername) || /\balpha\.localhost\./.test(servername)) { - console.log('1050'); + + // Poor-man's http proxy + // XXX SECURITY XXX: should strip existing X-Forwarded headers + newHeads = + [ "X-Forwarded-Proto: " + (opts.encrypted ? 'https' : 'http') + , "X-Forwarded-For: " + (conn.remoteAddress || opts.remoteAddress) + , "X-Forwarded-Host: " + hostname + ]; + + if (!opts.encrypted) { + // a exists-only header that a bad client could not remove + newHeads.push("X-Not-Encrypted: yes"); + } + if (opts.servername) { + newHeads.push("X-Forwarded-Sni: " + opts.servername); + if (opts.servername !== hostname) { + // an exists-only header that a bad client could not remove + newHeads.push("X-Two-Servernames: yes"); + } + } + + firstChunk = firstChunk.toString('utf8'); + // JSON.stringify("Host: example.com\r\nNext: Header".replace(/(Host: [^\r\n]*)/i, "$1" + "\r\n" + "X: XYZ")) + firstChunk = firstChunk.replace(/(Host: [^\r\n]*)/i, "$1" + "\r\n" + newHeads.join("\r\n")); + process.nextTick(function () { + conn.unshift(Buffer.from(firstChunk, 'utf8')); + }); + + // + // hard-coded routes for the admin interface + if ( + /\blocalhost\.admin\./.test(hostname) || /\badmin\.localhost\./.test(hostname) + || /\blocalhost\.alpha\./.test(hostname) || /\balpha\.localhost\./.test(hostname) + ) { if (!modules.admin) { modules.admin = require('./modules/admin.js').create(deps, config); } - console.log('1100'); modules.admin.emit('connection', conn); - console.log('1500'); return; } - if (!modules.http) { - if (!modules.http) { - modules.http = require('./modules/http.js').create(deps, config); - } - modules.http.emit('connection', conn); - } + // TODO static file handiling and such or whatever + if (!modules.http) { + modules.http = require('./modules/http.js').create(deps, config); + } + opts.hostname = hostname; + conn.__opts = opts; + modules.http.emit('connection', conn); }; } , get: function getTcpRouter(address, port) { @@ -93,23 +120,33 @@ module.exports.create = function (deps, config) { }; var tlsRouter = { _map: { } - , _create: function (address, port) { + , _create: function (address, port/*, nextServer*/) { // port provides hinting for https, smtps, etc - return function (socket, servername) { - //program.tlsTunnelServer.emit('connection', socket); - //return; - console.log('[tlsRouter] ' + address + ':' + port + ' servername', servername); - + return function (socket, firstChunk, opts) { + var servername = opts.servername; var packerStream = require('tunnel-packer').Stream; var myDuplex = packerStream.create(socket); + console.log('[tlsRouter] ' + address + ':' + port + ' servername', servername, myDuplex.remoteAddress); + // needs to wind up in one of 3 states: - // 1. Proxied / Tunneled (we don't even need to put it through the tlsSocket) - // 2. Admin (skips normal processing) + // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket) + // 2. Admin Interface (skips the proxying) // 3. Terminated (goes on to a particular module or route) //myDuplex.__tlsTerminated = true; + + process.nextTick(function () { + // this must happen after the socket is emitted to the next in the chain, + // but before any more data comes in via the network + socket.unshift(firstChunk); + }); + + // nextServer.emit could be used here program.tlsTunnelServer.emit('connection', myDuplex); + // Why all this wacky-do with the myDuplex? + // because https://github.com/nodejs/node/issues/8854, that's why + // (because node's internal networking layer == 💩 sometimes) socket.on('data', function (chunk) { console.log('[' + Date.now() + '] tls socket data', chunk.byteLength); myDuplex.push(chunk); @@ -120,7 +157,7 @@ module.exports.create = function (deps, config) { myDuplex.emit('error', err); }); socket.on('close', function () { - myDuplex.close(); + myDuplex.end(); }); }; } @@ -137,44 +174,27 @@ module.exports.create = function (deps, config) { }; + // opts = { servername, encrypted, remoteAddress, remotePort } function handler(conn, opts) { opts = opts || {}; - console.log('[handler]', conn.localAddres, conn.localPort, opts.secure); + console.log('[handler]', conn.localAddres, conn.localPort, opts.encrypted); - // TODO inspect SNI and HTTP Host conn.once('data', function (firstChunk) { var servername; - process.nextTick(function () { - conn.unshift(firstChunk); - }); - // copying stuff over to firstChunk because the network abstraction goes too deep to find these again - //firstChunk.__port = conn.__port; + // TODO port-based routing can do here // TLS if (22 === firstChunk[0]) { servername = (parseSni(firstChunk)||'').toLowerCase() || 'localhost.invalid'; - //conn.__servername = servername; - //conn.__tls = true; - //conn.__tlsTerminated = false; - //firstChunk.__servername = conn.__servername; - //firstChunk.__tls = true; - //firstChunk.__tlsTerminated = false; console.log('tryTls'); - tlsRouter.get(conn.localAddress, conn.localPort)(conn, servername); + tlsRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, opts); } else { - // TODO how to tag as insecure? console.log('tryTcp'); - tcpRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, { secure: opts.secure || false }); + tcpRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, opts); } }); - -/* - if ('http' === config.tcp.default || !config.tcp.default) { - console.log('deal with as http'); - } -*/ } function approveDomains(opts, certs, cb) { @@ -289,18 +309,24 @@ module.exports.create = function (deps, config) { if (!program.greenlock) { program.greenlock = getAcme(); } - (program.greenlock.tlsOptions||program.greenlock.httpsOptions).SNICallback(servername, cb); + (program.greenlock.tlsOptions||program.greenlock.httpsOptions).SNICallback(sni, cb); }; program.tlsTunnelServer = tls.createServer(tunnelAdminTlsOpts, function (tlsSocket) { - console.log('(pre-terminated) tls connection'); + console.log('(pre-terminated) tls connection, addr:', tlsSocket.remoteAddress); // things get a little messed up here //tlsSocket.on('data', function (chunk) { // console.log('terminated data:', chunk.toString()); //}); //(program.httpTunnelServer || program.httpServer).emit('connection', tlsSocket); - //tcpRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, { secure: false }); - handler(tlsSocket, { secure: true }); + //tcpRouter.get(conn.localAddress, conn.localPort)(conn, firstChunk, { encrypted: false }); + handler(tlsSocket, { + servername: tlsSocket.servername + , encrypted: true + // remoteAddress... ugh... https://github.com/nodejs/node/issues/8854 + , remoteAddress: tlsSocket.remoteAddress || tlsSocket._handle._parent.owner.stream.remoteAddress + , remotePort: tlsSocket.remotePort || tlsSocket._handle._parent.owner.stream.remotePort + }); }); PromiseA.all(config.tcp.ports.map(function (port) { diff --git a/lib/modules/http.js b/lib/modules/http.js index 7adf9a1..57a6c8d 100644 --- a/lib/modules/http.js +++ b/lib/modules/http.js @@ -1,392 +1,34 @@ - function run() { - var defaultServername = 'localhost.daplie.me'; - var minimist = require('minimist'); - var argv = minimist(process.argv.slice(2)); - var port = parseInt(argv.p || argv.port || argv._[0], 10) || httpsPort; - var livereload = argv.livereload; - var defaultWebRoot = path.normalize(argv['default-web-root'] || argv.d || argv._[1] || '.'); - var assetsPath = path.join(__dirname, '..', 'packages', 'assets'); - var content = argv.c; - var letsencryptHost = argv['letsencrypt-certs']; - var yaml = require('js-yaml'); - var fs = PromiseA.promisifyAll(require('fs')); - var configFile = argv.c || argv.conf || argv.config; - var config; - var DDNS; - console.log('defaultWebRoot', defaultWebRoot); +'use strict'; - try { - config = fs.readFileSync(configFile || 'goldilocks.yml'); - } catch(e) { - if (configFile) { - console.error('Failed to read config:', e); - process.exit(1); - } - } +module.exports.create = function (deps, conf) { + // This should be able to handle things like default web path (i.e. /srv/www/hostname), + // no-www redirect, and transpilation of static assets (i.e. cached versions of raw html) + // but right now it's a very dumb proxy - if (config) { - try { - config = yaml.safeLoad(config); - } catch(e) { - console.error('Failed to parse config:', e); - process.exit(1); - } - } + function createConnection(conn) { + var opts = conn.__opts; + var newConn = deps.net.createConnection({ + port: conf.http.proxy.port + , host: '127.0.0.1' - if (argv.V || argv.version || argv.v) { - if (argv.v) { - console.warn("flag -v is reserved for future use. Use -V or --version for version information."); - } - console.info('v' + require('../package.json').version); - return; - } + , servername: opts.servername + , data: opts.data + , remoteFamily: opts.family || conn.remoteFamily + , remoteAddress: opts.address || conn.remoteAddress + , remotePort: opts.port || conn.remotePort + }, function () { + //console.log("[=>] first packet from tunneler to '" + cid + "' as '" + opts.service + "'", opts.data.byteLength); + // this will happen before 'data' is triggered + //newConn.write(opts.data); + }); - argv.sites = argv.sites; + newConn.pipe(conn); + conn.pipe(newConn); + } - // letsencrypt - var httpsOptions = require('localhost.daplie.me-certificates').merge({}); - var secureContext; - - var opts = { - agreeTos: argv.agreeTos || argv['agree-tos'] - , debug: argv.debug - , device: argv.device - , provider: (argv.provider && 'false' !== argv.provider) ? argv.provider : 'oauth3.org' - , email: argv.email - , httpsOptions: { - key: httpsOptions.key - , cert: httpsOptions.cert - //, ca: httpsOptions.ca - } - , homedir: argv.homedir - , argv: argv - }; - var peerCa; - var p; - - opts.PromiseA = PromiseA; - opts.httpsOptions.SNICallback = function (sni, cb) { - if (!secureContext) { - secureContext = tls.createSecureContext(opts.httpsOptions); - } - cb(null, secureContext); - return; - }; - - if (letsencryptHost) { - // TODO remove in v3.x (aka goldilocks) - argv.key = argv.key || '/etc/letsencrypt/live/' + letsencryptHost + '/privkey.pem'; - argv.cert = argv.cert || '/etc/letsencrypt/live/' + letsencryptHost + '/fullchain.pem'; - argv.root = argv.root || argv.chain || ''; - argv.sites = argv.sites || letsencryptHost; - argv['serve-root'] = argv['serve-root'] || argv['serve-chain']; - // argv[express-app] - } - - if (argv['serve-root'] && !argv.root) { - console.error("You must specify bath --root to use --serve-root"); - return; - } - - if (argv.key || argv.cert || argv.root) { - if (!argv.key || !argv.cert) { - console.error("You must specify bath --key and --cert, and optionally --root (required with serve-root)"); - return; - } - - if (!Array.isArray(argv.root)) { - argv.root = [argv.root]; - } - - opts.httpsOptions.key = fs.readFileSync(argv.key); - opts.httpsOptions.cert = fs.readFileSync(argv.cert); - - // turn multiple-cert pemfile into array of cert strings - peerCa = argv.root.reduce(function (roots, fullpath) { - if (!fs.existsSync(fullpath)) { - return roots; - } - - return roots.concat(fs.readFileSync(fullpath, 'ascii') - .split('-----END CERTIFICATE-----') - .filter(function (ca) { - return ca.trim(); - }).map(function (ca) { - return (ca + '-----END CERTIFICATE-----').trim(); - })); - }, []); - - // TODO * `--verify /path/to/root.pem` require peers to present certificates from said authority - if (argv.verify) { - opts.httpsOptions.ca = peerCa; - opts.httpsOptions.requestCert = true; - opts.httpsOptions.rejectUnauthorized = true; - } - - if (argv['serve-root']) { - content = peerCa.join('\r\n'); - } - } - - - opts.cwd = process.cwd(); - opts.sites = []; - opts.sites._map = {}; - - if (argv.sites) { - opts._externalHost = false; - argv.sites.split(',').map(function (name) { - var nameparts = name.split('|'); - var servername = nameparts.shift(); - var modules; - - opts._externalHost = opts._externalHost || !/(^|\.)localhost\./.test(servername); - // TODO allow reverse proxy - if (!opts.sites._map[servername]) { - opts.sites._map[servername] = { $id: servername, paths: [] }; - opts.sites._map[servername].paths._map = {}; - opts.sites.push(opts.sites._map[servername]); - } - - if (!nameparts.length) { - return; - } - - if (!opts.sites._map[servername].paths._map['/']) { - opts.sites._map[servername].paths._map['/'] = { $id: '/', modules: [] }; - opts.sites._map[servername].paths.push(opts.sites._map[servername].paths._map['/']); - } - - modules = opts.sites._map[servername].paths._map['/'].modules; - modules.push({ - $id: 'serve' - , paths: nameparts - }); - modules.push({ - $id: 'indexes' - , paths: nameparts - }); - }); - } - - opts.groups = []; - - // 'packages', 'assets', 'com.daplie.caddy' - opts.global = { - modules: [ // TODO uh-oh we've got a mixed bag of modules (various types), a true map - { $id: 'greenlock', email: opts.email, tos: opts.tos } - , { $id: 'rvpn', email: opts.email, tos: opts.tos } - , { $id: 'content', content: content } - , { $id: 'livereload', on: opts.livereload } - , { $id: 'app', path: opts.expressApp } - ] - , paths: [ - { $id: '/assets/', modules: [ { $id: 'serve', paths: [ assetsPath ] } ] } - // TODO figure this b out - , { $id: '/.well-known/', modules: [ - { $id: 'serve', paths: [ path.join(assetsPath, 'well-known') ] } - ] } - ] - }; - opts.defaults = { - modules: [] - , paths: [ - { $id: '/', modules: [ - { $id: 'serve', paths: [ defaultWebRoot ] } - , { $id: 'indexes', paths: [ defaultWebRoot ] } - ] } - ] - }; - opts.sites.push({ - // greenlock: {} - $id: 'localhost.alpha.daplie.me' - , paths: [ - { $id: '/', modules: [ - { $id: 'serve', paths: [ path.resolve(__dirname, '..', 'admin', 'public') ] } - ] } - , { $id: '/api/', modules: [ - { $id: 'app', path: path.join(__dirname, 'admin') } - ] } - ] - }); - opts.sites.push({ - $id: 'localhost.daplie.invalid' - , paths: [ - { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', 'admin', 'public') ] } ] } - , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] } - ] - }); - - // ifaces - opts.ifaces = require('../lib/local-ip.js').find(); - - // TODO use arrays in all things - opts._old_server_name = opts.sites[0].$id; - opts.pubdir = defaultWebRoot.replace(/(:hostname|:servername).*/, ''); - - if (argv.p || argv.port || argv._[0]) { - opts.manualPort = true; - } - if (argv.t || argv.tunnel) { - opts.tunnel = true; - } - if (argv.i || argv['insecure-port']) { - opts.manualInsecurePort = true; - } - opts.insecurePort = parseInt(argv.i || argv['insecure-port'], 10) - || argv.i || argv['insecure-port'] - || httpPort - ; - opts.livereload = livereload; - - if (argv['express-app']) { - opts.expressApp = require(argv['express-app']); - } - - if (opts.email || opts._externalHost) { - if (!opts.agreeTos) { - console.warn("You may need to specify --agree-tos to agree to both the Let's Encrypt and Daplie DNS terms of service."); - } - if (!opts.email) { - // TODO store email in .ddnsrc.json - console.warn("You may need to specify --email to register with both the Let's Encrypt and Daplie DNS."); - } - DDNS = require('ddns-cli'); - p = DDNS.refreshToken({ - email: opts.email - , providerUrl: opts.provider - , silent: true - , homedir: opts.homedir - }, { - debug: false - , email: opts.argv.email - }).then(function (refreshToken) { - opts.refreshToken = refreshToken; - }); - } - else { - p = PromiseA.resolve(); - } - - return p.then(function () { - - // can be changed to tunnel external port - opts.redirectOptions = { - port: opts.port - }; - opts.redirectApp = require('redirect-https')(opts.redirectOptions); - - return createServer(port, null, content, opts).then(function (servers) { - var p; - var httpsUrl; - var httpUrl; - var promise; - - // TODO show all sites - console.info(''); - console.info('Serving ' + opts.pubdir + ' at '); - console.info(''); - - // Port - httpsUrl = 'https://' + opts._old_server_name; - p = opts.port; - if (httpsPort !== p) { - httpsUrl += ':' + p; - } - console.info('\t' + httpsUrl); - - // Insecure Port - httpUrl = 'http://' + opts._old_server_name; - p = opts.insecurePort; - if (httpPort !== p) { - httpUrl += ':' + p; - } - console.info('\t' + httpUrl + ' (redirecting to https)'); - console.info(''); - - if (!(argv.sites && (defaultServername !== argv.sites) && !(argv.key && argv.cert))) { - // TODO what is this condition actually intending to test again? - // (I think it can be replaced with if (!opts._externalHost) { ... } - - promise = PromiseA.resolve(); - } else { - console.info("Attempting to resolve external connection for '" + opts._old_server_name + "'"); - try { - promise = require('../lib/match-ips.js').match(opts._old_server_name, opts); - } catch(e) { - console.warn("Upgrade to version 2.x to use automatic certificate issuance for '" + opts._old_server_name + "'"); - promise = PromiseA.resolve(); - } - } - - return promise.then(function (matchingIps) { - if (matchingIps) { - if (!matchingIps.length) { - console.info("Neither the attached nor external interfaces match '" + opts._old_server_name + "'"); - } - } - opts.matchingIps = matchingIps || []; - - if (opts.matchingIps.length) { - console.info(''); - console.info('External IPs:'); - console.info(''); - opts.matchingIps.forEach(function (ip) { - if ('IPv4' === ip.family) { - httpsUrl = 'https://' + ip.address; - if (httpsPort !== opts.port) { - httpsUrl += ':' + opts.port; - } - console.info('\t' + httpsUrl); - } - else { - httpsUrl = 'https://[' + ip.address + ']'; - if (httpsPort !== opts.port) { - httpsUrl += ':' + opts.port; - } - console.info('\t' + httpsUrl); - } - }); - } - else if (!opts.tunnel) { - console.info("External IP address does not match local IP address."); - console.info("Use --tunnel to allow the people of the Internet to access your server."); - } - - if (opts.tunnel) { - require('../lib/tunnel.js').create(opts, servers); - } - else if (opts.ddns) { - require('../lib/ddns.js').create(opts, servers); - } - - Object.keys(opts.ifaces).forEach(function (iname) { - var iface = opts.ifaces[iname]; - - if (iface.ipv4.length) { - console.info(''); - console.info(iname + ':'); - - httpsUrl = 'https://' + iface.ipv4[0].address; - if (httpsPort !== opts.port) { - httpsUrl += ':' + opts.port; - } - console.info('\t' + httpsUrl); - - if (iface.ipv6.length) { - httpsUrl = 'https://[' + iface.ipv6[0].address + ']'; - if (httpsPort !== opts.port) { - httpsUrl += ':' + opts.port; - } - console.info('\t' + httpsUrl); - } - } - }); - - console.info(''); - }); - }); - }); - } - - run(); + return { + emit: function (type, conn) { + createConnection(conn); + } + }; +}; diff --git a/lib/worker.js b/lib/worker.js index 87f9338..84569e1 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -4,6 +4,7 @@ process.on('message', function (conf) { var deps = { messenger: process + , net: require('net') }; require('./goldilocks.js').create(deps, conf); });