Merge branch 'v1'

# Conflicts:
#	README.md
#	etc/goldilocks/goldilocks.example.yml

.gitignore

@@ -1,6 +1,7 @@
 *session*
 *secret*
 var/*
+packages/assets/org.oauth3
 
 # Logs
 logs

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "packages/assets/org.oauth3"]
-  path = packages/assets/org.oauth3
-  url = git@git.daplie.com:OAuth3/oauth3.js.git

.jshintrc

@@ -13,4 +13,5 @@
 , "latedef": true
 , "curly": true
 , "trailing": true
+, "esversion": 6
 }

API.md

@@ -0,0 +1,171 @@
+# API
+The API system is intended for use with Desktop and Mobile clients.
+It must be accessed using one of the following domains as the Host header:
+
+* localhost.alpha.daplie.me
+* localhost.admin.daplie.me
+* alpha.localhost.daplie.me
+* admin.localhost.daplie.me
+* localhost.daplie.invalid
+
+All requests require an OAuth3 token in the request headers.
+
+## Tokens
+
+Some of the functionality of goldilocks requires the use of OAuth3 tokens to
+perform tasks like setting DNS records. Management of these tokens can be done
+using the following APIs.
+
+### Get A Single Token
+  * **URL** `/api/goldilocks@daplie.com/tokens/:id`
+  * **Method** `GET`
+  * **Reponse**: The token matching the specified ID. Has the following properties.
+    * `id`: The hash used to identify the token. Based on several of the fields
+      inside the decoded token.
+    * `provider_uri`: The URI for the one who issued the token. Should be the same
+      as the `iss` field inside the decoded token.
+    * `client_uri`: The URI for the app authorized to use the token. Should be the
+      same as the `azp` field inside the decoded token.
+    * `scope`: The list of permissions granted by the token. Should be the same
+      as the `scp` field inside the decoded token.
+    * `access_token`: The encoded JWT.
+    * `token`: The decoded token.
+
+### Get All Tokens
+  * **URL** `/api/goldilocks@daplie.com/tokens`
+  * **Method** `GET`
+  * **Reponse**: An array of the tokens stored. Each item looks the same as if it
+    had been requested individually.
+
+### Save New Token
+  * **URL** `/api/goldilocks@daplie.com/tokens`
+  * **Method** `POST`
+  * **Body**: An object similar to an OAuth3 session used by the javascript
+    library. The only important fields are `refresh_token` or `access_token`, and
+    `refresh_token` will be used before `access_token`. (This is because the
+    `access_token` usually expires quickly, making it meaningless to store.)
+  * **Reponse**: The response looks the same as a single GET request.
+
+### Delete Token
+  * **URL** `/api/goldilocks@daplie.com/tokens/:id`
+  * **Method** `DELETE`
+  * **Reponse**: Either `{"success":true}` or `{"success":false}`, depending on
+    whether the token was present before the request.
+
+## Config
+
+### Get All Settings
+  * **URL** `/api/goldilocks@daplie.com/config`
+  * **Method** `GET`
+  * **Reponse**: The JSON representation of the current config. See the [README.md](/README.md)
+    for the structure of the config.
+
+### Get Group Setting
+  * **URL** `/api/goldilocks@daplie.com/config/:group`
+  * **Method** `GET`
+  * **Reponse**: The sub-object of the config relevant to the group specified in
+    the url (ie http, tls, tcp, etc.)
+
+### Get Group Module List
+  * **URL** `/api/goldilocks@daplie.com/config/:group/modules`
+  * **Method** `GET`
+  * **Reponse**: The list of modules relevant to the group specified in the url
+    (ie http, tls, tcp, etc.)
+
+### Get Specific Module
+  * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId`
+  * **Method** `GET`
+  * **Reponse**: The module with the specified module ID.
+
+### Get Domain Group
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId`
+  * **Method** `GET`
+  * **Reponse**: The domains specification with the specified domains ID.
+
+### Get Domain Group Modules
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules`
+  * **Method** `GET`
+  * **Reponse**: An object containing all of the relevant modules for the group
+    of domains.
+
+### Get Domain Group Module Category
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group`
+  * **Method** `GET`
+  * **Reponse**: A list of the specific category of modules for the group of domains.
+
+### Get Specific Domain Group Module
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId`
+  * **Method** `GET`
+  * **Reponse**: The module with the specified module ID.
+
+
+### Change Settings
+  * **URL** `/api/goldilocks@daplie.com/config`
+  * **URL** `/api/goldilocks@daplie.com/config/:group`
+  * **Method** `POST`
+  * **Body**: The changes to be applied on top of the current config. See the
+    [README.md](/README.md) for the settings. If modules or domains are specified
+    they are added to the current list.
+  * **Reponse**: The current config. If the group is specified in the URL it will
+    only be the config relevant to that group.
+
+### Add Module
+  * **URL** `/api/goldilocks@daplie.com/config/:group/modules`
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group`
+  * **Method** `POST`
+  * **Body**: The module to be added. Can also be provided an array of modules
+    to add multiple modules in the same request.
+  * **Reponse**: The current list of modules.
+
+### Add Domain Group
+  * **URL** `/api/goldilocks@daplie.com/config/domains`
+  * **Method** `POST`
+  * **Body**: The domains names and modules for the new domain group(s).
+  * **Reponse**: The current list of domain groups.
+
+
+### Edit Module
+  * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId`
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId`
+  * **Method** `PUT`
+  * **Body**: The new parameters for the module.
+  * **Reponse**: The editted module.
+
+### Edit Domain Group
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId`
+  * **Method** `PUT`
+  * **Body**: The new domains names for the domains group. The module list cannot
+    be editted through this route.
+  * **Reponse**: The editted domain group.
+
+
+### Remove Module
+  * **URL** `/api/goldilocks@daplie.com/config/:group/modules/:modId`
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId/modules/:group/:modId`
+  * **Method** `DELETE`
+  * **Reponse**: The list of modules.
+
+### Remove Domain Group
+  * **URL** `/api/goldilocks@daplie.com/config/domains/:domId`
+  * **Method** `DELETE`
+  * **Reponse**: The list of domain groups.
+
+
+## Socks5 Proxy
+
+### Check Status
+  * **URL** `/api/goldilocks@daplie.com/socks5`
+  * **Method** `GET`
+  * **Response**: The returned object will have up to two values inside
+    * `running`: boolean value to indicate if the proxy is currently active
+    * `port`: if the proxy is running this is the port it's running on
+
+### Start Proxy
+  * **URL** `/api/goldilocks@daplie.com/socks5`
+  * **Method** `POST`
+  * **Response**: Same response as for the `GET` request
+
+### Stop Proxy
+  * **URL** `/api/goldilocks@daplie.com/socks5`
+  * **Method** `DELETE`
+  * **Response**: Same response as for the `GET` request

CHANGELOG

@@ -0,0 +1,12 @@
+v1.1.5 - Implemented dns-01 ACME challenges
+
+v1.1.4 - Improved responsiveness to config updates
+  * changed which TCP/UDP ports are bound to on config update
+  * update tunnel server settings on config update
+  * update socks5 setting on config update
+
+v1.1.3 - Better late than never... here's some stuff we've got
+  * fixed (probably) network settings not being readable
+  * supports timeouts in loopback check
+  * loopback check less likely to fail / throw errors, will try again
+  * supports ddns using audience of token

LICENSE

@@ -0,0 +1,41 @@
+Copyright 2017 Daplie, Inc
+
+This is open source software; you can redistribute it and/or modify it under the
+terms of either:
+
+   a) the "MIT License"
+   b) the "Apache-2.0 License"
+
+MIT License
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+
+Apache-2.0 License Summary
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE.txt

@@ -1,3 +0,0 @@
-Hello all. We make our source code available to view, but we retain copyright.
-
-It's not because we're trying to be mean or anything, we just want to maintain our distribution channel.

README.md

@@ -1,45 +1,70 @@
-<!-- BANNER_TPL_BEGIN -->
-
-About Daplie: We're taking back the Internet!
---------------
-
-Down with Google, Apple, and Facebook!
-
-We're re-decentralizing the web and making it read-write again - one home cloud system at a time.
-
-Tired of serving the Empire? Come join the Rebel Alliance:
-
-<a href="mailto:jobs@daplie.com">jobs@daplie.com</a> | [Invest in Daplie on Wefunder](https://daplie.com/invest/) | [Pre-order Cloud](https://daplie.com/preorder/), The World's First Home Server for Everyone
-
-<!-- BANNER_TPL_END -->
-
 Goldilocks
 ==========
 
-The node.js webserver that's just right.
+The node.js netserver that's just right.
 
+* **HTTPS Web Server** with Automatic TLS (SSL) via ACME ([Let's Encrypt](https://letsencrypt.org))
+  * Static Web Server
+  * URL Redirects
+  * SSL on localhost (with bundled localhost.daplie.me certificates)
+  * Uses node cluster to take advantage of multiple CPUs (in progress)
+* **TLS** name-based (SNI) proxy
+* **TCP** port-based proxy
+* WS **Tunnel Server** (i.e. run on Digital Ocean and expose a home-firewalled Raspberry Pi to the Internet)
+* WS **Tunnel Client** (i.e. run on a Raspberry Pi and connect to a Daplie Tunnel)
+* UPnP / NAT-PMP forwarding and loopback testing (in progress)
+* Configurable via API
+* mDNS Discoverable (configure in home or office with mobile and desktop apps)
+* OAuth3 Authentication
 
-A simple HTTPS static file server with valid TLS (SSL) certs.
-
-Comes bundled a valid certificate for localhost.daplie.me,
-which is great for testing and development, and you can specify your own.
-
-Also great for testing ACME certs from letsencrypt.org.
-
-Install
+Install Standalone
 -------
 
+### curl | bash
+
 ```bash
-# v2 in npm
-npm install -g goldilocks
-
-# master in git (via ssh)
-npm install -g git+ssh://git@git.daplie.com:Daplie/goldilocks.js#v1
-
-# master in git (unauthenticated)
-npm install -g git+https://git@git.daplie.com:Daplie/goldilocks.js#v1
+curl -fsSL https://git.coolaj86.com/coolaj86/goldilocks.js/raw/v1.1/installer/get.sh | bash
 ```
 
+### git
+
+```bash
+git clone https://git.coolaj86.com/coolaj86/goldilocks.js
+pushd goldilocks.js
+git checkout v1.1
+bash installer/install.sh
+```
+
+### npm
+
+```bash
+# v1 in git (unauthenticated)
+npm install -g git+https://git@git.coolaj86.com:coolaj86/goldilocks.js#v1
+
+# v1 in git (via ssh)
+npm install -g git+ssh://git@git.coolaj86.com:coolaj86/goldilocks.js#v1
+
+# v1 in npm
+npm install -g goldilocks@v1
+```
+
+### Uninstall
+
+Remove goldilocks and services:
+
+```
+rm -rf /opt/goldilocks/ /srv/goldilocks/ /var/goldilocks/ /var/log/goldilocks/ /etc/tmpfiles.d/goldilocks.conf /etc/systemd/system/goldilocks.service
+```
+
+Remove config as well
+
+```
+rm -rf /etc/goldilocks/ /etc/ssl/goldilocks
+```
+
+Usage
+-----
+
 ```bash
 goldilocks
 ```
@@ -48,123 +73,581 @@ goldilocks
 Serving /Users/foo/ at https://localhost.daplie.me:8443
 ```
 
-With service support for
+Install as a System Service (daemon-mode)
 
-* systemd
-* launchd
+We have service support for
+
+* systemd (Linux, Ubuntu)
+* launchd (macOS)
 
 ```bash
-curl https://git.daplie.com/Daplie/goldilocks.js/raw/master/install.sh | bash
+curl https://git.coolaj86.com/coolaj86/goldilocks.js/raw/master/install.sh | bash
 ```
 
-Usage
+Modules & Configuration
 -----
 
-Examples:
+Goldilocks has several core systems, which all have their own configuration and
+some of which have modules:
 
-```
-# Install
-npm install -g git+https://git@git.daplie.com:Daplie/goldilocks.js
+* [http](#http)
+  - [proxy (reverse proxy)](#httpproxy-how-to-reverse-proxy-ruby-python-etc)
+  - [static](#httpstatic-how-to-serve-a-web-page)
+  - [redirect](#httpredirect-how-to-redirect-urls)
+* [tls](#tls)
+  - [proxy (reverse proxy)](#tlsproxy)
+  - [acme](#tlsacme)
+* [tcp](#tcp)
+  - [proxy](#tcpproxy)
+  - [forward](#tcpforward)
+* [udp](#udp)
+  - [forward](#udpforward)
+* [domains](#domains)
+* [tunnel_server](#tunnel_server)
+* [DDNS](#ddns)
+* [tunnel_client](#tunnel)
+* [mDNS](#mdns)
+* [socks5](#socks5)
+* api
 
-# Use tunnel
-goldilocks --sites jane.daplie.me --agree-tos --email jane@example.com --tunnel
+All modules require a `type` and an `id`, and any modules not defined inside the
+`domains` system also require a `domains` field (with the exception of the `forward`
+modules that require the `ports` field).
 
-# BEFORE you access in a browser for the first time, use curl
-# (because there's a concurrency bug in the greenlock setup)
-curl https://jane.daplie.me
+### http
+
+The HTTP system handles plain http (TLS / SSL is handled by the tls system)
+
+Example config:
+```yml
+http:
+  trust_proxy: true                 # allow localhost, 192.x, 10.x, 172.x, etc to set headers
+  allow_insecure: false             # allow non-https even without proxy https headers
+  primary_domain: example.com       # attempts to access via IP address will redirect here
+
+  # An array of modules that define how to handle incoming HTTP requests
+  modules:
+    - type: static
+      domains:
+        - example.com
+      root: /srv/www/:hostname
 ```
 
-Options:
+### http.proxy - how to reverse proxy (ruby, python, etc)
 
-* `-p <port>` - i.e. `sudo goldilocks -p 443` (defaults to 80+443 or 8443)
-* `-d <dirpath>` - i.e. `goldilocks -d /tmp/` (defaults to `pwd`)
-  * you can use `:hostname` as a template for multiple directories
-  * Example A: `goldilocks -d /srv/www/:hostname --sites localhost.foo.daplie.me,localhost.bar.daplie.me`
-  * Example B: `goldilocks -d ./:hostname/public/ --sites localhost.foo.daplie.me,localhost.bar.daplie.me`
-* `-c <content>` - i.e. `server-https -c 'Hello, World! '` (defaults to directory index)
-* `--express-app <path>` - path to a file the exports an express-style app (`function (req, res, next) { ... }`)
-* `--livereload` - inject livereload into all html pages (see also: [fswatch](http://stackoverflow.com/a/13807906/151312)), but be careful if `<dirpath>` has thousands of files it will spike your CPU usage to 100%
-
-* `--email <email>` - email to use for Let's Encrypt, Daplie DNS, Daplie Tunnel
-* `--agree-tos` - agree to terms for Let's Encrypt, Daplie DNS
-* `--sites <domain.tld>` comma-separated list of domains to respond to (default is `localhost.daplie.me`)
-  * optionally you may include the path to serve with `|` such as `example.com|/tmp,example.net/srv/www`
-* `--tunnel` - make world-visible (must use `--sites`)
-
-Specifying a custom HTTPS certificate:
-
-* `--key /path/to/privkey.pem` specifies the server private key
-* `--cert /path/to/fullchain.pem` specifies the bundle of server certificate and all intermediate certificates
-* `--root /path/to/root.pem` specifies the certificate authority(ies)
-
-Note: `--root` may specify single cert or a bundle, and may be used multiple times like so:
+The proxy module is for reverse proxying, typically to an application on the same machine.
+(Though it can also reverse proxy to other devices on the local network.)
 
+It has the following options:
 ```
---root /path/to/primary-root.pem --root /path/to/cross-root.pem
+address     The DNS-resolvable hostname (or IP address) and port connected by `:` to proxy the request to.
+            Takes priority over host and port if they are also specified.
+            ex: locahost:3000
+            ex: 192.168.1.100:80
+
+host        The DNS-resolvable hostname (or IP address) of the system to which the request will be proxied.
+            Defaults to localhost if only the port is specified.
+            ex: localhost
+            ex: 192.168.1.100
+
+port        The port on said system to which the request will be proxied
+            ex: 3000
+            ex: 80
 ```
 
-Other options:
+Example config:
+```yml
+http:
+  modules:
+    - type: proxy
+      domains:
+        - api.example.com
+      host: 192.168.1.100
+      port: 80
+    - type: proxy
+      domains:
+        - www.example.com
+      address: 192.168.1.16:80
+    - type: proxy
+      domains:
+        - '*'
+      port: 3000
+```
 
-* `--serve-root true` alias for `-c` with the contents of root.pem
-* `--sites example.com` changes the servername logged to the console
-* `--letsencrypt-certs example.com` sets and key, fullchain, and root to standard letsencrypt locations
+### http.static - how to serve a web page
 
-Examples
---------
+The static module is for serving static web pages and assets and has the following options:
+
+```
+root        The path to serve as a string.
+            The template variable `:hostname` represents the HTTP Host header without port information
+            ex: `root: /srv/www/example.com` would load the example.com folder for any domain listed
+            ex: `root: /srv/www/:hostname` would load `/srv/www/example.com` if so indicated by the Host header
+
+index       Set to `false` to disable the default behavior of loading `index.html` in directories
+            ex: `false`
+
+dotfiles    Set to `allow` to load dotfiles rather than ignoring them
+            ex: `"allow"`
+
+redirect    Set to `false` to disable the default behavior of ensuring that directory paths end in '/'
+            ex: `false`
+
+indexes     An array of directories which should be have indexes served rather than blocked
+            ex: `[ '/' ]` will allow all directories indexes to be served
+```
+
+Example config:
+```yml
+http:
+  modules:
+    - type: static
+      domains:
+        - example.com
+      root: /srv/www/:hostname
+```
+
+### http.redirect - how to redirect URLs
+
+The redirect module is for, you guessed it, redirecting URLs.
+
+It has the following options:
+```
+status      The HTTP status code to issue (301 is usual permanent redirect, 302 is temporary)
+            ex: 301
+
+from        The URL path that was used in the request.
+            The `*` wildcard character can be used for matching a full segment of the path
+            ex: /photos/
+            ex: /photos/*/*/
+
+to          The new URL path which should be used.
+            If wildcards matches were used they will be available as `:1`, `:2`, etc.
+            ex: /pics/
+            ex: /pics/:1/:2/
+            ex: https://mydomain.com/photos/:1/:2/
+```
+
+Example config:
+```yml
+http:
+  modules:
+    - type: proxy
+      domains:
+        - example.com
+      status: 301
+      from: /archives/*/*/*/
+      to: https://example.net/year/:1/month/:2/day/:3/
+```
+
+### tls
+
+The tls system handles encrypted connections, including fetching certificates,
+and uses ServerName Indication (SNI) to determine if the connection should be
+handled by the http system, a tls system module, or rejected.
+
+Example config:
+```yml
+tls:
+  modules:
+    - type: proxy
+      domains:
+        - example.com
+        - example.net
+      address: '127.0.0.1:6443'
+```
+
+Certificates are saved to `~/acme`, which may be `/var/www/acme` if Goldilocks is run as the www-data user.
+
+### tls.proxy
+
+The proxy module routes the traffic based on the ServerName Indication (SNI) **without decrypting** it.
+
+It has the same options as the [HTTP proxy module](#httpproxy-how-to-reverse-proxy-ruby-python-etc).
+
+Example config:
+```yml
+tls:
+  modules:
+    - type: proxy
+      domains:
+        - example.com
+      address: '127.0.0.1:5443'
+```
+
+### tls.acme
+
+The acme module defines the setting used when getting new certificates.
+
+It has the following options:
+```
+email              The email address for ACME certificate issuance
+                   ex: john.doe@example.com
+
+server             The ACME server to use
+                   ex: https://acme-v01.api.letsencrypt.org/directory
+                   ex: https://acme-staging.api.letsencrypt.org/directory
+
+challenge_type     The ACME challenge to request
+                   ex: http-01, dns-01, tls-01
+```
+
+Example config:
+```yml
+tls:
+  modules:
+    - type: acme
+      domains:
+        - example.com
+        - example.net
+      email: 'joe.shmoe@example.com'
+      server: 'https://acme-staging.api.letsencrypt.org/directory'
+      challenge_type: 'http-01'
+```
+
+**NOTE:** If you specify `dns-01` as the challenge type there must also be a
+[DDNS module](#ddns) defined for all of the relevant domains (though not all
+domains handled by a single TLS module need to be handled by the same DDNS
+module). The DDNS module provides all of the information needed to actually
+set the DNS records needed to verify ownership.
+
+### tcp
+
+The tcp system handles both *raw* and *tls-terminated* tcp network traffic
+(see the _Note_ section below the example). It may use port numbers
+or traffic sniffing to determine how the connection should be handled.
+
+It has the following options:
+```
+bind      An array of numeric ports on which to bind
+          ex: 80
+```
+
+Example Config:
+```yml
+tcp:
+  bind:
+    - 22
+    - 80
+    - 443
+  modules:
+    - type: forward
+      ports:
+        - 22
+      address: '127.0.0.1:2222'
+```
+
+_Note_: When tcp traffic comes into goldilocks it will be tested against the tcp modules.
+The connection may be handed to the TLS module if it appears to be a TLS/SSL/HTTPS connection
+and if the tls module terminates the traffic, the connection will be sent back to the TLS module.
+Due to the complexity of node.js' networking stack it is not currently possible to tell which
+port tls-terminated traffic came from, so only the SNI header (serername / domain name) may be used for
+modules matching terminated TLS.
+
+### tcp.proxy
+
+The proxy module routes traffic **after tls-termination** based on the servername (domain name)
+contained in a SNI header. As such this only works to route TCP connections wrapped in a TLS stream.
+
+It has the same options as the [HTTP proxy module](#httpproxy-how-to-reverse-proxy-ruby-python-etc).
+
+This is particularly useful for routing ssh and vpn traffic over tcp port 443 as wrapped TLS
+connections in order to access one of your servers even when connecting from a harsh or potentially
+misconfigured network environment (i.e. hotspots in public libraries and shopping malls).
+
+Example config:
+```yml
+tcp:
+  modules:
+    - type: proxy
+      domains:
+        - ssh.example.com      # Note: this domain would also listed in tls.acme.domains
+      host: localhost
+      port: 22
+    - type: proxy
+      domains:
+        - vpn.example.com      # Note: this domain would also listed in tls.acme.domains
+      host: localhost
+      port: 1194
+```
+
+_Note_: In same cases network administrators purposefully block ssh and vpn connections using
+Application Firewalls with DPI (deep packet inspection) enabled. You should read the ToS of the
+network you are connected to to ensure that you aren't subverting policies that are purposefully
+in place on such networks.
+
+#### Using with ssh
+
+In order to use this to route SSH connections you will need to use `ssh`'s
+`ProxyCommand` option. For example to use the TLS certificate for `ssh.example.com`
+to wrap an ssh connection you could use the following command:
 
 ```bash
-goldilocks -p 1443 -c 'Hello from 1443' &
-goldilocks -p 2443 -c 'Hello from 2443' &
-goldilocks -p 3443 -d /tmp &
-
-curl https://localhost.daplie.me:1443
-> Hello from 1443
-
-curl --insecure https://localhost:2443
-> Hello from 2443
-
-curl https://localhost.daplie.me:3443
-> [html index listing of /tmp]
+ssh user@example.com -o ProxyCommand='openssl s_client -quiet -connect example.com:443 -servername ssh.example.com'
 ```
 
-And if you tested <http://localhost.daplie.me:3443> in a browser,
-it would redirect to <https://localhost.daplie.me:3443> (on the same port).
+Alternatively you could add the following lines to your ssh config file.
+```
+Host example.com
+  ProxyCommand openssl s_client -quiet -connect example.com:443 -servername ssh.example.com
+```
 
-(in curl it would just show an error message)
+#### Using with OpenVPN
 
-### Testing ACME Let's Encrypt certs
+There are two strategies that will work well for you:
 
-In case you didn't know, you can get free https certificates from
-[letsencrypt.org](https://letsencrypt.org)
-(ACME letsencrypt)
-and even a free subdomain from <https://freedns.afraid.org>.
-
-If you want to quickly test the certificates you installed,
-you can do so like this:
+1) [Use ssh](https://redfern.me/tunneling-openvpn-through-ssh/) with the config above to reverse proxy tcp port 1194 to you.
 
 ```bash
-goldilocks -p 8443 \
-  --letsencrypt-certs test.mooo.com \
-  --serve-root true
+ssh -L 1194:localhost:1194 example.com
 ```
 
-which is equilavent to
+2) [Use stunnel]https://serverfault.com/questions/675553/stunnel-vpn-traffic-and-ensure-it-looks-like-ssl-traffic-on-port-443/681497)
 
-```bash
-goldilocks -p 8443 \
-  --sites test.mooo.com
-  --key /etc/letsencrypt/live/test.mooo.com/privkey.pem \
-  --cert /etc/letsencrypt/live/test.mooo.com/fullchain.pem \
-  --root /etc/letsencrypt/live/test.mooo.com/root.pem \
-  -c "$(cat 'sudo /etc/letsencrypt/live/test.mooo.com/root.pem')"
+```
+[openvpn-over-goldilocks]
+client = yes
+accept = 127.0.0.1:1194
+sni = vpn.example.com
+connect = example.com:443
 ```
 
-and can be tested like so
+3) [Use stunnel.js](https://git.coolaj86.com/coolaj86/tunnel-client.js) as described in the "tunnel_server" section below.
 
-```bash
-curl --insecure https://test.mooo.com:8443 > ./root.pem
-curl https://test.mooo.com:8843 --cacert ./root.pem
+### tcp.forward
+
+The forward module routes traffic based on port number **without decrypting** it.
+
+In addition to the same options as the [HTTP proxy module](#httpproxy-how-to-reverse-proxy-ruby-python-etc),
+the TCP forward modules also has the following options:
+
+```
+ports       A numeric array of source ports
+            ex: 22
 ```
 
-* [QuickStart Guide for Let's Encrypt](https://coolaj86.com/articles/lets-encrypt-on-raspberry-pi/)
-* [QuickStart Guide for FreeDNS](https://coolaj86.com/articles/free-dns-hosting-with-freedns-afraid-org.html)
+Example Config:
+```yml
+tcp:
+  bind:
+    - 22
+    - 80
+    - 443
+  modules:
+    - type: forward
+      ports:
+        - 22
+      port: 2222
+```
+
+### udp
+
+The udp system handles all udp network traffic. It currently only supports
+forwarding the messages without any examination.
+
+It has the following options:
+```
+bind      An array of numeric ports on which to bind
+          ex: 53
+```
+
+Example Config:
+```yml
+udp:
+  bind:
+    - 53
+  modules:
+    - type: forward
+      ports:
+        - 53
+      address: '127.0.0.1:8053'
+```
+
+### udp.forward
+
+The forward module routes traffic based on port number **without decrypting** it.
+
+It has the same options as the [TCP forward module](#tcpforward).
+
+Example Config:
+```yml
+udp:
+  bind:
+    - 53
+  modules:
+    - type: forward
+      ports:
+        - 53
+      address: '127.0.0.1:8053'
+```
+
+### domains
+
+To reduce repetition defining multiple modules that operate on the same domain
+name the `domains` field can define multiple modules of multiple types for a
+single list of names. The modules defined this way do not need to have their
+own `domains` field. Note that the [tcp.forward](#tcpforward) module is not
+allowed in a domains group since its routing is not based on domains.
+
+Example Config
+
+```yml
+domains:
+  - names:
+      - example.com
+      - www.example.com
+      - api.example.com
+    modules:
+      tls:
+        - type: acme
+          email: joe.schmoe@example.com
+          challenge_type: 'http-01'
+      http:
+        - type: redirect
+          from: /deprecated/path
+          to: /new/path
+        - type: proxy
+          port: 3000
+      dns:
+        - type: 'dns@oauth3.org'
+          token_id: user_token_id
+
+  - names:
+      - ssh.example.com
+    modules:
+      tls:
+        - type: acme
+          email: john.smith@example.com
+          challenge_type: 'http-01'
+      tcp:
+        - type: proxy
+          port: 22
+      dns:
+        - type: 'dns@oauth3.org'
+          token_id: user_token_id
+```
+
+
+
+### tunnel\_server
+
+The tunnel server system is meant to be run on a publicly accessible IP address to server tunnel clients
+which are behind firewalls, carrier-grade NAT, or otherwise Internet-connect but inaccessible devices.
+
+It has the following options:
+
+```
+secret          A 128-bit or greater string to use for signing tokens (HMAC JWT)
+                ex: abc123
+
+servernames     An array of string servernames that should be captured as the
+                tunnel server, ignoring the TLS forward module
+                ex: api.tunnel.example.com
+```
+
+Example config:
+
+```yml
+tunnel_server:
+  secret: abc123def456ghi789
+  servernames:
+    - 'api.tunnel.example.com'
+```
+
+### DDNS
+
+The DDNS module watches the network environment of the unit and makes sure the
+device is always accessible on the internet using the domains listed in the
+config. If the device has a public address or if it can automatically set up
+port forwarding the device will periodically check its public address to ensure
+the DNS records always point to it. Otherwise it will to connect to a tunnel
+server and set the DNS records to point to that server.
+
+The `loopback` setting specifies how the unit will check its public IP address
+and whether connections can reach it. Currently only `tunnel@oauth3.org` is
+supported. If the loopback setting is not defined it will default to using
+`oauth3.org`.
+
+The `tunnel` setting can be used to specify how to connect to the tunnel.
+Currently only `tunnel@oauth3.org` is supported. The token specified in the
+`tunnel` setting will be used to acquire the tokens that are used directly with
+the tunnel server. If the tunnel setting is not defined it will default to try
+using the tokens in the modules for the relevant domains.
+
+If a particular DDNS module has been disabled the device will still try to set
+up port forwarding (and connect to a tunnel if that doesn't work), but the DNS
+records will not be updated to point to the device. This is to allow a setup to
+be tested before transitioning services between devices.
+
+```yaml
+ddns:
+  disabled: false
+  loopback:
+    type: 'tunnel@oauth3.org'
+    domain: oauth3.org
+  tunnel:
+    type: 'tunnel@oauth3.org'
+    token_id: user_token_id
+  modules:
+    - type: 'dns@oauth3.org'
+      token_id: user_token_id
+      domains:
+        - www.example.com
+        - api.example.com
+        - test.example.com
+```
+
+### mDNS
+
+enabled by default
+
+Although it does not announce itself, Goldilocks is discoverable via mDNS with the special query `_cloud._tcp.local`.
+This is so that it can be easily configured via Desktop and Mobile apps when run on devices such as a Raspberry Pi or
+SOHO servers.
+
+```yaml
+mdns:
+  disabled: false
+  port: 5353
+  broadcast: '224.0.0.251'
+  ttl: 300
+```
+
+You can discover goldilocks with `mdig`.
+
+```
+npm install -g git+https://git.coolaj86.com/coolaj86/mdig.js.git
+
+mdig _cloud._tcp.local
+```
+
+### socks5
+
+Run a Socks5 proxy server.
+
+```yaml
+socks5:
+  enable: true
+  port: 1080
+```
+
+### api
+
+See [API.md](/API.md)
+
+@tigerbot: How are the APIs used (in terms of URL, Method, Headers, etc)?
+
+TODO
+----
+
+* [ ] http - nowww module
+* [ ] http - Allow match styles of `www.*`, `*`, and `*.example.com` equally
+* [ ] http - redirect based on domain name (not just path)
+* [ ] tcp - bind should be able to specify localhost, uniquelocal, private, or ip
+* [ ] tcp - if destination host is omitted default to localhost, if dst port is missing, default to src
+* [ ] sys - `curl https://coolaj86.com/goldilocks | bash -s example.com`
+* [ ] oauth3 - `example.com/.well-known/domains@oauth3.org/directives.json`
+* [ ] oauth3 - commandline questionnaire
+* [x] modules - use consistent conventions (i.e. address vs host + port)
+  * [x] tls - tls.acme vs tls.modules.acme
+* [ ] tls - forward should be able to match on source port to reach different destination ports

admin/public/js/app.js

@@ -11,7 +11,12 @@ angular.module('com.daplie.cloud', [ 'org.oauth3' ])
       return Oauth3.PromiseA.resolve(session);
     };
     var auth = Oauth3.create();
-    auth.setProvider('oauth3.org');
+    auth.setProvider('oauth3.org').then(function () {
+      auth.checkSession().then(function (session) {
+        console.log('hasSession?', session);
+      });
+    });
+
     window.oauth3 = auth; // debug
     return auth;
   } ])
@@ -139,8 +144,13 @@ angular.module('com.daplie.cloud', [ 'org.oauth3' ])
 
     vm.authenticate = function () {
       // TODO authorization redirect /api/org.oauth3.consumer/authorization_redirect/:provider_uri
+      var opts = {
+        type: 'popup'
+      , scope: 'domains,dns'
+      // , debug: true
+      };
 
-      return oauth3.authenticate().then(function (session) {
+      return oauth3.authenticate(opts).then(function (session) {
         console.info("Authorized Session", session);
 
         return oauth3.api('domains.list').then(function (domains) {
@@ -240,24 +250,15 @@ angular.module('com.daplie.cloud', [ 'org.oauth3' ])
     };
 
     vm.enableTunnel = function (/*opts*/) {
-      vm.admin.network.iface = 'oauth3-tunnel';
-
       return oauth3.request({
         method: 'POST'
       , url: 'https://' + vm.clientUri + '/api/com.daplie.goldilocks/tunnel'
-      /*
-      , data: {
-          method: 'GET'
-        , url: 'https://api.ipify.org?format=json'
-        }
-      */
+      }).then(function (result) {
+        // vm.admin.network.iface = 'oauth3-tunnel';
+        return result;
       });
     };
 
-    oauth3.checkSession().then(function (session) {
-      console.log('hasSession?', session);
-    });
-
     /*
     console.log('OAUTH3.PromiseA', OAUTH3.PromiseA);
     return oauth3.setProvider('oauth3.org').then(function () {

bin/goldilocks.js

@@ -8,137 +8,352 @@ if (!cluster.isMaster) {
   return;
 }
 
-function run(config) {
-  // TODO spin up multiple workers
-  // TODO use greenlock-cluster
-  function work() {
-    var worker = cluster.fork();
-    worker.on('exit', work).on('online', function () {
-      console.log('[worker]', worker.id, 'online');
-      // Worker is listening
-      worker.send(config);
-    });
-  }
-  console.log('config.tcp.bind', config.tcp.bind);
-  work();
-}
-
-function readConfigAndRun(args) {
-  var fs = require('fs');
-  var path = require('path');
-  var cwd = args.cwd || process.cwd();
-  var text;
-  var filename;
-  var config;
-
-  if (args.config) {
-    filename = path.resolve(cwd, args.config);
-    text = fs.readFileSync(filename, 'utf8');
-  }
-  else {
-    filename = path.resolve(cwd, 'goldilocks.yml');
-
-    if (fs.existsSync(filename)) {
-      text = fs.readFileSync(filename, 'utf8');
+var crypto = require('crypto');
+var PromiseA = require('bluebird');
+var fs = PromiseA.promisifyAll(require('fs'));
+var configStorage;
+function mergeSettings(orig, changes) {
+  Object.keys(changes).forEach(function (key) {
+    // TODO: use an API that can properly handle updating arrays.
+    if (!changes[key] || (typeof changes[key] !== 'object') || Array.isArray(changes[key])) {
+      orig[key] = changes[key];
+    }
+    else if (!orig[key] || typeof orig[key] !== 'object') {
+      orig[key] = changes[key];
     }
     else {
-      filename = path.resolve(cwd, 'goldilocks.json');
-      if (fs.existsSync(filename)) {
-        text = fs.readFileSync(filename, 'utf8');
+      mergeSettings(orig[key], changes[key]);
+    }
+  });
+}
+
+function fixRawConfig(config) {
+  var updated = false;
+
+  // First converge all of the `bind` properties for protocols that are on top
+  // of TCP to `tcp.bind`.
+  if (config.tcp && config.tcp.bind && !Array.isArray(config.tcp.bind)) {
+    config.tcp.bind = [ config.tcp.bind ];
+    updated = true;
+  }
+  if (config.http && config.http.bind) {
+    config.tcp = config.tcp || { bind: [] };
+    config.tcp.bind = (config.tcp.bind || []).concat(config.http.bind);
+    delete config.http.bind;
+    updated = true;
+  }
+  if (config.tls && config.tls.bind) {
+    config.tcp = config.tcp || { bind: [] };
+    config.tcp.bind = (config.tcp.bind || []).concat(config.tls.bind);
+    delete config.tls.bind;
+    updated = true;
+  }
+
+  // Then we rename dns to udp since the only thing we currently do with those
+  // modules is proxy the packets without inspecting them at all.
+  if (config.dns) {
+    config.udp = config.dns;
+    delete config.dns;
+    updated = true;
+  }
+  // Convert all 'proxy' UDP modules to 'forward' modules that specify which
+  // incoming ports are relevant. Primarily to make 'proxy' modules consistent
+  // in needing relevant domain names.
+  if (config.udp && !Array.isArray(config.udp.bind)) {
+    config.udp.bind = [].concat(config.udp.bind || []);
+    updated = true;
+  }
+  if (config.udp && config.udp.modules) {
+    if (!config.udp.bind.length || !Array.isArray(config.udp.modules)) {
+      delete config.udp.modules;
+      updated = true;
+    } else {
+      config.udp.modules.forEach(function (mod) {
+        if (mod.type === 'proxy') {
+          mod.type = 'forward';
+          mod.ports = config.udp.bind.slice();
+          updated = true;
+        }
+      });
+    }
+  }
+
+  // This we take the old way of defining ACME options and put them into a tls module.
+  if (config.tls) {
+    var oldPropMap = {
+      email:              'email'
+    , acme_directory_url: 'server'
+    , challenge_type:     'challenge_type'
+    , servernames:        'approved_domains'
+    };
+    if (Object.keys(oldPropMap).some(config.tls.hasOwnProperty, config.tls)) {
+      updated = true;
+      if (config.tls.acme) {
+        console.warn('TLS config has `acme` field and old style definitions');
       } else {
-        text = '{}';
+        config.tls.acme = {};
+        Object.keys(oldPropMap).forEach(function (oldKey) {
+          if (config.tls[oldKey]) {
+            config.tls.acme[oldPropMap[oldKey]] = config.tls[oldKey];
+          }
+        });
       }
     }
+    if (config.tls.acme) {
+      updated = true;
+      config.tls.acme.domains = config.tls.acme.approved_domains;
+      delete config.tls.acme.approved_domains;
+      config.tls.modules = config.tls.modules || [];
+      config.tls.modules.push(Object.assign({}, config.tls.acme, {type: 'acme'}));
+      delete config.tls.acme;
+    }
+  }
+
+  // Then we make sure all modules have an ID and type, and makes sure all domains
+  // are in the right spot and also have an ID.
+  function updateModules(list) {
+    if (!Array.isArray(list)) {
+      return;
+    }
+    list.forEach(function (mod) {
+      if (!mod.id) {
+        mod.id = crypto.randomBytes(4).toString('hex');
+        updated = true;
+      }
+      if (mod.name) {
+        mod.type = mod.type || mod.name;
+        delete mod.name;
+        updated = true;
+      }
+    });
+  }
+  function moveDomains(name) {
+    if (!config[name].domains) {
+      return;
+    }
+    updated = true;
+    var domList = config[name].domains;
+    delete config[name].domains;
+
+    if (!Array.isArray(domList)) {
+      return;
+    }
+    if (!Array.isArray(config.domains)) {
+      config.domains = [];
+    }
+    domList.forEach(function (dom) {
+      updateModules(dom.modules);
+
+      var strDoms = dom.names.slice().sort().join(',');
+      var added = config.domains.some(function (existing) {
+        if (strDoms !== existing.names.slice().sort().join(',')) {
+          return;
+        }
+        existing.modules = existing.modules || {};
+        existing.modules[name] = (existing.modules[name] || []).concat(dom.modules);
+        return true;
+      });
+      if (added) {
+        return;
+      }
+
+      var newDom = {
+        id:    crypto.randomBytes(4).toString('hex')
+      , names: dom.names
+      , modules: {}
+      };
+      newDom.modules[name] = dom.modules;
+      config.domains.push(newDom);
+    });
+  }
+
+  [ 'udp', 'tcp', 'http', 'tls' ].forEach(function (key) {
+    if (!config[key]) {
+      return;
+    }
+    updateModules(config[key].modules);
+    moveDomains(key);
+  });
+
+  return updated;
+}
+async function createStorage(filename, filetype) {
+  var recase = require('recase').create({});
+  var snakeCopy = recase.snakeCopy.bind(recase);
+  var camelCopy = recase.camelCopy.bind(recase);
+
+  var parse, dump;
+  if (filetype === 'json') {
+    parse = JSON.parse;
+    dump  = function (arg) { return JSON.stringify(arg, null, '  '); };
+  } else {
+    var yaml = require('js-yaml');
+    parse = function (text) { return yaml.safeLoad(text) || {}; };
+    dump  = yaml.safeDump;
+  }
+
+  async function read() {
+    var text;
+    try {
+      text = await fs.readFileAsync(filename);
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        return {};
+      }
+      throw err;
+    }
+
+    var rawConfig = parse(text);
+    if (fixRawConfig(rawConfig)) {
+      await fs.writeFileAsync(filename, dump(rawConfig));
+      text = await fs.readFileAsync(filename);
+      rawConfig = parse(text);
+    }
+
+    return rawConfig;
+  }
+
+  var result = {
+    read: function () {
+      return read().then(camelCopy);
+    }
+  , save: function (changes) {
+      if (!changes || typeof changes !== 'object' || Array.isArray(changes)) {
+        return PromiseA.reject(new Error('invalid config'));
+      }
+      changes = snakeCopy(changes);
+      return read()
+        .then(snakeCopy)
+        .then(function (current) {
+          mergeSettings(current, changes);
+          // TODO: validate/lint the config before we actually write it.
+          return dump(current);
+        })
+        .then(function (newText) {
+          return fs.writeFileAsync(filename, newText);
+        })
+        .then(function () {
+          return result.read();
+        })
+        ;
+    }
+  };
+  return result;
+}
+async function checkConfigLocation(cwd, configFile) {
+  cwd = cwd || process.cwd();
+  var path = require('path');
+  var filename, text;
+
+  if (configFile) {
+    filename = path.resolve(cwd, configFile);
+    try {
+      text = await fs.readFileAsync(filename);
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+      if (path.extname(filename) === '.json') {
+        return { name: filename, type: 'json' };
+      } else {
+        return { name: filename, type: 'yaml' };
+      }
+    }
+  } else {
+    // Note that `path.resolve` can handle both relative and absolute paths.
+    var defLocations = [
+      path.resolve(cwd, 'goldilocks.yml')
+    , path.resolve(cwd, 'goldilocks.json')
+    , path.resolve(cwd, 'etc/goldilocks/goldilocks.yml')
+    , '/etc/goldilocks/goldilocks.yml'
+    ];
+
+    var ind;
+    for (ind = 0; ind < defLocations.length; ind += 1) {
+      try {
+        text = await fs.readFileAsync(defLocations[ind]);
+        filename = defLocations[ind];
+        break;
+      } catch (err) {
+        if (err.code !== 'ENOENT') {
+          throw err;
+        }
+      }
+    }
+
+    if (!filename) {
+      filename = defLocations[0];
+      text = '';
+    }
   }
 
   try {
-    config = JSON.parse(text);
-  } catch(e) {
-    try {
-      config = require('js-yaml').safeLoad(text);
-      // blank config file
-      if ('undefined' === typeof config) {
-        config = {};
-      }
-    } catch(e) {
-      throw new Error(
-        "Could not load '" + filename + "' as JSON nor YAML"
-      );
+    JSON.parse(text);
+    return { name: filename, type: 'json' };
+  } catch (err) {}
+
+  try {
+    require('js-yaml').safeLoad(text);
+    return { name: filename, type: 'yaml' };
+  } catch (err) {}
+
+  throw new Error('Could not load "' + filename + '" as JSON nor YAML');
+}
+async function createConfigStorage(args) {
+  var result = await checkConfigLocation(args.cwd, args.config);
+  console.log('config file', result.name, 'is of type', result.type);
+  configStorage = await createStorage(result.name, result.type);
+  return configStorage.read();
+}
+
+var tcpProm;
+function fillConfig(config, args) {
+  config.debug = config.debug || args.debug;
+
+  config.socks5 = config.socks5 || { enabled: false };
+
+  // Use Object.assign to copy any real config values over the default values so we can
+  // easily make sure all the fields we need exist .
+  var mdnsDefaults = { disabled: false, port: 5353, broadcast: '224.0.0.251', ttl: 300 };
+  config.mdns = Object.assign(mdnsDefaults, config.mdns);
+
+  if (!Array.isArray(config.domains)) {
+    config.domains = [];
+  }
+
+  function fillComponent(name, fillBind) {
+    if (!config[name]) {
+      config[name] = {};
     }
-  }
-
-  var recase = require('recase').create({});
-  config = recase.camelCopy(config);
-
-  if (!config.dns) {
-    config.dns = { proxy: { port: 3053 } };
-  }
-  if (!config.tcp) {
-    config.tcp = {};
-  }
-  if (!config.http) {
-    config.http = { proxy: { port: 3000 } };
-  }
-  if (!config.tls) {
-    config.tls = {
-      agreeTos: args.agreeTos || args.agree || args['agree-tos']
-    , servernames: (args.servernames||'').split(',').filter(Boolean).map(function (str) { return str.toLowerCase(); })
-    };
-  }
-  if (args.email) {
-    config.email = args.email;
-    config.tls.email = args.email;
-  }
-
-  // maybe this should not go in config... but be ephemeral in some way?
-  if (args.cwd) {
-    config.cwd = args.cwd;
-  }
-  if (!config.cwd) {
-    config.cwd = process.cwd();
-  }
-
-  var ipaddr = require('ipaddr.js');
-  var addresses = [];
-  var ifaces = require('../lib/local-ip.js').find();
-
-  Object.keys(ifaces).forEach(function (ifacename) {
-    var iface = ifaces[ifacename];
-    iface.ipv4.forEach(function (ip) {
-      addresses.push(ip);
-    });
-    iface.ipv6.forEach(function (ip) {
-      addresses.push(ip);
-    });
-  });
-
-  addresses.sort(function (a, b) {
-    if (a.family !== b.family) {
-      return 'IPv4' === a.family ? 1 : -1;
+    if (!Array.isArray(config[name].modules)) {
+      config[name].modules = [];
     }
 
-    return a.address > b.address ? 1 : -1;
-  });
+    if (fillBind && !Array.isArray(config[name].bind)) {
+      config[name].bind = [];
+    }
+  }
+  fillComponent('udp',   true);
+  fillComponent('tcp',   true);
+  fillComponent('http',  false);
+  fillComponent('tls',   false);
+  fillComponent('ddns',  false);
 
-  addresses.forEach(function (addr) {
-    addr.range = ipaddr.parse(addr.address).range();
-  });
+  config.device = { hostname: require('os').hostname() };
 
-  // TODO maybe move to config.state.addresses (?)
-  config.addresses = addresses;
-  config.device = { hostname: 'daplien-pod' };
+  if (Array.isArray(config.tcp.bind) && config.tcp.bind.length) {
+    return PromiseA.resolve(config);
+  }
 
-  var PromiseA = require('bluebird');
-  var tcpProm, dnsProm;
-
-  if (config.tcp.bind) {
-    tcpProm = PromiseA.resolve();
-  } else {
+  // We need to make sure we only check once, because even though our workers can
+  // all bind on the same port witout issue we cannot. This will lead to failure
+  // to determine which ports will work once the first worker starts.
+  if (!tcpProm) {
     tcpProm = new PromiseA(function (resolve, reject) {
       require('../lib/check-ports').checkTcpPorts(function (failed, bound) {
-        config.tcp.bind = Object.keys(bound);
-        if (config.tcp.bind.length) {
-          resolve();
+        var result = Object.keys(bound).map(Number);
+        if (result.length > 0) {
+          resolve(result);
         } else {
           reject(failed);
         }
@@ -146,75 +361,98 @@ function readConfigAndRun(args) {
     });
   }
 
-  if (config.dns.bind) {
-    dnsProm = PromiseA.resolve();
-  } else {
-    dnsProm = new PromiseA(function (resolve) {
-      require('../lib/check-ports').checkUdpPorts(function (failed, bound) {
-        var ports = Object.keys(bound);
+  return tcpProm.then(
+    function (bound) {
+      config.tcp.bind = bound;
+      return config;
+    }, function (failed) {
+      Object.keys(failed).forEach(function (key) {
+        console.log('[error bind]', key, failed[key].code);
+      });
+      return PromiseA.reject(new Error("could not bind to the default ports"));
+    });
+}
 
-        if (ports.length === 0) {
-          // I don't think we want to prevent the rest of the app from running in
-          // this case like we do for TCP, do don't call reject.
-          console.warn('could not bind to the desired ports for DNS');
-          Object.keys(failed).forEach(function (key) {
-            console.log('[error bind]', key, failed[key].code);
-          });
-        }
-        else if (ports.length === 1) {
-          config.dns.bind = parseInt(ports[0], 10);
-        }
-        else {
-          config.dns.bind = ports.map(function (numStr) {
-            return parseInt(numStr, 10);
-          });
-        }
+function run(args) {
+  var workers = {};
+  var cachedConfig;
 
-        resolve();
+  function updateConfig(config) {
+    fillConfig(config, args).then(function (config) {
+      cachedConfig = config;
+      console.log('changed config', config);
+      Object.keys(workers).forEach(function (key) {
+        workers[key].send(cachedConfig);
       });
     });
   }
 
-  PromiseA.all([tcpProm, dnsProm])
-    .then(function () {
-      run(config);
-    })
-    .catch(function (failed) {
-      console.warn("could not bind to the desired ports");
-      Object.keys(failed).forEach(function (key) {
-        console.log('[error bind]', key, failed[key].code);
-      });
+  process.on('SIGHUP', function () {
+    configStorage.read().then(updateConfig).catch(function (err) {
+      console.error('error updating config after SIGHUP', err);
     });
+  });
+
+  cluster.on('message', function (worker, message) {
+    if (message.type !== 'com.daplie.goldilocks/config') {
+      return;
+    }
+    configStorage.save(message.changes).then(updateConfig).catch(function (err) {
+      console.error('error changing config', err);
+    });
+  });
+
+  cluster.on('online', function (worker) {
+    console.log('[worker]', worker.id, 'online');
+    workers[worker.id] = worker;
+    // Worker is listening
+    worker.send(cachedConfig);
+  });
+
+  cluster.on('exit', function (worker) {
+    delete workers[worker.id];
+    cluster.fork();
+  });
+
+  createConfigStorage(args)
+    .then(function (config) {
+      return fillConfig(config, args);
+    })
+    .then(function (config) {
+      console.log('config.tcp.bind', config.tcp.bind);
+      cachedConfig = config;
+      // TODO spin up multiple workers
+      // TODO use greenlock-cluster
+      cluster.fork();
+    }).catch(function (err) {
+      console.error(err);
+      process.exit(1);
+    })
+    ;
 }
 
 function readEnv(args) {
   // TODO
+  try {
+    if (process.env.GOLDILOCKS_HOME) {
+      process.chdir(process.env.GOLDILOCKS_HOME);
+    }
+  } catch (err) {}
+
   var env = {
-    tunnel: process.env.GOLDILOCKS_TUNNEL_TOKEN || process.env.GOLDILOCKS_TUNNEL && true
-  , email: process.env.GOLDILOCKS_EMAIL
-  , cwd: process.env.GOLDILOCKS_HOME
+    cwd: process.env.GOLDILOCKS_HOME || process.cwd()
   , debug: process.env.GOLDILOCKS_DEBUG && true
   };
-  args.cwd = args.cwd || env.cwd;
-  Object.keys(env).forEach(function (key) {
-    if ('undefined' === typeof args[key]) {
-      args[key] = env[key];
-    }
-  });
 
-  readConfigAndRun(args);
+  run(Object.assign({}, env, args));
 }
 
 var program = require('commander');
 
 program
   .version(require('../package.json').version)
-  .option('--agree-tos [url1,url2]', "agree to all Terms of Service for Daplie, Let's Encrypt, etc (or specific URLs only)")
   .option('-c --config <file>', 'Path to config file (Goldilocks.json or Goldilocks.yml) example: --config /etc/goldilocks/Goldilocks.json')
-  .option('--tunnel [token]', 'Turn tunnel on. This will enter interactive mode for login if no token is specified.')
-  .option('--email <email>', "(Re)set default email to use for Daplie, Let's Encrypt, ACME, etc.")
   .option('--debug', "Enable debug output")
   .parse(process.argv);
 
-program.cwd = process.cwd();
 readEnv(program);

dist/etc/goldilocks/goldilocks.example.yml

@@ -0,0 +1,106 @@
+tcp:
+  bind:
+    - 22
+    - 80
+    - 443
+  modules:
+    - type: forward
+      ports:
+        - 22
+      address: '127.0.0.1:8022'
+
+udp:
+  bind:
+    - 53
+  modules:
+    - type: forward
+      ports:
+        - 53
+      port: 5353
+      # default host is localhost
+
+
+tls:
+  modules:
+    - type: proxy
+      domains:
+        - localhost.bar.daplie.me
+        - localhost.foo.daplie.me
+      address: '127.0.0.1:5443'
+    - type: acme
+      domains:
+        - '*.localhost.daplie.me'
+      email: 'guest@example.com'
+      challenge_type: 'http-01'
+
+http:
+  trust_proxy: true
+  allow_insecure: false
+  primary_domain: localhost.daplie.me
+
+  modules:
+    - type: redirect
+      domains:
+        - localhost.beta.daplie.me
+      status: 301
+      from: /old/path/*/other/*
+      to: https://example.com/path/new/:2/something/:1
+    - type: proxy
+      domains:
+        - localhost.daplie.me
+      host: localhost
+      port: 4000
+    - type: static
+      domains:
+        - '*.localhost.daplie.me'
+      root: '/srv/www/:hostname'
+
+domains:
+  - names:
+      - localhost.gamma.daplie.me
+    modules:
+      tls:
+        - type: proxy
+          port: 6443
+  - names:
+      - beta.localhost.daplie.me
+      - baz.localhost.daplie.me
+    modules:
+      tls:
+        - type: acme
+          email: 'owner@example.com'
+          challenge_type: 'tls-sni-01'
+          # default server is 'https://acme-v01.api.letsencrypt.org/directory'
+      http:
+        - type: redirect
+          from: /nowhere/in/particular
+          to: /just/an/example
+        - type: proxy
+          address: '127.0.0.1:3001'
+
+
+mdns:
+  disabled: false
+  port: 5353
+  broadcast: '224.0.0.251'
+  ttl: 300
+
+tunnel_server:
+  secret: abc123
+  servernames:
+    - 'tunnel.localhost.com'
+
+ddns:
+  loopback:
+    type: 'tunnel@oauth3.org'
+    domain: oauth3.org
+  tunnel:
+    type: 'tunnel@oauth3.org'
+    token: user_token_id
+  modules:
+    - type: 'dns@oauth3.org'
+      token: user_token_id
+      domains:
+        - www.example.com
+        - api.example.com
+        - test.example.com

dist/etc/systemd/system/goldilocks.service

@@ -19,14 +19,14 @@ StartLimitBurst=3
 
 # User and group the process will run as
 # (www-data is the de facto standard on most systems)
-User=www-data
-Group=www-data
+User=MY_USER
+Group=MY_GROUP
 
 # If we need to pass environment variables in the future
 Environment=GOLDILOCKS_PATH=/srv/www NODE_PATH=/opt/goldilocks/lib/node_modules NPM_CONFIG_PREFIX=/opt/goldilocks
 
 # Set a sane working directory, sane flags, and specify how to reload the config file
-WorkingDirectory=/srv/www
+WorkingDirectory=/opt/goldilocks
 ExecStart=/opt/goldilocks/bin/node /opt/goldilocks/bin/goldilocks --config /etc/goldilocks/goldilocks.yml
 ExecReload=/bin/kill -USR1 $MAINPID
 
@@ -46,7 +46,7 @@ ProtectSystem=full
 # … except TLS/SSL, ACME, and Let's Encrypt certificates
 #   and /var/log/goldilocks, because we want a place where logs can go.
 #   This merely retains r/w access rights, it does not add any new. Must still be writable on the host!
-ReadWriteDirectories=/etc/goldilocks /etc/ssl /srv/www /var/log/goldilocks
+ReadWriteDirectories=/etc/goldilocks /etc/ssl /srv/www /var/log/goldilocks /opt/goldilocks
 # you may also want to add other directories such as /opt/goldilocks /etc/acme /etc/letsencrypt
 
 # Note: in v231 and above ReadWritePaths has been renamed to ReadWriteDirectories

dist/etc/tmpfiles.d/goldilocks.conf

@@ -0,0 +1,5 @@
+# /etc/tmpfiles.d/goldilocks.conf
+# See https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html
+
+# Type Path           Mode UID      GID      Age Argument
+d /run/goldilocks     0755 MY_USER  MY_GROUP -   -

etc/tmpfiles.d/goldilocks.conf

@@ -1,10 +0,0 @@
-# /etc/tmpfiles.d/goldilocks.conf
-# See https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html
-
-# Type Path           Mode UID      GID      Age Argument
-d /etc/goldilocks          0755 www-data www-data -   -
-d /opt/goldilocks          0775 www-data www-data -   -
-d /srv/www                 0775 www-data www-data -   -
-d /etc/ssl/goldilocks      0750 www-data www-data -   -
-d /var/log/goldilocks      0750 www-data www-data -   -
-#d /run/goldilocks          0755 www-data www-data -   -

goldilocks.example.yml

@@ -1,32 +0,0 @@
-tcp:
-  bind:
-    - 22
-    - 80
-    - 443
-  modules:
-    - name: forward
-      ports:
-        - 22
-      address: '127.0.0.1:8022'
-
-tls:
-  modules:
-    - name: proxy
-      domains:
-        - localhost.bar.daplie.me
-        - localhost.foo.daplie.me
-      address: '127.0.0.1:5443'
-
-http:
-  trust_proxy: true
-  allow_insecure: false
-  primary_domain: localhost.foo.daplie.me
-  modules:
-    - name: proxy
-      domains:
-        - localhost.daplie.me
-      address: '127.0.0.1:4000'
-    - name: static
-      domains:
-        - '*.localhost.daplie.me'
-      root: '/srv/www/:hostname'

install.sh

@@ -1,194 +0,0 @@
-#!/bin/bash
-
-# something or other about android and tmux using PREFIX
-#: "${PREFIX:=''}"
-MY_ROOT=""
-if [ -z "${PREFIX-}" ]; then
-  MY_ROOT=""
-else
-  MY_ROOT="$PREFIX"
-fi
-# Not every platform has or needs sudo, gotta save them O(1)s...
-sudo_cmd=""
-((EUID)) && [[ -z "$ANDROID_ROOT" ]] && sudo_cmd="sudo"
-
-###############################
-#                             #
-#         http_get            #
-# boilerplate for curl / wget #
-#                             #
-###############################
-
-# See https://git.daplie.com/Daplie/daplie-snippets/blob/master/bash/http-get.sh
-
-http_get=""
-http_opts=""
-http_out=""
-
-detect_http_get()
-{
-  if type -p curl >/dev/null 2>&1; then
-    http_get="curl"
-    http_opts="-fsSL"
-    http_out="-o"
-  elif type -p wget >/dev/null 2>&1; then
-    http_get="wget"
-    http_opts="--quiet"
-    http_out="-O"
-  else
-    echo "Aborted, could not find curl or wget"
-    return 7
-  fi
-}
-
-dap_dl()
-{
-  $http_get $http_opts $http_out "$2" "$1"
-}
-
-dap_dl_bash()
-{
-  dap_url=$1
-  #dap_args=$2
-  rm -rf dap-tmp-runner.sh
-  $http_get $http_opts $http_out dap-tmp-runner.sh "$dap_url"; bash dap-tmp-runner.sh; rm dap-tmp-runner.sh
-}
-
-detect_http_get
-
-## END HTTP_GET ##
-
-
-
-###################
-#                 #
-# Install service #
-#                 #
-###################
-
-my_app_name=goldilocks
-my_app_pkg_name=com.daplie.goldilocks.web
-my_app_dir=$(mktemp -d)
-installer_base="https://git.daplie.com/Daplie/goldilocks.js/raw/master"
-
-my_app_etc_config="etc/${my_app_name}/${my_app_name}.yml"
-my_app_systemd_service="etc/systemd/system/${my_app_name}.service"
-my_app_systemd_tmpfiles="etc/tmpfiles.d/${my_app_name}.conf"
-my_app_launchd_service="Library/LaunchDaemons/${my_app_pkg_name}.plist"
-
-install_for_systemd()
-{
-  echo ""
-  echo "Installing as systemd service"
-  echo ""
-  mkdir -p $(dirname "$my_app_dir/$my_app_systemd_service")
-  dap_dl "$installer_base/$my_app_systemd_service" "$my_app_dir/$my_app_systemd_service"
-  $sudo_cmd mv "$my_app_dir/$my_app_systemd_service" "$MY_ROOT/$my_app_systemd_service"
-  $sudo_cmd chown -R root:root "$MY_ROOT/$my_app_systemd_service"
-  $sudo_cmd chmod 644 "$MY_ROOT/$my_app_systemd_service"
-
-  mkdir -p $(dirname "$my_app_dir/$my_app_systemd_tmpfiles")
-  dap_dl "$installer_base/$my_app_systemd_tmpfiles" "$my_app_dir/$my_app_systemd_tmpfiles"
-  $sudo_cmd mv "$my_app_dir/$my_app_systemd_tmpfiles" "$MY_ROOT/$my_app_systemd_tmpfiles"
-  $sudo_cmd chown -R root:root "$MY_ROOT/$my_app_systemd_tmpfiles"
-  $sudo_cmd chmod 644 "$MY_ROOT/$my_app_systemd_tmpfiles"
-
-  $sudo_cmd systemctl stop "${my_app_name}.service" >/dev/null 2>/dev/null
-  $sudo_cmd systemctl daemon-reload
-  $sudo_cmd systemctl start "${my_app_name}.service"
-  $sudo_cmd systemctl enable "${my_app_name}.service"
-
-  echo "$my_app_name started with systemctl, check its status like so"
-  echo "  $sudo_cmd systemctl status $my_app_name"
-  echo "  $sudo_cmd journalctl -xe -u goldilocks"
-}
-
-install_for_launchd()
-{
-  echo ""
-  echo "Installing as launchd service"
-  echo ""
-  # See http://www.launchd.info/
-  mkdir -p $(dirname "$my_app_dir/$my_app_launchd_service")
-  dap_dl "$installer_base/$my_app_launchd_service" "$my_app_dir/$my_app_launchd_service"
-  $sudo_cmd mv "$my_app_dir/$my_app_launchd_service" "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd chown root:wheel "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd chmod 0644 "$MY_ROOT/$my_app_launchd_service"
-  $sudo_cmd launchctl unload -w "$MY_ROOT/$my_app_launchd_service" >/dev/null 2>/dev/null
-  $sudo_cmd launchctl load -w "$MY_ROOT/$my_app_launchd_service"
-
-  echo "$my_app_name started with launchd"
-}
-
-install_etc_config()
-{
-  if [ ! -e "$MY_ROOT/$my_app_etc_config" ]; then
-    $sudo_cmd mkdir -p $(dirname "$MY_ROOT/$my_app_etc_config")
-    mkdir -p $(dirname "$my_app_dir/$my_app_etc_config")
-    dap_dl "$installer_base/$my_app_etc_config" "$my_app_dir/$my_app_etc_config"
-    $sudo_cmd mv "$my_app_dir/$my_app_etc_config" "$MY_ROOT/$my_app_etc_config"
-  fi
-
-  # OS X
-  $sudo_cmd chown -R _www:_www $(dirname "$MY_ROOT/$my_app_etc_config") || true
-  # Linux
-  $sudo_cmd chown -R www-data:www-data $(dirname "$MY_ROOT/$my_app_etc_config") || true
-  $sudo_cmd chmod 775 $(dirname "$MY_ROOT/$my_app_etc_config")
-  $sudo_cmd chmod 664 "$MY_ROOT/$my_app_etc_config"
-}
-
-install_service()
-{
-  install_etc_config
-
-  installable=""
-  if [ -d "$MY_ROOT/etc/systemd/system" ]; then
-    install_for_systemd
-    installable="true"
-  fi
-  if [ -d "/Library/LaunchDaemons" ]; then
-    install_for_launchd
-    installable="true"
-  fi
-  if [ -z "$installable" ]; then
-    echo ""
-    echo "Unknown system service init type. You must install as a system service manually."
-    echo '(please file a bug with the output of "uname -a")'
-    echo ""
-  fi
-}
-
-## END SERVICE_INSTALL ##
-
-
-
-set -e
-set -u
-
-# Dependencies
-dap_dl_bash "https://git.daplie.com/coolaj86/node-install-script/raw/master/setup-min.sh"
-
-# Install
-export NODE_PATH=/opt/goldilocks/lib/node_modules
-export NPM_CONFIG_PREFIX=/opt/goldilocks
-$sudo_cmd mkdir -p /etc/goldilocks
-$sudo_cmd mkdir -p /var/log/goldilocks
-$sudo_cmd mkdir -p /srv/www
-$sudo_cmd mkdir -p /var/www
-$sudo_cmd mkdir -p /opt/goldilocks/{lib,bin,etc}
-# OS X
-$sudo_cmd chown _www:_www /var/www /srv/www || true
-$sudo_cmd chown -R _www:_www /opt/goldilocks/{lib,bin,etc} || true
-# Linux
-$sudo_cmd chown www-data:www-data /var/www /srv/www || true
-$sudo_cmd chown -R www-data:www-data /opt/goldilocks/{lib,bin,etc} || true
-npm install -g 'git+https://git@git.daplie.com/Daplie/goldilocks.js.git#v1'
-
-# Unistall
-dap_dl "https://git.daplie.com/Daplie/goldilocks.js/raw/master/uninstall.sh" "./goldilocks-uninstall"
-$sudo_cmd chmod 755 "./goldilocks-uninstall"
-$sudo_cmd chown root:root "./goldilocks-uninstall"
-$sudo_cmd mv "./goldilocks-uninstall" "/usr/local/bin/uninstall-goldilocks"
-
-# Install Service
-install_service

installer/get.sh

@@ -0,0 +1,20 @@
+set -e
+set -u
+
+my_name=goldilocks
+# TODO provide an option to supply my_ver and my_tmp
+my_ver=master
+my_tmp=$(mktemp -d)
+
+mkdir -p $my_tmp/opt/$my_name/lib/node_modules/$my_name
+git clone https://git.coolaj86.com/coolaj86/goldilocks.js.git $my_tmp/opt/$my_name/lib/node_modules/$my_name
+
+echo "Installing to $my_tmp (will be moved after install)"
+pushd $my_tmp/opt/$my_name/lib/node_modules/$my_name
+  git checkout $my_ver
+  source ./installer/install.sh
+popd
+
+echo "Installation successful, now cleaning up $my_tmp ..."
+rm -rf $my_tmp
+echo "Done"

installer/http-get.sh

@@ -0,0 +1,48 @@
+###############################
+#                             #
+#         http_get            #
+# boilerplate for curl / wget #
+#                             #
+###############################
+
+# See https://git.coolaj86.com/coolaj86/snippets/blob/master/bash/http-get.sh
+
+_h_http_get=""
+_h_http_opts=""
+_h_http_out=""
+
+detect_http_get()
+{
+  set +e
+  if type -p curl >/dev/null 2>&1; then
+    _h_http_get="curl"
+    _h_http_opts="-fsSL"
+    _h_http_out="-o"
+  elif type -p wget >/dev/null 2>&1; then
+    _h_http_get="wget"
+    _h_http_opts="--quiet"
+    _h_http_out="-O"
+  else
+    echo "Aborted, could not find curl or wget"
+    return 7
+  fi
+  set -e
+}
+
+http_get()
+{
+  $_h_http_get $_h_http_opts $_h_http_out "$2" "$1"
+  touch "$2"
+}
+
+http_bash()
+{
+  _http_url=$1
+  #dap_args=$2
+  rm -rf dap-tmp-runner.sh
+  $_h_http_get $_h_http_opts $_h_http_out dap-tmp-runner.sh "$_http_url"; bash dap-tmp-runner.sh; rm dap-tmp-runner.sh
+}
+
+detect_http_get
+
+## END HTTP_GET ##

installer/install-for-launchd.sh

@@ -0,0 +1,17 @@
+set -u
+
+my_app_launchd_service="Library/LaunchDaemons/${my_app_pkg_name}.plist"
+
+echo ""
+echo "Installing as launchd service"
+echo ""
+
+# See http://www.launchd.info/
+safe_copy_config "$my_app_dist/$my_app_launchd_service" "$my_root/$my_app_launchd_service"
+
+$sudo_cmd chown root:wheel "$my_root/$my_app_launchd_service"
+
+$sudo_cmd launchctl unload -w "$my_root/$my_app_launchd_service" >/dev/null 2>/dev/null
+$sudo_cmd launchctl load -w "$my_root/$my_app_launchd_service"
+
+echo "$my_app_name started with launchd"

installer/install-for-systemd.sh

@@ -0,0 +1,37 @@
+set -u
+
+my_app_systemd_service="etc/systemd/system/${my_app_name}.service"
+my_app_systemd_tmpfiles="etc/tmpfiles.d/${my_app_name}.conf"
+
+echo ""
+echo "Installing as systemd service"
+echo ""
+
+sed "s/MY_USER/$my_user/g" "$my_app_dist/$my_app_systemd_service" > "$my_app_dist/$my_app_systemd_service.2"
+sed "s/MY_GROUP/$my_group/g" "$my_app_dist/$my_app_systemd_service.2" > "$my_app_dist/$my_app_systemd_service"
+rm "$my_app_dist/$my_app_systemd_service.2"
+safe_copy_config "$my_app_dist/$my_app_systemd_service" "$my_root/$my_app_systemd_service"
+$sudo_cmd chown root:root "$my_root/$my_app_systemd_service"
+
+sed "s/MY_USER/$my_user/g" "$my_app_dist/$my_app_systemd_tmpfiles" > "$my_app_dist/$my_app_systemd_tmpfiles.2"
+sed "s/MY_GROUP/$my_group/g" "$my_app_dist/$my_app_systemd_tmpfiles.2" > "$my_app_dist/$my_app_systemd_tmpfiles"
+rm "$my_app_dist/$my_app_systemd_tmpfiles.2"
+safe_copy_config "$my_app_dist/$my_app_systemd_tmpfiles" "$my_root/$my_app_systemd_tmpfiles"
+$sudo_cmd chown root:root "$my_root/$my_app_systemd_tmpfiles"
+
+$sudo_cmd systemctl stop "${my_app_name}.service" >/dev/null 2>/dev/null || true
+$sudo_cmd systemctl daemon-reload
+$sudo_cmd systemctl start "${my_app_name}.service"
+$sudo_cmd systemctl enable "${my_app_name}.service"
+
+echo ""
+echo ""
+echo "Fun systemd commands to remember:"
+echo "  $sudo_cmd systemctl daemon-reload"
+echo "  $sudo_cmd systemctl restart $my_app_name.service"
+echo ""
+echo "$my_app_name started with systemctl, check its status like so:"
+echo "  $sudo_cmd systemctl status $my_app_name"
+echo "  $sudo_cmd journalctl -xefu $my_app_name"
+echo ""
+echo ""

installer/install-system-service.sh

@@ -0,0 +1,37 @@
+safe_copy_config()
+{
+  src=$1
+  dst=$2
+  $sudo_cmd mkdir -p $(dirname "$dst")
+  if [ -f "$dst" ]; then
+    $sudo_cmd rsync -a "$src" "$dst.latest"
+    # TODO edit config file with $my_user and $my_group
+    if [ "$(cat $dst)" == "$(cat $dst.latest)" ]; then
+      $sudo_cmd rm $dst.latest
+    else
+      echo "MANUAL INTERVENTION REQUIRED: check the systemd script update and manually decide what you want to do"
+      echo "diff $dst $dst.latest"
+      $sudo_cmd chown -R root:root "$dst.latest"
+    fi
+  else
+    $sudo_cmd rsync -a --ignore-existing "$src" "$dst"
+  fi
+  $sudo_cmd chown -R root:root "$dst"
+  $sudo_cmd chmod 644 "$dst"
+}
+
+installable=""
+if [ -d "$my_root/etc/systemd/system" ]; then
+  source ./installer/install-for-systemd.sh
+  installable="true"
+fi
+if [ -d "/Library/LaunchDaemons" ]; then
+  source ./installer/install-for-launchd.sh
+  installable="true"
+fi
+if [ -z "$installable" ]; then
+  echo ""
+  echo "Unknown system service init type. You must install as a system service manually."
+  echo '(please file a bug with the output of "uname -a")'
+  echo ""
+fi

installer/install.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+set -e
+set -u
+
+
+### IMPORTANT ###
+###  VERSION  ###
+my_name=goldilocks
+my_app_pkg_name=com.coolaj86.goldilocks.web
+my_app_ver="v1.1"
+my_azp_oauth3_ver="v1.2.3"
+export NODE_VERSION="v8.9.3"
+
+if [ -z "${my_tmp-}" ]; then
+  my_tmp="$(mktemp -d)"
+  mkdir -p $my_tmp/opt/$my_name/lib/node_modules/$my_name
+  echo "Installing to $my_tmp (will be moved after install)"
+  git clone ./ $my_tmp/opt/$my_name/lib/node_modules/$my_name
+  pushd $my_tmp/opt/$my_name/lib/node_modules/$my_name
+fi
+
+#################
+export NODE_PATH=$my_tmp/opt/$my_name/lib/node_modules
+export PATH=$my_tmp/opt/$my_name/bin/:$PATH
+export NPM_CONFIG_PREFIX=$my_tmp/opt/$my_name
+my_npm="$NPM_CONFIG_PREFIX/bin/npm"
+#################
+
+
+my_app_dist=$my_tmp/opt/$my_name/lib/node_modules/$my_name/dist
+installer_base="https://git.coolaj86.com/coolaj86/goldilocks.js/raw/$my_app_ver"
+
+# Backwards compat
+# some scripts still use the old names
+my_app_dir=$my_tmp
+my_app_name=$my_name
+
+
+
+git checkout $my_app_ver
+
+mkdir -p "$my_tmp/opt/$my_name"/{lib,bin,etc}
+ln -s ../lib/node_modules/$my_name/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name
+ln -s ../lib/node_modules/$my_name/bin/$my_name.js $my_tmp/opt/$my_name/bin/$my_name.js
+mkdir -p "$my_tmp/etc/$my_name"
+chmod 775 "$my_tmp/etc/$my_name"
+cat "$my_app_dist/etc/$my_name/$my_name.example.yml" > "$my_tmp/etc/$my_name/$my_name.example.yml"
+chmod 664 "$my_tmp/etc/$my_name/$my_name.example.yml"
+mkdir -p $my_tmp/srv/www
+mkdir -p $my_tmp/var/www
+mkdir -p $my_tmp/var/log/$my_name
+
+
+
+#
+# Helpers
+#
+source ./installer/sudo-cmd.sh
+source ./installer/http-get.sh
+
+
+
+#
+# Dependencies
+#
+echo $NODE_VERSION > /tmp/NODEJS_VER
+http_bash "https://git.coolaj86.com/coolaj86/node-installer.sh/raw/v1.1/install.sh"
+$my_npm install -g npm@4
+pushd $my_tmp/opt/$my_name/lib/node_modules/$my_name
+  $my_npm install
+popd
+pushd $my_tmp/opt/$my_name/lib/node_modules/$my_name/packages/assets
+  OAUTH3_GIT_URL="https://git.oauth3.org/OAuth3/oauth3.js.git"
+  git clone ${OAUTH3_GIT_URL} oauth3.org || true
+  ln -s oauth3.org org.oauth3
+  pushd oauth3.org
+    git remote set-url origin ${OAUTH3_GIT_URL}
+    git checkout $my_azp_oauth3_ver
+    #git pull
+  popd
+
+  mkdir -p jquery.com
+  ln -s jquery.com com.jquery
+  pushd jquery.com
+    http_get 'https://code.jquery.com/jquery-3.1.1.js' jquery-3.1.1.js
+  popd
+
+  mkdir -p google.com
+  ln -s google.com com.google
+  pushd google.com
+    http_get 'https://ajax.googleapis.com/ajax/libs/angularjs/1.6.2/angular.min.js' angular.1.6.2.min.js
+  popd
+
+  mkdir -p well-known
+  ln -s well-known .well-known
+  pushd well-known
+    ln -snf ../oauth3.org/well-known/oauth3 ./oauth3
+  popd
+  echo "installed dependencies"
+popd
+
+
+
+#
+# System Service
+#
+source ./installer/my-root.sh
+echo "Pre-installation to $my_tmp complete, now installing to $my_root/ ..."
+set +e
+if type -p tree >/dev/null 2>/dev/null; then
+  #tree -I "node_modules|include|share" $my_tmp
+  tree -L 6 -I "include|share|npm" $my_tmp
+else
+  ls $my_tmp
+fi
+set -e
+
+source ./installer/my-user-my-group.sh
+echo "User $my_user Group $my_group"
+
+source ./installer/install-system-service.sh
+
+$sudo_cmd chown -R $my_user:$my_group $my_tmp/*
+$sudo_cmd chown root:root $my_tmp/*
+$sudo_cmd chown root:root $my_tmp
+$sudo_cmd chmod 0755 $my_tmp
+# don't change permissions on /, /etc, etc
+$sudo_cmd rsync -a --ignore-existing $my_tmp/ $my_root/
+$sudo_cmd rsync -a --ignore-existing $my_app_dist/etc/$my_name/$my_name.yml $my_root/etc/$my_name/$my_name.yml
+
+# Change to admin perms
+$sudo_cmd chown -R $my_user:$my_group $my_root/opt/$my_name
+$sudo_cmd chown -R $my_user:$my_group $my_root/var/www $my_root/srv/www
+
+# make sure the files are all read/write for the owner and group, and then set
+# the setuid and setgid bits so that any files/directories created inside these
+# directories have the same owner and group.
+$sudo_cmd chmod -R ug+rwX $my_root/opt/$my_name
+find $my_root/opt/$my_name -type d -exec $sudo_cmd chmod ug+s {} \;
+
+
+
+echo ""
+echo "$my_name installation complete!"
+echo ""
+echo ""
+echo "Update the config at: /etc/$my_name/$my_name.yml"
+echo ""
+echo "Unistall: rm -rf /srv/$my_name/ /var/$my_name/ /etc/$my_name/ /opt/$my_name/ /var/log/$my_name/ /etc/tmpfiles.d/$my_name.conf /etc/systemd/system/$my_name.service /etc/ssl/$my_name"

installer/my-root.sh

@@ -0,0 +1,8 @@
+# something or other about android and tmux using PREFIX
+#: "${PREFIX:=''}"
+my_root=""
+if [ -z "${PREFIX-}" ]; then
+  my_root=""
+else
+  my_root="$PREFIX"
+fi

installer/my-user-my-group.sh

@@ -0,0 +1,19 @@
+if type -p adduser >/dev/null 2>/dev/null; then
+  if [ -z "$(cat $my_root/etc/passwd | grep $my_app_name)" ]; then
+    $sudo_cmd adduser --home $my_root/opt/$my_app_name --gecos '' --disabled-password $my_app_name
+  fi
+  my_user=$my_app_name
+  my_group=$my_app_name
+elif [ -n "$(cat /etc/passwd | grep www-data:)" ]; then
+  # Linux (Ubuntu)
+  my_user=www-data
+  my_group=www-data
+elif [ -n "$(cat /etc/passwd | grep _www:)" ]; then
+  # Mac
+  my_user=_www
+  my_group=_www
+else
+  # Unsure
+  my_user=$(whoami)
+  my_group=$(id -g -n)
+fi

installer/sudo-cmd.sh

@@ -0,0 +1,7 @@
+# Not every platform has or needs sudo, gotta save them O(1)s...
+sudo_cmd=""
+set +e
+if type -p sudo >/dev/null 2>/dev/null; then
+  ((EUID)) && [[ -z "${ANDROID_ROOT-}" ]] && sudo_cmd="sudo"
+fi
+set -e

lib/admin/apis.js

@@ -0,0 +1,585 @@
+'use strict';
+
+module.exports.dependencies = [ 'OAUTH3', 'storage.owners', 'options.device' ];
+module.exports.create = function (deps, conf) {
+  var scmp = require('scmp');
+  var crypto = require('crypto');
+  var jwt = require('jsonwebtoken');
+  var bodyParser = require('body-parser');
+  var jsonParser = bodyParser.json({
+    inflate: true, limit: '100kb', reviver: null, strict: true /* type, verify */
+  });
+
+  function handleCors(req, res, methods) {
+    if (!methods) {
+      methods = ['GET', 'POST'];
+    }
+    if (!Array.isArray(methods)) {
+      methods = [ methods ];
+    }
+
+    res.setHeader('Access-Control-Allow-Origin', req.headers.origin || '*');
+    res.setHeader('Access-Control-Allow-Methods', methods.join(', '));
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+    res.setHeader('Access-Control-Allow-Credentials', 'true');
+
+    if (req.method.toUpperCase() === 'OPTIONS') {
+      res.setHeader('Allow', methods.join(', '));
+      res.end();
+      return true;
+    }
+
+    if (methods.indexOf('*') >= 0) {
+      return false;
+    }
+    if (methods.indexOf(req.method.toUpperCase()) < 0) {
+      res.statusCode = 405;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(JSON.stringify({ error: { message: 'method '+req.method+' not allowed', code: 'EBADMETHOD'}}));
+      return true;
+    }
+  }
+  function makeCorsHandler(methods) {
+    return function corsHandler(req, res, next) {
+      if (!handleCors(req, res, methods)) {
+        next();
+      }
+    };
+  }
+
+  function handlePromise(req, res, prom) {
+    prom.then(function (result) {
+      res.send(deps.recase.snakeCopy(result));
+    }).catch(function (err) {
+      if (conf.debug) {
+        console.log(err);
+      }
+      res.statusCode = err.statusCode || 500;
+      err.message = err.message || err.toString();
+      res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+    });
+  }
+
+  function isAuthorized(req, res, fn) {
+    var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, ''));
+    if (!auth) {
+      res.statusCode = 401;
+      res.setHeader('Content-Type', 'application/json;');
+      res.end(JSON.stringify({ error: { message: "no token", code: 'E_NO_TOKEN', uri: undefined } }));
+      return;
+    }
+
+    var id = crypto.createHash('sha256').update(auth.sub).digest('hex');
+    return deps.storage.owners.exists(id).then(function (exists) {
+      if (!exists) {
+        res.statusCode = 401;
+        res.setHeader('Content-Type', 'application/json;');
+        res.end(JSON.stringify({ error: { message: "not authorized", code: 'E_NO_AUTHZ', uri: undefined } }));
+        return;
+      }
+
+      req.userId = id;
+      fn();
+    });
+  }
+
+  function checkPaywall() {
+    var url = require('url');
+    var PromiseA = require('bluebird');
+    var testDomains = [
+      'daplie.com'
+    , 'duckduckgo.com'
+    , 'google.com'
+    , 'amazon.com'
+    , 'facebook.com'
+    , 'msn.com'
+    , 'yahoo.com'
+    ];
+
+    // While this is not being developed behind a paywall the current idea is that
+    // a paywall will either manipulate DNS queries to point to the paywall gate,
+    // or redirect HTTP requests to the paywall gate. So we check for both and
+    // hope we can detect most hotel/ISP paywalls out there in the world.
+    //
+    // It is also possible that the paywall will prevent any unknown traffic from
+    // leaving the network, so the DNS queries could fail if the unit is set to
+    // use nameservers other than the paywall router.
+    return PromiseA.resolve()
+    .then(function () {
+      var dns = PromiseA.promisifyAll(require('dns'));
+      var proms = testDomains.map(function (dom) {
+        return dns.resolve6Async(dom)
+          .catch(function () {
+            return dns.resolve4Async(dom);
+          })
+          .then(function (result) {
+            return result[0];
+          }, function () {
+            return null;
+          });
+      });
+
+      return PromiseA.all(proms).then(function (addrs) {
+        var unique = addrs.filter(function (value, ind, self) {
+          return value && self.indexOf(value) === ind;
+        });
+        // It is possible some walls might have exceptions that leave some of the domains
+        // we test alone, so we might have more than one unique address even behind an
+        // active paywall.
+        return unique.length < addrs.length;
+      });
+    })
+    .then(function (paywall) {
+      if (paywall) {
+        return paywall;
+      }
+      var request = deps.request.defaults({
+        followRedirect: false
+      , headers: {
+          connection: 'close'
+        }
+      });
+
+      var proms = testDomains.map(function (dom) {
+        return request('http://'+dom).then(function (resp) {
+          if (resp.statusCode >= 300 && resp.statusCode < 400) {
+            return url.parse(resp.headers.location).hostname;
+          } else {
+            return dom;
+          }
+        });
+      });
+
+      return PromiseA.all(proms).then(function (urls) {
+        var unique = urls.filter(function (value, ind, self) {
+          return value && self.indexOf(value) === ind;
+        });
+        return unique.length < urls.length;
+      });
+    })
+    ;
+  }
+
+  // This object contains all of the API endpoints written before we changed how
+  // the API routing is handled. Eventually it will hopefully disappear, but for
+  // now we're focusing on the things that need changing more.
+  var oldEndPoints = {
+    init: function (req, res) {
+      if (handleCors(req, res, ['GET', 'POST'])) {
+        return;
+      }
+
+      if ('POST' !== req.method) {
+        // It should be safe to give the list of owner IDs to an un-authenticated
+        // request because the ID is the sha256 of the PPID and shouldn't be reversible
+        return deps.storage.owners.all().then(function (results) {
+          var ids = results.map(function (owner) {
+            return owner.id;
+          });
+          res.setHeader('Content-Type', 'application/json');
+          res.end(JSON.stringify(ids));
+        });
+      }
+
+      jsonParser(req, res, function () {
+
+      return deps.PromiseA.resolve().then(function () {
+        console.log('init POST body', req.body);
+
+        var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, ''));
+        var token = jwt.decode(req.body.access_token);
+        var refresh = jwt.decode(req.body.refresh_token);
+        auth.sub = auth.sub || auth.acx.id;
+        token.sub = token.sub || token.acx.id;
+        refresh.sub = refresh.sub || refresh.acx.id;
+
+        // TODO validate token with issuer, but as-is the sub is already a secret
+        var id = crypto.createHash('sha256').update(auth.sub).digest('hex');
+        var tid = crypto.createHash('sha256').update(token.sub).digest('hex');
+        var rid = crypto.createHash('sha256').update(refresh.sub).digest('hex');
+        var session = {
+          access_token: req.body.access_token
+        , token: token
+        , refresh_token: req.body.refresh_token
+        , refresh: refresh
+        };
+
+        console.log('ids', id, tid, rid);
+
+        if (req.body.ip_url) {
+          // TODO set options / GunDB
+          conf.ip_url = req.body.ip_url;
+        }
+
+        return deps.storage.owners.all().then(function (results) {
+          console.log('results', results);
+          var err;
+
+          // There is no owner yet. First come, first serve.
+          if (!results || !results.length) {
+            if (tid !== id || rid !== id) {
+              err = new Error(
+                "When creating an owner the Authorization Bearer and Token and Refresh must all match"
+              );
+              err.statusCode = 400;
+              return deps.PromiseA.reject(err);
+            }
+            console.log('no owner, creating');
+            return deps.storage.owners.set(id, session);
+          }
+          console.log('has results');
+
+          // There are onwers. Is this one of them?
+          if (!results.some(function (token) {
+            return scmp(id, token.id);
+          })) {
+            err = new Error("Authorization token does not belong to an existing owner.");
+            err.statusCode = 401;
+            return deps.PromiseA.reject(err);
+          }
+          console.log('has correct owner');
+
+          // We're adding an owner, unless it already exists
+          if (!results.some(function (token) {
+            return scmp(tid, token.id);
+          })) {
+            console.log('adds new owner with existing owner');
+            return deps.storage.owners.set(tid, session);
+          }
+        }).then(function () {
+          res.setHeader('Content-Type', 'application/json;');
+          res.end(JSON.stringify({ success: true }));
+        });
+      })
+      .catch(function (err) {
+        res.setHeader('Content-Type', 'application/json;');
+        res.statusCode = err.statusCode || 500;
+        res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } }));
+      });
+
+      });
+    }
+  , request: function (req, res) {
+      if (handleCors(req, res, '*')) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+      jsonParser(req, res, function () {
+
+        deps.request({
+          method: req.body.method || 'GET'
+        , url: req.body.url
+        , headers: req.body.headers
+        , body: req.body.data
+        }).then(function (resp) {
+          if (resp.body instanceof Buffer || 'string' === typeof resp.body) {
+            resp.body = JSON.parse(resp.body);
+          }
+
+          return {
+            statusCode: resp.statusCode
+          , status: resp.status
+          , headers: resp.headers
+          , body: resp.body
+          , data: resp.data
+          };
+        }).then(function (result) {
+          res.send(result);
+        });
+
+      });
+      });
+    }
+  , paywall_check: function (req, res) {
+      if (handleCors(req, res, 'GET')) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+        res.setHeader('Content-Type', 'application/json;');
+
+        checkPaywall().then(function (paywall) {
+          res.end(JSON.stringify({paywall: paywall}));
+        }, function (err) {
+          err.message = err.message || err.toString();
+          res.statusCode = 500;
+          res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+        });
+      });
+    }
+  , socks5: function (req, res) {
+      if (handleCors(req, res, ['GET', 'POST', 'DELETE'])) {
+        return;
+      }
+      isAuthorized(req, res, function () {
+        var method = req.method.toUpperCase();
+        var prom;
+
+        if (method === 'POST') {
+          prom = deps.socks5.start();
+        } else if (method === 'DELETE') {
+          prom = deps.socks5.stop();
+        } else {
+          prom = deps.socks5.curState();
+        }
+
+        res.setHeader('Content-Type', 'application/json;');
+        prom.then(function (result) {
+          res.end(JSON.stringify(result));
+        }, function (err) {
+          err.message = err.message || err.toString();
+          res.statusCode = 500;
+          res.end(JSON.stringify({error: {message: err.message, code: err.code}}));
+        });
+      });
+    }
+  };
+
+  function handleOldApis(req, res, next) {
+    if (typeof oldEndPoints[req.params.name] === 'function') {
+      oldEndPoints[req.params.name](req, res);
+    } else {
+      next();
+    }
+  }
+
+  var config = { restful: {} };
+  config.restful.readConfig = function (req, res, next) {
+    var part = new (require('./config').ConfigChanger)(conf);
+    if (req.params.group) {
+      part = part[req.params.group];
+    }
+    if (part && req.params.domId) {
+      part = part.domains.findId(req.params.domId);
+    }
+    if (part && req.params.mod) {
+      part = part[req.params.mod];
+    }
+    if (part && req.params.modGrp) {
+      part = part[req.params.modGrp];
+    }
+    if (part && req.params.modId) {
+      part = part.findId(req.params.modId);
+    }
+
+    if (part) {
+      res.send(deps.recase.snakeCopy(part));
+    } else {
+      next();
+    }
+  };
+
+  config.save = function (changer) {
+    var errors = changer.validate();
+    if (errors.length) {
+      throw Object.assign(new Error(), errors[0], {statusCode: 400});
+    }
+
+    return deps.storage.config.save(changer);
+  };
+  config.restful.saveBaseConfig = function (req, res, next) {
+    console.log('config POST body', JSON.stringify(req.body));
+    if (req.params.group === 'domains') {
+      next();
+      return;
+    }
+
+    var promise = deps.PromiseA.resolve().then(function () {
+      var update;
+      if (req.params.group) {
+        update = {};
+        update[req.params.group] = req.body;
+      } else {
+        update = req.body;
+      }
+
+      var changer = new (require('./config').ConfigChanger)(conf);
+      changer.update(update);
+      return config.save(changer);
+    }).then(function (newConf) {
+      if (req.params.group) {
+        return newConf[req.params.group];
+      }
+      return newConf;
+    });
+    handlePromise(req, res, promise);
+  };
+
+  config.extractModList = function (changer, params) {
+    var err;
+    if (params.domId) {
+      var dom = changer.domains.find(function (dom) {
+        return dom.id === params.domId;
+      });
+
+      if (!dom) {
+        err = new Error("no domain with ID '"+params.domId+"'");
+      } else if (!dom.modules[params.group]) {
+        err = new Error("domains don't contain '"+params.group+"' modules");
+      } else {
+        return dom.modules[params.group];
+      }
+    } else {
+      if (!changer[params.group] || !changer[params.group].modules) {
+        err = new Error("'"+params.group+"' is not a valid settings group or doesn't support modules");
+      } else {
+        return changer[params.group].modules;
+      }
+    }
+
+    err.statusCode = 404;
+    throw err;
+  };
+  config.restful.createModule = function (req, res, next) {
+    if (req.params.group === 'domains') {
+      next();
+      return;
+    }
+
+    var promise = deps.PromiseA.resolve().then(function () {
+      var changer = new (require('./config').ConfigChanger)(conf);
+      var modList = config.extractModList(changer, req.params);
+
+      var update = req.body;
+      if (!Array.isArray(update)) {
+        update = [ update ];
+      }
+      update.forEach(modList.add, modList);
+
+      return config.save(changer);
+    }).then(function (newConf) {
+      return config.extractModList(newConf, req.params);
+    });
+    handlePromise(req, res, promise);
+  };
+  config.restful.updateModule = function (req, res, next) {
+    if (req.params.group === 'domains') {
+      next();
+      return;
+    }
+
+    var promise = deps.PromiseA.resolve().then(function () {
+      var changer = new (require('./config').ConfigChanger)(conf);
+      var modList = config.extractModList(changer, req.params);
+      modList.update(req.params.modId, req.body);
+      return config.save(changer);
+    }).then(function (newConf) {
+      return config.extractModule(newConf, req.params).find(function (mod) {
+        return mod.id === req.params.modId;
+      });
+    });
+    handlePromise(req, res, promise);
+  };
+  config.restful.removeModule = function (req, res, next) {
+    if (req.params.group === 'domains') {
+      next();
+      return;
+    }
+
+    var promise = deps.PromiseA.resolve().then(function () {
+      var changer = new (require('./config').ConfigChanger)(conf);
+      var modList = config.extractModList(changer, req.params);
+      modList.remove(req.params.modId);
+      return config.save(changer);
+    }).then(function (newConf) {
+      return config.extractModList(newConf, req.params);
+    });
+    handlePromise(req, res, promise);
+  };
+
+  config.restful.createDomain = function (req, res) {
+    var promise = deps.PromiseA.resolve().then(function () {
+      var changer = new (require('./config').ConfigChanger)(conf);
+
+      var update = req.body;
+      if (!Array.isArray(update)) {
+        update = [ update ];
+      }
+      update.forEach(changer.domains.add, changer.domains);
+      return config.save(changer);
+    }).then(function (newConf) {
+      return newConf.domains;
+    });
+    handlePromise(req, res, promise);
+  };
+  config.restful.updateDomain = function (req, res) {
+    var promise = deps.PromiseA.resolve().then(function () {
+      if (req.body.modules) {
+        throw Object.assign(new Error('do not add modules with this route'), {statusCode: 400});
+      }
+
+      var changer = new (require('./config').ConfigChanger)(conf);
+      changer.domains.update(req.params.domId, req.body);
+      return config.save(changer);
+    }).then(function (newConf) {
+      return newConf.domains.find(function (dom) {
+        return dom.id === req.params.domId;
+      });
+    });
+    handlePromise(req, res, promise);
+  };
+  config.restful.removeDomain = function (req, res) {
+    var promise = deps.PromiseA.resolve().then(function () {
+      var changer = new (require('./config').ConfigChanger)(conf);
+      changer.domains.remove(req.params.domId);
+      return config.save(changer);
+    }).then(function (newConf) {
+      return newConf.domains;
+    });
+    handlePromise(req, res, promise);
+  };
+
+  var tokens = { restful: {} };
+  tokens.restful.getAll = function (req, res) {
+    handlePromise(req, res, deps.storage.tokens.all());
+  };
+  tokens.restful.getOne = function (req, res) {
+    handlePromise(req, res, deps.storage.tokens.get(req.params.id));
+  };
+  tokens.restful.save = function (req, res) {
+    handlePromise(req, res, deps.storage.tokens.save(req.body));
+  };
+  tokens.restful.revoke = function (req, res) {
+    var promise = deps.storage.tokens.remove(req.params.id).then(function (success) {
+      return {success: success};
+    });
+    handlePromise(req, res, promise);
+  };
+
+
+  var app = require('express')();
+
+  // Handle all of the API endpoints using the old definition style, and then we can
+  // add middleware without worrying too much about the consequences to older code.
+  app.use('/:name', handleOldApis);
+
+  // Not all routes support all of these methods, but not worth making this more specific
+  app.use('/', makeCorsHandler(['GET', 'POST', 'PUT', 'DELETE']), isAuthorized, jsonParser);
+
+  app.get(   '/config',                                                 config.restful.readConfig);
+  app.get(   '/config/:group',                                          config.restful.readConfig);
+  app.get(   '/config/:group/:mod(modules)/:modId?',                    config.restful.readConfig);
+  app.get(   '/config/domains/:domId/:mod(modules)?',                   config.restful.readConfig);
+  app.get(   '/config/domains/:domId/:mod(modules)/:modGrp/:modId?',    config.restful.readConfig);
+
+  app.post(  '/config',                                       config.restful.saveBaseConfig);
+  app.post(  '/config/:group',                                config.restful.saveBaseConfig);
+
+  app.post(  '/config/:group/modules',                        config.restful.createModule);
+  app.put(   '/config/:group/modules/:modId',                 config.restful.updateModule);
+  app.delete('/config/:group/modules/:modId',                 config.restful.removeModule);
+
+  app.post(  '/config/domains/:domId/modules/:group',         config.restful.createModule);
+  app.put(   '/config/domains/:domId/modules/:group/:modId',  config.restful.updateModule);
+  app.delete('/config/domains/:domId/modules/:group/:modId',  config.restful.removeModule);
+
+  app.post(  '/config/domains',                               config.restful.createDomain);
+  app.put(   '/config/domains/:domId',                        config.restful.updateDomain);
+  app.delete('/config/domains/:domId',                        config.restful.removeDomain);
+
+  app.get(   '/tokens',         tokens.restful.getAll);
+  app.get(   '/tokens/:id',     tokens.restful.getOne);
+  app.post(  '/tokens',         tokens.restful.save);
+  app.delete('/tokens/:id',     tokens.restful.revoke);
+
+  return app;
+};

lib/admin/config.js

@@ -0,0 +1,398 @@
+'use strict';
+
+var validator = new (require('jsonschema').Validator)();
+var recase = require('recase').create({});
+
+var portSchema = { type: 'number', minimum: 1, maximum: 65535 };
+
+var moduleSchemas = {
+  // the proxy module is common to basically all categories.
+  proxy: {
+    type: 'object'
+  , oneOf: [
+      { required: [ 'address' ] }
+    , { required: [ 'port' ] }
+    ]
+  , properties: {
+      address: { type: 'string' }
+    , host:    { type: 'string' }
+    , port:    portSchema
+    }
+  }
+
+  // redirect and static modules are for HTTP
+, redirect: {
+    type: 'object'
+  , required: [ 'to', 'from' ]
+  , properties: {
+      to:     { type: 'string'}
+    , from:   { type: 'string'}
+    , status: { type: 'integer', minimum: 1, maximum: 999 }
+  , }
+  }
+, static: {
+    type: 'object'
+  , required: [ 'root' ]
+  , properties: {
+      root: { type: 'string' }
+    }
+  }
+
+  // the acme module is for TLS
+, acme: {
+    type: 'object'
+  , required: [ 'email' ]
+  , properties: {
+      email:          { type: 'string' }
+    , server:         { type: 'string' }
+    , challenge_type: { type: 'string' }
+    }
+  }
+
+  // the dns control modules for DDNS
+, 'dns@oauth3.org': {
+    type: 'object'
+  , required: [ 'token_id' ]
+  , properties: {
+      token_id: { type: 'string' }
+    }
+  }
+};
+// forward is basically the same as proxy, but specifies the relevant incoming port(s).
+// only allows for the raw transport layers (TCP/UDP)
+moduleSchemas.forward = JSON.parse(JSON.stringify(moduleSchemas.proxy));
+moduleSchemas.forward.required = [ 'ports' ];
+moduleSchemas.forward.properties.ports = { type: 'array', items: portSchema };
+
+Object.keys(moduleSchemas).forEach(function (name) {
+  var schema = moduleSchemas[name];
+  schema.id = '/modules/'+name;
+  schema.required = ['id', 'type'].concat(schema.required || []);
+  schema.properties.id   = { type: 'string' };
+  schema.properties.type = { type: 'string', const: name };
+  validator.addSchema(schema, schema.id);
+});
+
+function addDomainRequirement(itemSchema) {
+  var result = Object.assign({}, itemSchema);
+  result.required = (result.required || []).concat('domains');
+  result.properties = Object.assign({}, result.properties);
+  result.properties.domains = { type: 'array', items: { type: 'string' }, minLength: 1};
+  return result;
+}
+
+function toSchemaRef(name) {
+  return { '$ref': '/modules/'+name };
+}
+var moduleRefs = {
+  http: [ 'proxy', 'static', 'redirect' ].map(toSchemaRef)
+, tls:  [ 'proxy', 'acme' ].map(toSchemaRef)
+, tcp:  [ 'forward' ].map(toSchemaRef)
+, udp:  [ 'forward' ].map(toSchemaRef)
+, ddns: [ 'dns@oauth3.org' ].map(toSchemaRef)
+};
+
+// TCP is a bit special in that it has a module that doesn't operate based on domain name
+// (ie forward), and a modules that does (ie proxy). It therefore has different module
+// when part of the `domains` config, and when not part of the `domains` config the proxy
+// modules must have the `domains` property while forward should not have it.
+moduleRefs.tcp.push(addDomainRequirement(toSchemaRef('proxy')));
+
+var domainSchema = {
+  type: 'array'
+, items: {
+    type: 'object'
+  , properties: {
+      id:      { type: 'string' }
+    , names:   { type: 'array', items: { type: 'string' }, minLength: 1}
+    , modules: {
+        type: 'object'
+      , properties: {
+          tls:  { type: 'array', items: { oneOf: moduleRefs.tls }}
+        , http: { type: 'array', items: { oneOf: moduleRefs.http }}
+        , ddns: { type: 'array', items: { oneOf: moduleRefs.ddns }}
+        , tcp:  { type: 'array', items: { oneOf: ['proxy'].map(toSchemaRef)}}
+        }
+      , additionalProperties: false
+      }
+    }
+  }
+};
+
+var httpSchema = {
+  type: 'object'
+, properties: {
+    modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.http }) }
+
+    // These properties should be snake_case to match the API and config format
+  , primary_domain: { type: 'string' }
+  , allow_insecure: { type: 'boolean' }
+  , trust_proxy:    { type: 'boolean' }
+
+    // these are forbidden deprecated settings.
+  , bind:    { not: {} }
+  , domains: { not: {} }
+  }
+};
+
+var tlsSchema = {
+  type: 'object'
+, properties: {
+    modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.tls }) }
+
+    // these are forbidden deprecated settings.
+  , acme:    { not: {} }
+  , bind:    { not: {} }
+  , domains: { not: {} }
+  }
+};
+
+var tcpSchema = {
+  type: 'object'
+, required: [ 'bind' ]
+, properties: {
+    bind:    { type: 'array', items: portSchema, minLength: 1 }
+  , modules: { type: 'array', items: { oneOf: moduleRefs.tcp }}
+  }
+};
+
+var udpSchema = {
+  type: 'object'
+, properties: {
+    bind:    { type: 'array', items: portSchema }
+  , modules: { type: 'array', items: { oneOf: moduleRefs.udp }}
+  }
+};
+
+var mdnsSchema = {
+  type: 'object'
+, required: [ 'port', 'broadcast', 'ttl' ]
+, properties: {
+    port:      portSchema
+  , broadcast: { type: 'string' }
+  , ttl:       { type: 'integer', minimum: 0, maximum: 2147483647 }
+  }
+};
+
+var tunnelSvrSchema = {
+  type: 'object'
+, properties: {
+    servernames: { type: 'array', items: { type: 'string' }}
+  , secret:      { type: 'string' }
+  }
+};
+
+var ddnsSchema = {
+  type: 'object'
+, properties: {
+    loopback: {
+      type: 'object'
+    , required: [ 'type', 'domain' ]
+    , properties: {
+        type:   { type: 'string', const: 'tunnel@oauth3.org' }
+      , domain: { type: 'string'}
+      }
+    }
+  , tunnel: {
+      type: 'object'
+    , required: [ 'type', 'token_id' ]
+    , properties: {
+        type:  { type: 'string', const: 'tunnel@oauth3.org' }
+      , token_id: { type: 'string'}
+      }
+    }
+  , modules: { type: 'array', items: addDomainRequirement({ oneOf: moduleRefs.ddns })}
+  }
+};
+var socks5Schema = {
+  type: 'object'
+, properties: {
+    enabled: { type: 'boolean' }
+  , port:    portSchema
+  }
+};
+var deviceSchema = {
+  type: 'object'
+, properties: {
+    hostname: { type: 'string' }
+  }
+};
+
+var mainSchema = {
+  type: 'object'
+, required: [ 'domains', 'http', 'tls', 'tcp', 'udp', 'mdns', 'ddns' ]
+, properties: {
+    domains:domainSchema
+  , http:   httpSchema
+  , tls:    tlsSchema
+  , tcp:    tcpSchema
+  , udp:    udpSchema
+  , mdns:   mdnsSchema
+  , ddns:   ddnsSchema
+  , socks5: socks5Schema
+  , device: deviceSchema
+  , tunnel_server: tunnelSvrSchema
+  }
+, additionalProperties: false
+};
+
+function validate(config) {
+  return validator.validate(recase.snakeCopy(config), mainSchema).errors;
+}
+module.exports.validate = validate;
+
+class IdList extends Array {
+  constructor(rawList) {
+    super();
+    if (Array.isArray(rawList)) {
+      Object.assign(this, JSON.parse(JSON.stringify(rawList)));
+    }
+    this._itemName = 'item';
+  }
+
+  findId(id) {
+    return Array.prototype.find.call(this, function (dom) {
+      return dom.id === id;
+    });
+  }
+
+  add(item) {
+    item.id = require('crypto').randomBytes(4).toString('hex');
+    this.push(item);
+  }
+
+  update(id, update) {
+    var item = this.findId(id);
+    if (!item) {
+      var error = new Error("no "+this._itemName+" with ID '"+id+"'");
+      error.statusCode = 404;
+      throw error;
+    }
+    Object.assign(this.findId(id), update);
+  }
+
+  remove(id) {
+    var index = this.findIndex(function (dom) {
+      return dom.id === id;
+    });
+    if (index < 0) {
+      var error = new Error("no "+this._itemName+" with ID '"+id+"'");
+      error.statusCode = 404;
+      throw error;
+    }
+    this.splice(index, 1);
+  }
+}
+class ModuleList extends IdList {
+  constructor(rawList) {
+    super(rawList);
+    this._itemName = 'module';
+  }
+
+  add(mod) {
+    if (!mod.type) {
+      throw new Error("module must have a 'type' defined");
+    }
+    if (!moduleSchemas[mod.type]) {
+      throw new Error("invalid module type '"+mod.type+"'");
+    }
+
+    mod.id = require('crypto').randomBytes(4).toString('hex');
+    this.push(mod);
+  }
+}
+class DomainList extends IdList {
+  constructor(rawList) {
+    super(rawList);
+    this._itemName = 'domain';
+    this.forEach(function (dom) {
+      dom.modules = {
+        http: new ModuleList((dom.modules || {}).http)
+      , tls:  new ModuleList((dom.modules || {}).tls)
+      , ddns: new ModuleList((dom.modules || {}).ddns)
+      , tcp:  new ModuleList((dom.modules || {}).tcp)
+      };
+    });
+  }
+
+  add(dom) {
+    if (!Array.isArray(dom.names) || !dom.names.length) {
+      throw new Error("domains must have a non-empty array for 'names'");
+    }
+    if (dom.names.some(function (name) { return typeof name !== 'string'; })) {
+      throw new Error("all domain names must be strings");
+    }
+
+    var modLists = {
+      http: new ModuleList()
+    , tls:  new ModuleList()
+    , ddns: new ModuleList()
+    , tcp:  new ModuleList()
+    };
+    // We add these after instead of in the constructor to run the validation and manipulation
+    // in the ModList add function since these are all new modules.
+    if (dom.modules) {
+      Object.keys(modLists).forEach(function (key) {
+        if (Array.isArray(dom.modules[key])) {
+          dom.modules[key].forEach(modLists[key].add, modLists[key]);
+        }
+      });
+    }
+
+    dom.id = require('crypto').randomBytes(4).toString('hex');
+    dom.modules = modLists;
+    this.push(dom);
+  }
+}
+
+class ConfigChanger {
+  constructor(start) {
+    Object.assign(this, JSON.parse(JSON.stringify(start)));
+    delete this.device;
+    delete this.debug;
+
+    this.domains = new DomainList(this.domains);
+    this.http.modules = new ModuleList(this.http.modules);
+    this.tls.modules  = new ModuleList(this.tls.modules);
+    this.tcp.modules  = new ModuleList(this.tcp.modules);
+    this.udp.modules  = new ModuleList(this.udp.modules);
+    this.ddns.modules = new ModuleList(this.ddns.modules);
+  }
+
+  update(update) {
+    var self = this;
+
+    if (update.domains) {
+      update.domains.forEach(self.domains.add, self.domains);
+    }
+    [ 'http', 'tls', 'tcp', 'udp', 'ddns' ].forEach(function (name) {
+      if (update[name] && update[name].modules) {
+        update[name].modules.forEach(self[name].modules.add, self[name].modules);
+        delete update[name].modules;
+      }
+    });
+
+    function mergeSettings(orig, changes) {
+      Object.keys(changes).forEach(function (key) {
+        // TODO: use an API that can properly handle updating arrays.
+        if (!changes[key] || (typeof changes[key] !== 'object') || Array.isArray(changes[key])) {
+          orig[key] = changes[key];
+        }
+        else if (!orig[key] || typeof orig[key] !== 'object') {
+          orig[key] = changes[key];
+        }
+        else {
+          mergeSettings(orig[key], changes[key]);
+        }
+      });
+    }
+    mergeSettings(this, update);
+
+    return validate(this);
+  }
+
+  validate() {
+    return validate(this);
+  }
+}
+module.exports.ConfigChanger = ConfigChanger;

lib/admin/index.js

@@ -0,0 +1,31 @@
+var adminDomains = [
+  'localhost.alpha.daplie.me'
+, 'localhost.admin.daplie.me'
+, 'alpha.localhost.daplie.me'
+, 'admin.localhost.daplie.me'
+, 'localhost.daplie.invalid'
+];
+module.exports.adminDomains = adminDomains;
+
+module.exports.create = function (deps, conf) {
+  'use strict';
+
+  var path = require('path');
+  var express = require('express');
+  var app = express();
+
+  var apis = require('./apis').create(deps, conf);
+  app.use('/api/goldilocks@daplie.com', apis);
+  app.use('/api/com.daplie.goldilocks', apis);
+
+  // Serve the static assets for the UI (even though it probably won't be used very
+  // often since it only works on localhost domains). Note that we are using the default
+  // .well-known directory from the oauth3 library even though it indicates we have
+  // capabilities we don't support because it's simpler and it's unlikely anything will
+  // actually use it to determine our API (it is needed to log into the web page).
+  app.use('/.well-known', express.static(path.join(__dirname, '../../packages/assets/well-known')));
+  app.use('/assets',      express.static(path.join(__dirname, '../../packages/assets')));
+  app.use('/',            express.static(path.join(__dirname, '../../admin/public')));
+
+  return require('http').createServer(app);
+};

lib/app.js

@@ -1,522 +0,0 @@
-'use strict';
-
-module.exports = function (myDeps, conf, overrideHttp) {
-  var express = require('express');
-  //var finalhandler = require('finalhandler');
-  var serveStatic = require('serve-static');
-  var serveIndex = require('serve-index');
-  //var assetServer = serveStatic(opts.assetsPath);
-  var path = require('path');
-  //var wellKnownServer = serveStatic(path.join(opts.assetsPath, 'well-known'));
-
-  var serveStaticMap = {};
-  var serveIndexMap = {};
-  var content = conf.content;
-  //var server;
-  var serveInit;
-  var app;
-  var tun;
-  var request;
-
-  /*
-  function _reloadWrite(data, enc, cb) {
-    // /*jshint validthis: true */ /*
-    if (this.headersSent) {
-      this.__write(data, enc, cb);
-      return;
-    }
-
-    if (!/html/i.test(this.getHeader('Content-Type'))) {
-      this.__write(data, enc, cb);
-      return;
-    }
-
-    if (this.getHeader('Content-Length')) {
-      this.setHeader('Content-Length', this.getHeader('Content-Length') + this.__my_addLen);
-    }
-
-    this.__write(this.__my_livereload);
-    this.__write(data, enc, cb);
-  }
-  */
-
-
-  function createServeInit() {
-    var PromiseA = require('bluebird');
-    var stunnel = require('stunnel');
-    var OAUTH3 = require('../packages/assets/org.oauth3');
-    require('../packages/assets/org.oauth3/oauth3.domains.js');
-    require('../packages/assets/org.oauth3/oauth3.dns.js');
-    require('../packages/assets/org.oauth3/oauth3.tunnel.js');
-    OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js');
-    var fs = PromiseA.promisifyAll(require('fs'));
-    var ownersPath = path.join(__dirname, '..', 'var', 'owners.json');
-
-    var scmp = require('scmp');
-    request = request || PromiseA.promisify(require('request'));
-
-    var owners = {
-      all: function () {
-        var owners;
-        try {
-          owners = require(ownersPath);
-        } catch(e) {
-          owners = {};
-        }
-
-        return PromiseA.resolve(Object.keys(owners).map(function (key) {
-          var owner = owners[key];
-          owner.id = key;
-          return owner;
-        }));
-      }
-    , get: function (id) {
-        var me = this;
-
-        return me.all().then(function (owners) {
-          return owners.filter(function (owner) {
-            return scmp(id, owner.id);
-          })[0];
-        });
-      }
-    , exists: function (id) {
-        var me = this;
-
-        return me.get(id).then(function (owner) {
-          return !!owner;
-        });
-      }
-    , set: function (id, obj) {
-        var owners;
-        try {
-          owners = require(ownersPath);
-        } catch(e) {
-          owners = {};
-        }
-        obj.id = id;
-        owners[id] = obj;
-
-        return fs.writeFileAsync(ownersPath, JSON.stringify(owners), 'utf8');
-      }
-    };
-
-    myDeps.PromiseA = PromiseA;
-    myDeps.OAUTH3 = OAUTH3;
-    myDeps.storage = { owners: owners };
-    myDeps.recase = require('recase').create({});
-    myDeps.request = request;
-    myDeps.api = {
-      // TODO move loopback to oauth3.api('tunnel:loopback')
-      loopback: function (deps, session, opts2) {
-        var crypto = require('crypto');
-        var token = crypto.randomBytes(16).toString('hex');
-        var keyAuthorization = crypto.randomBytes(16).toString('hex');
-        var nonce = crypto.randomBytes(16).toString('hex');
-
-        // TODO set token and keyAuthorization to /.well-known/cloud-challenge/:token
-        return request({
-          method: 'POST'
-        , url: 'https://oauth3.org/api/org.oauth3.tunnel/loopback'
-        , json: {
-            address: opts2.address
-          , port: opts2.port
-          , token: token
-          , keyAuthorization: keyAuthorization
-          , servername: opts2.servername
-          , nonce: nonce
-          , scheme: 'https'
-          , iat: Date.now()
-          }
-        }).then(function (result) {
-          // TODO this will always fail at the moment
-          console.log('loopback result:');
-          return result;
-        });
-      }
-    , tunnel: function (deps, session) {
-        // TODO save session to config and turn tunnel on
-        var OAUTH3 = deps.OAUTH3;
-        var url = require('url');
-        var providerUri = session.token.aud;
-        var urlObj = url.parse(OAUTH3.url.normalize(session.token.azp));
-        var oauth3 = OAUTH3.create(urlObj, {
-          providerUri: providerUri
-        , session: session
-        });
-        //var crypto = require('crypto');
-        //var id = crypto.createHash('sha256').update(session.token.sub).digest('hex');
-        return oauth3.setProvider(providerUri).then(function () {
-          /*
-          return oauth3.api('domains.list').then(function (domains) {
-            var domainsMap = {};
-            domains.forEach(function (d) {
-              if (!d.device) {
-                return;
-              }
-              if (d.device !== conf.device.hostname) {
-                return;
-              }
-              domainsMap[d.name] = true;
-            });
-          */
-
-            //console.log('domains matching hostname', Object.keys(domainsMap));
-            //console.log('device', conf.device);
-            return oauth3.api('tunnel.token', {
-              data: {
-                // filter to all domains that are on this device
-                //domains: Object.keys(domainsMap)
-                device: {
-                  hostname: conf.device.hostname
-                , id: conf.device.uid || conf.device.id
-                }
-              }
-            }).then(function (result) {
-              console.log('got a token from the tunnel server?');
-              console.log(result);
-              if (!result.tunnelUrl) {
-                result.tunnelUrl = ('wss://' + (new Buffer(result.jwt.split('.')[1], 'base64').toString('ascii')).aud + '/');
-              }
-              var services = { https: { '*': 443 }, http: { '*': 80 }, smtp: { '*': 25}, smtps: { '*': 587 /*also 465/starttls*/ } /*, ssh: { '*': 22 }*/ };
-              /*
-              console.log('blah');
-              console.log(result.jwt);
-              console.log(result.tunnelUrl);
-              console.log(services);
-              console.log('deps.tunnel');
-              console.log(deps.tunnel);
-              console.log('deps.tunnel.net');
-              console.log(deps.tunnel.net.toString());
-              console.log('deps.net');
-              console.log(deps.net);
-              */
-              var opts3 = {
-                token: result.jwt
-              , stunneld: result.tunnelUrl
-                // we'll provide faux networking and pipe as we please
-              , services: services
-              , net: myDeps.tunnel.net
-              };
-
-              console.log('blah 2');
-              if (tun) {
-                console.log('balh 3');
-                if (tun.append) {
-                  tun.append(result.jwt);
-                }
-                else if (tun.end) {
-                  tun.end();
-                  tun = null;
-                }
-              }
-
-              console.log('might have tunnel?');
-              if (!tun) {
-                console.log('connecting to the tunnel');
-                tun = stunnel.connect(opts3);
-                conf.tun = true;
-              }
-            });
-          /*
-          });
-          */
-        });
-        //, { token: token, refresh: refresh });
-      }
-    };
-
-    return require('../packages/apis/com.daplie.goldilocks').create(myDeps, conf);
-  }
-
-  app = express();
-
-  var Sites = {
-    add: function (sitesMap, site) {
-      if (!sitesMap[site.$id]) {
-        sitesMap[site.$id] = site;
-      }
-
-      if (!site.paths) {
-        site.paths = [];
-      }
-      if (!site.paths._map) {
-        site.paths._map = {};
-      }
-      site.paths.forEach(function (path) {
-
-        site.paths._map[path.$id] = path;
-
-        if (!path.modules) {
-          path.modules = [];
-        }
-        if (!path.modules._map) {
-          path.modules._map = {};
-        }
-        path.modules.forEach(function (module) {
-
-          path.modules._map[module.$id] = module;
-        });
-      });
-    }
-  };
-
-  var opts = overrideHttp || conf.http;
-  if (!opts.defaults) {
-    opts.defaults = {};
-  }
-  if (!opts.global) {
-    opts.global = {};
-  }
-  if (!opts.sites) {
-    opts.sites = [];
-  }
-  opts.sites._map = {};
-  opts.sites.forEach(function (site) {
-
-    Sites.add(opts.sites._map, site);
-  });
-
-  function mapMap(el, i, arr) {
-    arr._map[el.$id] = el;
-  }
-  opts.global.modules._map = {};
-  opts.global.modules.forEach(mapMap);
-  opts.global.paths._map = {};
-  opts.global.paths.forEach(function (path, i, arr) {
-    mapMap(path, i, arr);
-    //opts.global.paths._map[path.$id] = path;
-    path.modules._map = {};
-    path.modules.forEach(mapMap);
-  });
-  opts.sites.forEach(function (site) {
-    site.paths._map = {};
-    site.paths.forEach(function (path, i, arr) {
-      mapMap(path, i, arr);
-      //site.paths._map[path.$id] = path;
-      path.modules._map = {};
-      path.modules.forEach(mapMap);
-    });
-  });
-  opts.defaults.modules._map = {};
-  opts.defaults.modules.forEach(mapMap);
-  opts.defaults.paths._map = {};
-  opts.defaults.paths.forEach(function (path, i, arr) {
-    mapMap(path, i, arr);
-    //opts.global.paths._map[path.$id] = path;
-    path.modules._map = {};
-    path.modules.forEach(mapMap);
-  });
-
-  return app.use('/', function (req, res, next) {
-    if (!req.headers.host) {
-      next(new Error('missing HTTP Host header'));
-      return;
-    }
-
-    if (0 === req.url.indexOf('/api/com.daplie.goldilocks/')) {
-      if (!serveInit) {
-        serveInit = createServeInit();
-      }
-    }
-    if ('/api/com.daplie.goldilocks/init' === req.url) {
-      serveInit.init(req, res);
-      return;
-    }
-    if ('/api/com.daplie.goldilocks/tunnel' === req.url) {
-      serveInit.tunnel(req, res);
-      return;
-    }
-    if ('/api/com.daplie.goldilocks/config' === req.url) {
-      serveInit.config(req, res);
-      return;
-    }
-    if ('/api/com.daplie.goldilocks/request' === req.url) {
-      serveInit.request(req, res);
-      return;
-    }
-
-    if (content && '/' === req.url) {
-      // res.setHeader('Content-Type', 'application/octet-stream');
-      res.end(content);
-      return;
-    }
-
-    //var done = finalhandler(req, res);
-    var host = req.headers.host;
-    var hostname = (host||'').split(':')[0].toLowerCase();
-
-    console.log('opts.global', opts.global);
-    var sites = [ opts.global || null, opts.sites._map[hostname] || null, opts.defaults || null ];
-    var loadables = {
-      serve: function (config, hostname, pathname, req, res, next) {
-        var originalUrl = req.url;
-        var dirpaths = config.paths.slice(0);
-
-        function nextServe() {
-          var dirname = dirpaths.pop();
-          if (!dirname) {
-            req.url = originalUrl;
-            next();
-            return;
-          }
-
-          console.log('[serve]', req.url, hostname, pathname, dirname);
-          dirname = path.resolve(conf.cwd, dirname.replace(/:hostname/, hostname));
-          if (!serveStaticMap[dirname]) {
-            serveStaticMap[dirname] = serveStatic(dirname);
-          }
-
-          serveStaticMap[dirname](req, res, nextServe);
-        }
-
-        req.url = req.url.substr(pathname.length - 1);
-        nextServe();
-      }
-    , indexes: function (config, hostname, pathname, req, res, next) {
-        var originalUrl = req.url;
-        var dirpaths = config.paths.slice(0);
-
-        function nextIndex() {
-          var dirname = dirpaths.pop();
-          if (!dirname) {
-            req.url = originalUrl;
-            next();
-            return;
-          }
-
-          console.log('[indexes]', req.url, hostname, pathname, dirname);
-          dirname = path.resolve(conf.cwd, dirname.replace(/:hostname/, hostname));
-          if (!serveStaticMap[dirname]) {
-            serveIndexMap[dirname] = serveIndex(dirname);
-          }
-          serveIndexMap[dirname](req, res, nextIndex);
-        }
-
-        req.url = req.url.substr(pathname.length - 1);
-        nextIndex();
-      }
-    , app: function (config, hostname, pathname, req, res, next) {
-        //var appfile = path.resolve(/*process.cwd(), */config.path.replace(/:hostname/, hostname));
-        var appfile = config.path.replace(/:hostname/, hostname);
-        var app = require(appfile);
-        app(req, res, next);
-      }
-    };
-
-    function runModule(module, hostname, pathname, modulename, req, res, next) {
-      if (!loadables[modulename]) {
-        next(new Error("no module '" + modulename + "' found"));
-        return;
-      }
-      loadables[modulename](module, hostname, pathname, req, res, next);
-    }
-
-    function iterModules(modules, hostname, pathname, req, res, next) {
-      console.log('modules');
-      console.log(modules);
-      var modulenames = Object.keys(modules._map);
-
-      function nextModule() {
-        var modulename = modulenames.pop();
-        if (!modulename) {
-          next();
-          return;
-        }
-
-        console.log('modules', modules);
-        runModule(modules._map[modulename], hostname, pathname, modulename, req, res, nextModule);
-      }
-
-      nextModule();
-    }
-
-    function iterPaths(site, hostname, req, res, next) {
-      console.log('site', hostname);
-      console.log(site);
-      var pathnames = Object.keys(site.paths._map);
-      console.log('pathnames', pathnames);
-      pathnames = pathnames.filter(function (pathname) {
-        // TODO ensure that pathname has trailing /
-        return (0 === req.url.indexOf(pathname));
-        //return req.url.match(pathname);
-      });
-      pathnames.sort(function (a, b) {
-        return b.length - a.length;
-      });
-      console.log('pathnames', pathnames);
-
-      function nextPath() {
-        var pathname = pathnames.shift();
-        if (!pathname) {
-          next();
-          return;
-        }
-
-        console.log('iterPaths', hostname, pathname, req.url);
-        iterModules(site.paths._map[pathname].modules, hostname, pathname, req, res, nextPath);
-      }
-
-      nextPath();
-    }
-
-    function nextSite() {
-      console.log('hostname', hostname, sites);
-      var site;
-      if (!sites.length) {
-        next(); // 404
-        return;
-      }
-      site = sites.shift();
-      if (!site) {
-        nextSite();
-        return;
-      }
-      iterPaths(site, hostname, req, res, nextSite);
-    }
-
-    nextSite();
-
-    /*
-    function serveStaticly(server) {
-      function serveTheStatic() {
-        server.serve(req, res, function (err) {
-          if (err) { return done(err); }
-          server.index(req, res, function (err) {
-            if (err) { return done(err); }
-            req.url = req.url.replace(/\/assets/, '');
-            assetServer(req, res, function  () {
-              if (err) { return done(err); }
-              req.url = req.url.replace(/\/\.well-known/, '');
-              wellKnownServer(req, res, done);
-            });
-          });
-        });
-      }
-
-      if (server.expressApp) {
-        server.expressApp(req, res, serveTheStatic);
-        return;
-      }
-
-      serveTheStatic();
-    }
-
-    if (opts.livereload) {
-      res.__my_livereload = '<script src="//'
-        + (host || opts.sites[0].name).split(':')[0]
-        + ':35729/livereload.js?snipver=1"></script>';
-      res.__my_addLen = res.__my_livereload.length;
-
-      // TODO modify prototype instead of each instance?
-      res.__write = res.write;
-      res.write = _reloadWrite;
-    }
-
-    console.log('hostname:', hostname, opts.sites[0].paths);
-
-    addServer(hostname);
-    server = hostsMap[hostname] || hostsMap[opts.sites[0].name];
-    serveStaticly(server);
-    */
-  });
-};

lib/check-ports.js

@@ -11,17 +11,6 @@ function bindTcpAndRelease(port, cb) {
   });
 }
 
-function bindUdpAndRelease(port, cb) {
-  var socket = require('dgram').createSocket('udp4');
-  socket.on('error', function (e) {
-    cb(e);
-  });
-  socket.bind(port, function () {
-    socket.close();
-    cb();
-  });
-}
-
 function checkTcpPorts(cb) {
   var bound = {};
   var failed = {};
@@ -62,34 +51,4 @@ function checkTcpPorts(cb) {
   });
 }
 
-function checkUdpPorts(cb) {
-  var bound = {};
-  var failed = {};
-
-  bindUdpAndRelease(53, function (e) {
-    if (e) {
-      failed[53] = e;
-    } else {
-      bound[53] = true;
-    }
-
-    if (bound[53]) {
-      cb(null, bound);
-      return;
-    }
-
-    console.warn("default DNS port 53 not available, trying 8053");
-    bindUdpAndRelease(8053, function (e) {
-      if (e) {
-        failed[8053] = e;
-      } else {
-        bound[8053] = true;
-      }
-
-      cb(failed, bound);
-    });
-  });
-}
-
 module.exports.checkTcpPorts = checkTcpPorts;
-module.exports.checkUdpPorts = checkUdpPorts;

lib/ddns.js

@@ -1,88 +0,0 @@
-'use strict';
-
-module.exports.create = function (opts/*, servers*/) {
-  var PromiseA = opts.PromiseA;
-  var dns = PromiseA.promisifyAll(require('dns'));
-
-  return PromiseA.all([
-    dns.resolve4Async(opts._old_server_name).then(function (results) {
-      return results;
-    }, function () {})
-  , dns.resolve6Async(opts._old_server_name).then(function (results) {
-      return results;
-    }, function () {})
-  ]).then(function (results) {
-    var ipv4 = results[0] || [];
-    var ipv6 = results[1] || [];
-    var record;
-
-    opts.dnsRecords = {
-      A: ipv4
-    , AAAA: ipv6
-    };
-
-    Object.keys(opts.ifaces).some(function (ifacename) {
-      var iface = opts.ifaces[ifacename];
-
-      return iface.ipv4.some(function (localIp) {
-        return ipv4.some(function (remoteIp) {
-          if (localIp.address === remoteIp) {
-            record = localIp;
-            return record;
-          }
-        });
-      }) || iface.ipv6.some(function (localIp) {
-        return ipv6.forEach(function (remoteIp) {
-          if (localIp.address === remoteIp) {
-            record = localIp;
-            return record;
-          }
-        });
-      });
-    });
-
-    if (!record) {
-      console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address.");
-      console.info("Use --ddns to allow the people of the Internet to access your server.");
-    }
-
-    opts.externalIps.ipv4.some(function (localIp) {
-      return ipv4.some(function (remoteIp) {
-        if (localIp.address === remoteIp) {
-          record = localIp;
-          return record;
-        }
-      });
-    });
-
-    opts.externalIps.ipv6.some(function (localIp) {
-      return ipv6.some(function (remoteIp) {
-        if (localIp.address === remoteIp) {
-          record = localIp;
-          return record;
-        }
-      });
-    });
-
-    if (!record) {
-      console.info("DNS Record '" + ipv4.concat(ipv6).join(',') + "' does not match any local IP address.");
-      console.info("Use --ddns to allow the people of the Internet to access your server.");
-    }
-  });
-};
-
-if (require.main === module) {
-  var opts = {
-    _old_server_name: 'aj.daplie.me'
-  , PromiseA: require('bluebird')
-  };
-  // ifaces
-  opts.ifaces = require('./local-ip.js').find();
-  console.log('opts.ifaces');
-  console.log(opts.ifaces);
-  require('./match-ips.js').match(opts._old_server_name, opts).then(function (ips) {
-    opts.matchingIps = ips.matchingIps || [];
-    opts.externalIps = ips.externalIps;
-    module.exports.create(opts);
-  });
-}

lib/ddns/challenge-responder.js

@@ -0,0 +1,122 @@
+'use strict';
+
+// Much of this file was based on the `le-challenge-ddns` library (which we are not using
+// here because it's method of setting records requires things we don't really want).
+module.exports.create = function (deps, conf, utils) {
+
+  function getReleventSessionId(domain) {
+    var sessId;
+
+    utils.iterateAllModules(function (mod, domainList) {
+      // We return a truthy value in these cases because of the way the iterate function
+      // handles modules grouped by domain. By returning true we are saying these domains
+      // are "handled" and so if there are multiple modules we won't be given the rest.
+      if (sessId) { return true; }
+      if (domainList.indexOf(domain) < 0) { return true; }
+
+      // But if the domains are relevant but we don't know how to handle the module we
+      // return false to allow us to look at any other modules that might exist here.
+      if (mod.type !== 'dns@oauth3.org')  { return false; }
+
+      sessId = mod.tokenId || mod.token_id;
+      return true;
+    });
+
+    return sessId;
+  }
+
+  function get(args, domain, challenge, done) {
+    done(new Error("Challenge.get() does not need an implementation for dns-01. (did you mean Challenge.loopback?)"));
+  }
+  // same as get, but external
+  function loopback(args, domain, challenge, done) {
+    var challengeDomain = (args.test || '') + args.acmeChallengeDns + domain;
+    require('dns').resolveTxt(challengeDomain, done);
+  }
+
+  var activeChallenges = {};
+  async function removeAsync(args, domain) {
+    var data = activeChallenges[domain];
+    if (!data) {
+      console.warn(new Error('cannot remove DNS challenge for ' + domain + ': already removed'));
+      return;
+    }
+
+    var session = await utils.getSession(data.sessId);
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+    var apiOpts = {
+      api: 'dns.unset'
+    , session: session
+    , type: 'TXT'
+    , value: data.keyAuthDigest
+    };
+    await deps.OAUTH3.api(directives.api, Object.assign({}, apiOpts, data.splitDomain));
+
+    delete activeChallenges[domain];
+  }
+  async function setAsync(args, domain, challenge, keyAuth) {
+    if (activeChallenges[domain]) {
+      await removeAsync(args, domain, challenge);
+    }
+
+    var sessId = getReleventSessionId(domain);
+    if (!sessId) {
+      throw new Error('no DDNS module handles the domain ' + domain);
+    }
+    var session = await utils.getSession(sessId);
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+
+    // I'm not sure what role challenge is supposed to play since even in the library
+    // this code is based on it was never used, but check for it anyway because ...
+    if (!challenge || keyAuth) {
+      console.warn(new Error('DDNS challenge missing challenge or keyAuth'));
+    }
+    var keyAuthDigest = require('crypto').createHash('sha256').update(keyAuth || '').digest('base64')
+      .replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+
+    var challengeDomain = (args.test || '') + args.acmeChallengeDns + domain;
+    var splitDomain = (await utils.splitDomains(directives.api, [challengeDomain]))[0];
+
+    var apiOpts = {
+      api: 'dns.set'
+    , session: session
+    , type: 'TXT'
+    , value: keyAuthDigest
+    , ttl: args.ttl || 0
+    };
+    await deps.OAUTH3.api(directives.api, Object.assign({}, apiOpts, splitDomain));
+
+    activeChallenges[domain] = {
+      sessId
+    , keyAuthDigest
+    , splitDomain
+    };
+
+    return new Promise(res => setTimeout(res, 1000));
+  }
+
+  // It might be slightly easier to use arguments and apply, but the library that will use
+  // this function counts the arguments we expect.
+  function set(a, b, c, d, done) {
+    setAsync(a, b, c, d).then(result => done(null, result), done);
+  }
+  function remove(a, b, c, done) {
+    removeAsync(a, b, c).then(result => done(null, result), done);
+  }
+
+  function getOptions() {
+    return {
+      oauth3: 'oauth3.org'
+    , debug: conf.debug
+    , acmeChallengeDns: '_acme-challenge.'
+    };
+  }
+
+  return {
+    getOptions
+  , set
+  , get
+  , remove
+  , loopback
+  };
+};

lib/ddns/dns-ctrl.js

@@ -0,0 +1,132 @@
+'use strict';
+
+module.exports.create = function (deps, conf, utils) {
+  function dnsType(addr) {
+    if (/^\d+\.\d+\.\d+\.\d+$/.test(addr)) {
+      return 'A';
+    }
+    if (-1 !== addr.indexOf(':') && /^[a-f:\.\d]+$/i.test(addr)) {
+      return 'AAAA';
+    }
+  }
+
+  async function setDeviceAddress(session, addr, domains) {
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+
+    // Set the address of the device to our public address.
+    await deps.request({
+      url: deps.OAUTH3.url.normalize(directives.api)+'/api/com.daplie.domains/acl/devices/' + conf.device.hostname
+    , method: 'POST'
+    , headers: {
+        'Authorization': 'Bearer ' + session.refresh_token
+      , 'Accept': 'application/json; charset=utf-8'
+      }
+    , json: {
+        addresses: [
+          { value: addr, type:  dnsType(addr) }
+        ]
+      }
+    });
+
+    // Then update all of the records attached to our hostname, first removing the old records
+    // to remove the reference to the old address, then creating new records for the same domains
+    // using our new address.
+    var allDns = await deps.OAUTH3.api(directives.api, {session: session, api: 'dns.list'});
+    var ourDns = allDns.filter(function (record) {
+      if (record.device !== conf.device.hostname) {
+        return false;
+      }
+      if ([ 'A', 'AAAA' ].indexOf(record.type) < 0) {
+        return false;
+      }
+      return domains.indexOf(record.host) !== -1;
+    });
+
+    // Of all the DNS records referring to our device and the current list of domains determine
+    // which domains have records with outdated address, and which ones we can just leave be
+    // without updating them.
+    var badAddrDomains = ourDns.filter(function (record) {
+      return record.value !== addr;
+    }).map(record => record.host);
+    var goodAddrDomains = ourDns.filter(function (record) {
+      return record.value === addr && badAddrDomains.indexOf(record.host) < 0;
+    }).map(record => record.host);
+    var requiredUpdates = domains.filter(function (domain) {
+      return goodAddrDomains.indexOf(domain) < 0;
+    });
+
+    var oldDns = await utils.splitDomains(directives.api, badAddrDomains);
+    var common = {
+      api: 'devices.detach'
+    , session: session
+    , device: conf.device.hostname
+    };
+    await deps.PromiseA.all(oldDns.map(function (record) {
+      return deps.OAUTH3.api(directives.api, Object.assign({}, common, record));
+    }));
+    if (conf.debug && badAddrDomains.length) {
+      console.log('removed bad DNS records for ' + badAddrDomains.join(', '));
+    }
+
+    var newDns = await utils.splitDomains(directives.api, requiredUpdates);
+    common = {
+      api: 'devices.attach'
+    , session: session
+    , device: conf.device.hostname
+    , ip: addr
+    , ttl: 300
+    };
+    await deps.PromiseA.all(newDns.map(function (record) {
+      return deps.OAUTH3.api(directives.api, Object.assign({}, common, record));
+    }));
+    if (conf.debug && requiredUpdates.length) {
+      console.log('set new DNS records for ' + requiredUpdates.join(', '));
+    }
+  }
+
+  async function getDeviceAddresses(session) {
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+
+    var result = await deps.request({
+      url: deps.OAUTH3.url.normalize(directives.api)+'/api/org.oauth3.dns/acl/devices'
+    , method: 'GET'
+    , headers: {
+        'Authorization': 'Bearer ' + session.refresh_token
+      , 'Accept': 'application/json; charset=utf-8'
+      }
+    , json: true
+    });
+
+    if (!result.body) {
+      throw new Error('No response body in request for device addresses');
+    }
+    if (result.body.error) {
+      throw Object.assign(new Error('error getting device list'), result.body.error);
+    }
+
+    var dev = result.body.devices.filter(function (dev) {
+      return dev.name === conf.device.hostname;
+    })[0];
+    return (dev || {}).addresses || [];
+  }
+
+  async function removeDomains(session, domains) {
+    var directives = await deps.OAUTH3.discover(session.token.aud);
+
+    var oldDns = await utils.splitDomains(directives.api, domains);
+    var common = {
+      api: 'devices.detach'
+    , session: session
+    , device: conf.device.hostname
+    };
+    await deps.PromiseA.all(oldDns.map(function (record) {
+      return deps.OAUTH3.api(directives.api, Object.assign({}, common, record));
+    }));
+  }
+
+  return {
+    getDeviceAddresses
+  , setDeviceAddress
+  , removeDomains
+  };
+};

lib/ddns/index.js

@@ -0,0 +1,326 @@
+'use strict';
+
+module.exports.create = function (deps, conf) {
+  var dns = deps.PromiseA.promisifyAll(require('dns'));
+  var network = deps.PromiseA.promisifyAll(deps.recase.camelCopy(require('network')));
+  var equal = require('deep-equal');
+
+  var utils = require('./utils').create(deps, conf);
+  var loopback = require('./loopback').create(deps, conf, utils);
+  var dnsCtrl = require('./dns-ctrl').create(deps, conf, utils);
+  var challenge = require('./challenge-responder').create(deps, conf, utils);
+  var tunnelClients = require('./tunnel-client-manager').create(deps, conf, utils);
+
+  var loopbackDomain;
+
+  var tunnelActive = false;
+  async function startTunnel(tunnelSession, mod, domainList) {
+    try {
+      var dnsSession = await utils.getSession(mod.tokenId);
+      var tunnelDomain = await tunnelClients.start(tunnelSession || dnsSession, domainList);
+
+      var addrList;
+      try {
+        addrList = await dns.resolve4Async(tunnelDomain);
+      } catch (e) {}
+      if (!addrList || !addrList.length) {
+        try {
+          addrList = await dns.resolve6Async(tunnelDomain);
+        } catch (e) {}
+      }
+      if (!addrList || !addrList.length || !addrList[0]) {
+        throw new Error('failed to lookup IP for tunnel domain "' + tunnelDomain + '"');
+      }
+
+      if (!mod.disabled) {
+        await dnsCtrl.setDeviceAddress(dnsSession, addrList[0], domainList);
+      }
+    } catch (err) {
+      console.log('error starting tunnel for', domainList.join(', '));
+      console.log(err);
+    }
+  }
+  async function connectAllTunnels() {
+    var tunnelSession;
+    if (conf.ddns.tunnel) {
+      // In the case of a non-existant token, I'm not sure if we want to throw here and prevent
+      // any tunnel connections, or if we  want to ignore it and fall back to the DNS tokens
+      tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId);
+    }
+
+    await utils.iterateAllModules(function (mod, domainList) {
+      if (mod.type !== 'dns@oauth3.org') { return null; }
+
+      return startTunnel(tunnelSession, mod, domainList);
+    });
+
+    tunnelActive = true;
+  }
+  async function disconnectTunnels() {
+    tunnelClients.disconnect();
+    tunnelActive = false;
+    await Promise.resolve();
+  }
+  async function checkTunnelTokens() {
+    var oldTokens = tunnelClients.current();
+
+    var newTokens = await utils.iterateAllModules(function checkTokens(mod, domainList) {
+      if (mod.type !== 'dns@oauth3.org') { return null; }
+
+      var domainStr = domainList.slice().sort().join(',');
+      // If there is already a token handling exactly the domains this modules
+      // needs handled remove it from the list of tokens to be removed. Otherwise
+      // return the module and domain list so we can get new tokens.
+      if (oldTokens[domainStr]) {
+        delete oldTokens[domainStr];
+      } else {
+        return Promise.resolve({ mod, domainList });
+      }
+    });
+
+    await Promise.all(Object.values(oldTokens).map(tunnelClients.remove));
+
+    if (!newTokens.length) { return; }
+
+    var tunnelSession;
+    if (conf.ddns.tunnel) {
+      // In the case of a non-existant token, I'm not sure if we want to throw here and prevent
+      // any tunnel connections, or if we  want to ignore it and fall back to the DNS tokens
+      tunnelSession = await deps.storage.tokens.get(conf.ddns.tunnel.tokenId);
+    }
+
+    await Promise.all(newTokens.map(function ({mod, domainList}) {
+      return startTunnel(tunnelSession, mod, domainList);
+    }));
+  }
+
+  var localAddr, gateway;
+  async function checkNetworkEnv() {
+    // Since we can't detect the OS level events when a user plugs in an ethernet cable to recheck
+    // what network environment we are in we check our local network address and the gateway to
+    // determine if we need to run the loopback check and router configuration again.
+    var addr = await network.getPrivateIpAsync();
+    // Until the author of the `network` package publishes the pull request we gave him
+    // checking the gateway on our units fails because we have the busybox versions of
+    // the linux commands. Gateway is realistically less important than address, so if
+    // we fail in getting it go ahead and use the null value.
+    var gw;
+    try {
+      gw = await network.getGatewayIpAsync();
+    } catch (err) {
+      gw = null;
+    }
+    if (localAddr === addr && gateway === gw) {
+      return;
+    }
+
+    var loopResult = await loopback(loopbackDomain);
+    var notLooped = Object.keys(loopResult.ports).filter(function (port) {
+      return !loopResult.ports[port];
+    });
+
+    // if (notLooped.length) {
+    //   // TODO: try to automatically configure router to forward ports to us.
+    // }
+
+    // If we are on a public address or all ports we are listening on are forwarded to us then
+    // we don't need the tunnel and we can set the DNS records for all our domains to our public
+    // address. Otherwise we need to use the tunnel to accept traffic. Also since the tunnel will
+    // only be listening on ports 80 and 443 if those are forwarded to us we don't want the tunnel.
+    if (!notLooped.length || (loopResult.ports['80'] && loopResult.ports['443'])) {
+      if (tunnelActive) {
+        await disconnectTunnels();
+      }
+    } else {
+      if (!tunnelActive) {
+        await connectAllTunnels();
+      }
+    }
+
+    // Don't assign these until the end of the function. This means that if something failed
+    // in the loopback or tunnel connection that we will try to go through the whole process
+    // again next time and hopefully the error is temporary (but if not I'm not sure what the
+    // correct course of action would be anyway).
+    localAddr = addr;
+    gateway = gw;
+  }
+
+  var publicAddress;
+  async function recheckPubAddr() {
+    await checkNetworkEnv();
+    if (tunnelActive) {
+      return;
+    }
+
+    var addr = await loopback.checkPublicAddr(loopbackDomain);
+    if (publicAddress === addr) {
+      return;
+    }
+
+    if (conf.debug) {
+      console.log('previous public address',publicAddress, 'does not match current public address', addr);
+    }
+    publicAddress = addr;
+
+    await utils.iterateAllModules(function setModuleDNS(mod, domainList) {
+      if (mod.type !== 'dns@oauth3.org' || mod.disabled) { return null; }
+
+      return utils.getSession(mod.tokenId).then(function (session) {
+        return dnsCtrl.setDeviceAddress(session, addr, domainList);
+      }).catch(function (err) {
+        console.log('error setting DNS records for', domainList.join(', '));
+        console.log(err);
+      });
+    });
+  }
+
+  function getModuleDiffs(prevConf) {
+    var prevMods = {};
+    var curMods = {};
+
+    // this returns a Promise, but since the functions we use are synchronous
+    // and change our enclosed variables we don't need to wait for the return.
+    utils.iterateAllModules(function (mod, domainList) {
+      if (mod.type !== 'dns@oauth3.org') { return; }
+
+      prevMods[mod.id] = { mod, domainList };
+      return true;
+    }, prevConf);
+    utils.iterateAllModules(function (mod, domainList) {
+      if (mod.type !== 'dns@oauth3.org') { return; }
+
+      curMods[mod.id] = { mod, domainList };
+      return true;
+    });
+
+    // Filter out all of the modules that are exactly the same including domainList
+    // since there is no required action to transition.
+    Object.keys(prevMods).map(function (id) {
+      if (equal(prevMods[id], curMods[id])) {
+        delete prevMods[id];
+        delete curMods[id];
+      }
+    });
+
+    return {prevMods, curMods};
+  }
+  async function cleanOldDns(prevConf) {
+    var {prevMods, curMods} = getModuleDiffs(prevConf);
+
+    // Then remove DNS records for the domains that we are no longer responsible for.
+    await Promise.all(Object.values(prevMods).map(function ({mod, domainList}) {
+      // If the module was disabled before there should be any records that we need to clean up
+      if (mod.disabled) { return; }
+
+      var oldDomains;
+      if (!curMods[mod.id] || curMods[mod.id].disabled || mod.tokenId !== curMods[mod.id].mod.tokenId) {
+        oldDomains = domainList.slice();
+      } else {
+        oldDomains = domainList.filter(function (domain) {
+          return curMods[mod.id].domainList.indexOf(domain) < 0;
+        });
+      }
+      if (conf.debug) {
+        console.log('removing old domains for module', mod.id, oldDomains.join(', '));
+      }
+      if (!oldDomains.length) {
+        return;
+      }
+
+      return utils.getSession(mod.tokenId).then(function (session) {
+        return dnsCtrl.removeDomains(session, oldDomains);
+      });
+    }).filter(Boolean));
+  }
+  async function setNewDns(prevConf) {
+    var {prevMods, curMods} = getModuleDiffs(prevConf);
+
+    // And add DNS records for any newly added domains.
+    await Promise.all(Object.values(curMods).map(function ({mod, domainList}) {
+      // Don't set any new records if the module has been disabled.
+      if (mod.disabled) { return; }
+
+      var newDomains;
+      if (!prevMods[mod.id] || mod.tokenId !== prevMods[mod.id].mod.tokenId) {
+        newDomains = domainList.slice();
+      } else {
+        newDomains = domainList.filter(function (domain) {
+          return prevMods[mod.id].domainList.indexOf(domain) < 0;
+        });
+      }
+      if (conf.debug) {
+        console.log('adding new domains for module', mod.id, newDomains.join(', '));
+      }
+      if (!newDomains.length) {
+        return;
+      }
+
+      return utils.getSession(mod.tokenId).then(function (session) {
+        return dnsCtrl.setDeviceAddress(session, publicAddress, newDomains);
+      });
+    }).filter(Boolean));
+  }
+
+  function check() {
+    recheckPubAddr().catch(function (err) {
+      console.error('failed to handle all actions needed for DDNS');
+      console.error(err);
+    });
+  }
+  check();
+  setInterval(check, 5*60*1000);
+
+  var curConf;
+  function updateConf() {
+    if (curConf && equal(curConf.ddns, conf.ddns) && equal(curConf.domains, conf.domains)) {
+      // We could update curConf, but since everything we care about is the same...
+      return;
+    }
+
+    if (!curConf || !equal(curConf.ddns.loopback, conf.ddns.loopback)) {
+      loopbackDomain = 'oauth3.org';
+      if (conf.ddns && conf.ddns.loopback) {
+        if (conf.ddns.loopback.type === 'tunnel@oauth3.org' && conf.ddns.loopback.domain) {
+          loopbackDomain = conf.ddns.loopback.domain;
+        } else {
+          console.error('invalid loopback configuration: bad type or missing domain');
+        }
+      }
+    }
+
+    if (!curConf) {
+      // We need to make a deep copy of the config so we can use it next time to
+      // compare and see what setup/cleanup is needed to adapt to the changes.
+      curConf = JSON.parse(JSON.stringify(conf));
+      return;
+    }
+
+    cleanOldDns(curConf).then(function () {
+      if (!tunnelActive) {
+        return setNewDns(curConf);
+      }
+      if (equal(curConf.ddns.tunnel, conf.ddns.tunnel)) {
+        return checkTunnelTokens();
+      } else {
+        return disconnectTunnels().then(connectAllTunnels);
+      }
+    }).catch(function (err) {
+      console.error('error transitioning DNS between configurations');
+      console.error(err);
+    }).then(function () {
+      // We need to make a deep copy of the config so we can use it next time to
+      // compare and see what setup/cleanup is needed to adapt to the changes.
+      curConf = JSON.parse(JSON.stringify(conf));
+    });
+  }
+  updateConf();
+
+  return {
+    loopbackServer:     loopback.server
+  , setDeviceAddress:   dnsCtrl.setDeviceAddress
+  , getDeviceAddresses: dnsCtrl.getDeviceAddresses
+  , recheckPubAddr:     recheckPubAddr
+  , updateConf:         updateConf
+  , challenge
+  };
+};

lib/ddns/loopback.js

@@ -0,0 +1,116 @@
+'use strict';
+
+module.exports.create = function (deps, conf) {
+  var pending = {};
+
+  async function _checkPublicAddr(host) {
+    var result = await deps.request({
+      method: 'GET'
+    , url: deps.OAUTH3.url.normalize(host)+'/api/org.oauth3.tunnel/checkip'
+    , json: true
+    });
+
+    if (!result.body) {
+      throw new Error('No response body in request for public address');
+    }
+    if (result.body.error) {
+      // Note that the error on the body will probably have a message that overwrites the default
+      throw Object.assign(new Error('error in check IP response'), result.body.error);
+    }
+    if (!result.body.address) {
+      throw new Error("public address resonse doesn't contain address: "+JSON.stringify(result.body));
+    }
+    return result.body.address;
+  }
+  async function checkPublicAddr(provider) {
+    var directives = await deps.OAUTH3.discover(provider);
+    return _checkPublicAddr(directives.api);
+  }
+
+  async function checkSinglePort(host, address, port) {
+    var crypto = require('crypto');
+    var token   = crypto.randomBytes(8).toString('hex');
+    var keyAuth = crypto.randomBytes(32).toString('hex');
+    pending[token] = keyAuth;
+
+    var reqObj = {
+      method: 'POST'
+    , url: deps.OAUTH3.url.normalize(host)+'/api/org.oauth3.tunnel/loopback'
+    , timeout: 20*1000
+    , json: {
+        address: address
+      , port: port
+      , token: token
+      , keyAuthorization: keyAuth
+      , iat: Date.now()
+      , timeout: 18*1000
+      }
+    };
+
+    var result;
+    try {
+      result = await deps.request(reqObj);
+    } catch (err) {
+      delete pending[token];
+      if (conf.debug) {
+        console.log('error making loopback request for port ' + port + ' loopback', err.message);
+      }
+      return false;
+    }
+
+    delete pending[token];
+    if (!result.body) {
+      if (conf.debug) {
+        console.log('No response body in loopback request for port '+port);
+      }
+      return false;
+    }
+    // If the loopback requests don't go to us then there are all kinds of ways it could
+    // error, but none of them really provide much extra information so we don't do
+    // anything that will break the PromiseA.all out and mask the other results.
+    if (conf.debug && result.body.error) {
+      console.log('error on remote side of port '+port+' loopback', result.body.error);
+    }
+    return !!result.body.success;
+  }
+
+  async function loopback(provider) {
+    var directives = await deps.OAUTH3.discover(provider);
+    var address = await _checkPublicAddr(directives.api);
+    if (conf.debug) {
+      console.log('checking to see if', address, 'gets back to us');
+    }
+
+    var ports = require('../servers').listeners.tcp.list();
+    var values = await deps.PromiseA.all(ports.map(function (port) {
+      return checkSinglePort(directives.api, address, port);
+    }));
+
+    if (conf.debug && Object.keys(pending).length) {
+      console.log('remaining loopback tokens', pending);
+    }
+
+    return {
+      address: address
+    , ports: ports.reduce(function (obj, port, ind) {
+        obj[port] = values[ind];
+        return obj;
+      }, {})
+    };
+  }
+
+  loopback.checkPublicAddr = checkPublicAddr;
+  loopback.server = require('http').createServer(function (req, res) {
+    var parsed = require('url').parse(req.url);
+    var token = parsed.pathname.replace('/.well-known/cloud-challenge/', '');
+    if (pending[token]) {
+      res.setHeader('Content-Type', 'text/plain');
+      res.end(pending[token]);
+    } else {
+      res.statusCode = 404;
+      res.end();
+    }
+  });
+
+  return loopback;
+};

lib/ddns/tunnel-client-manager.js

@@ -0,0 +1,191 @@
+'use strict';
+
+module.exports.create = function (deps, config) {
+  var stunnel = require('stunnel');
+  var jwt = require('jsonwebtoken');
+  var activeTunnels = {};
+  var activeDomains = {};
+
+  var customNet = {
+    createConnection: function (opts, cb) {
+      console.log('[gl.tunnel] creating connection');
+
+      // here "reader" means the socket that looks like the connection being accepted
+      // here "writer" means the remote-looking part of the socket that driving the connection
+      var writer;
+
+      function usePair(err, reader) {
+        if (err) {
+          process.nextTick(function () {
+            writer.emit('error', err);
+          });
+          return;
+        }
+
+        var wrapOpts = Object.assign({localAddress: '127.0.0.2', localPort: 'tunnel-0'}, opts);
+        wrapOpts.firstChunk = opts.data;
+        wrapOpts.hyperPeek = !!opts.data;
+
+        // Also override the remote and local address info. We use `defineProperty` because
+        // otherwise we run into problems of setting properties with only getters defined.
+        Object.defineProperty(reader, 'remoteAddress', { value: wrapOpts.remoteAddress });
+        Object.defineProperty(reader, 'remotePort',    { value: wrapOpts.remotePort });
+        Object.defineProperty(reader, 'remoteFamiliy', { value: wrapOpts.remoteFamiliy });
+        Object.defineProperty(reader, 'localAddress',  { value: wrapOpts.localAddress });
+        Object.defineProperty(reader, 'localPort',     { value: wrapOpts.localPort });
+        Object.defineProperty(reader, 'localFamiliy',  { value: wrapOpts.localFamiliy });
+
+        deps.tcp.handler(reader, wrapOpts);
+        process.nextTick(function () {
+          // this cb will cause the stream to emit its (actually) first data event
+          // (even though it already gave a peek into that first data chunk)
+          console.log('[tunnel] callback, data should begin to flow');
+          cb();
+        });
+      }
+
+      // We used to use `stream-pair` for non-tls connections, but there are places
+      // that require properties/functions to be present on the socket that aren't
+      // present on a JSStream so it caused problems.
+      writer = require('socket-pair').create(usePair);
+      return writer;
+    }
+  };
+
+  function fillData(data) {
+    if (typeof data === 'string') {
+      data = { jwt: data };
+    }
+
+    if (!data.jwt) {
+      throw new Error("missing 'jwt' from tunnel data");
+    }
+    var decoded = jwt.decode(data.jwt);
+    if (!decoded) {
+      throw new Error('invalid JWT');
+    }
+
+    if (!data.tunnelUrl) {
+      if (!decoded.aud) {
+        throw new Error('missing tunnelUrl and audience');
+      }
+      data.tunnelUrl = 'wss://' + decoded.aud + '/';
+    }
+
+    data.domains = (decoded.domains || []).slice().sort().join(',');
+    if (!data.domains) {
+      throw new Error('JWT contains no domains to be forwarded');
+    }
+
+    return data;
+  }
+
+  async function removeToken(data) {
+    data = fillData(data);
+
+    // Not sure if we might want to throw an error indicating the token didn't
+    // even belong to a  server that existed, but since it never existed we can
+    // consider it as "removed".
+    if (!activeTunnels[data.tunnelUrl]) {
+      return;
+    }
+
+    console.log('removing token from tunnel at', data.tunnelUrl);
+    return activeTunnels[data.tunnelUrl].clear(data.jwt).then(function () {
+      delete activeDomains[data.domains];
+    });
+  }
+
+  async function addToken(data) {
+    data = fillData(data);
+
+    if (activeDomains[data.domains]) {
+      // If already have a token with the exact same domains and to the same tunnel
+      // server there isn't really a need to add a new one
+      if (activeDomains[data.domains].tunnelUrl === data.tunnelUrl) {
+        return;
+      }
+      // Otherwise we want to detach from the other tunnel server in favor of the new one
+      console.warn('added token with the exact same domains as another');
+      await removeToken(activeDomains[data.domains]);
+    }
+
+    if (!activeTunnels[data.tunnelUrl]) {
+      console.log('creating new tunnel client for', data.tunnelUrl);
+      // We create the tunnel without an initial token so we can append the token and
+      // get the promise that should tell us more about if it worked or not.
+      activeTunnels[data.tunnelUrl] = stunnel.connect({
+        stunneld: data.tunnelUrl
+      , net: customNet
+        // NOTE: the ports here aren't that important since we are providing a custom
+        // `net.createConnection` that doesn't actually use the port. What is important
+        // is that any services we are interested in are listed in this object and have
+        // a '*' sub-property.
+      , services: {
+          https: { '*': 443 }
+        , http:  { '*': 80 }
+        , smtp:  { '*': 25 }
+        , smtps: { '*': 587 /*also 465/starttls*/ }
+        , ssh:   { '*': 22 }
+        }
+      });
+    }
+
+    console.log('appending token to tunnel at', data.tunnelUrl, 'for domains', data.domains);
+    await activeTunnels[data.tunnelUrl].append(data.jwt);
+
+    // Now that we know the tunnel server accepted our token we can save it
+    // to keep record of what domains we are handling and what tunnel server
+    // those domains should go to.
+    activeDomains[data.domains] = data;
+
+    // This is mostly for the start, but return the host for the tunnel server
+    // we've connected to (after stripping the protocol and path away).
+    return data.tunnelUrl.replace(/^[a-z]*:\/\//i, '').replace(/\/.*/, '');
+  }
+
+  async function acquireToken(session, domains) {
+    var OAUTH3 = deps.OAUTH3;
+
+    // The OAUTH3 library stores some things on the root session object that we usually
+    // just leave inside the token, but we need to pull those out before we use it here
+    session.provider_uri = session.provider_uri || session.token.provider_uri || session.token.iss;
+    session.client_uri = session.client_uri || session.token.azp;
+    session.scope = session.scope || session.token.scp;
+
+    console.log('asking for tunnel token from', session.token.aud);
+    var opts = {
+      api: 'tunnel.token'
+    , session: session
+    , data: {
+        domains: domains
+      , device: {
+          hostname: config.device.hostname
+        , id: config.device.uid || config.device.id
+        }
+      }
+    };
+
+    var directives = await OAUTH3.discover(session.token.aud);
+    var tokenData = await OAUTH3.api(directives.api, opts);
+    return addToken(tokenData);
+  }
+
+  function disconnectAll() {
+    Object.keys(activeTunnels).forEach(function (key) {
+      activeTunnels[key].end();
+    });
+  }
+
+  function currentTokens() {
+    return JSON.parse(JSON.stringify(activeDomains));
+  }
+
+  return {
+    start:       acquireToken
+  , startDirect: addToken
+  , remove:      removeToken
+  , disconnect:  disconnectAll
+  , current:     currentTokens
+  };
+};

lib/ddns/utils.js

@@ -0,0 +1,102 @@
+'use strict';
+
+module.exports.create = function (deps, conf) {
+
+  async function getSession(id) {
+    var session = await deps.storage.tokens.get(id);
+    if (!session) {
+      throw new Error('no user token with ID "' + id + '"');
+    }
+    return session;
+  }
+
+  function iterateAllModules(action, curConf) {
+    curConf = curConf || conf;
+    var promises = [];
+
+    curConf.domains.forEach(function (dom) {
+      if (!dom.modules || !Array.isArray(dom.modules.ddns) || !dom.modules.ddns.length) {
+        return null;
+      }
+
+      // For the time being all of our things should only be tried once (regardless if it succeeded)
+      // TODO: revisit this behavior when we support multiple ways of setting records, and/or
+      // if we want to allow later modules to run if early modules fail.
+      promises.push(dom.modules.ddns.reduce(function (prom, mod) {
+        if (prom) { return prom; }
+        return action(mod, dom.names);
+      }, null));
+    });
+
+    curConf.ddns.modules.forEach(function (mod) {
+      promises.push(action(mod, mod.domains));
+    });
+
+    return Promise.all(promises.filter(Boolean));
+  }
+
+  var tldCache = {};
+  async function updateTldCache(provider) {
+    var reqObj = {
+      url: deps.OAUTH3.url.normalize(provider) + '/api/com.daplie.domains/prices'
+    , method: 'GET'
+    , json: true
+    };
+
+    var resp = await deps.OAUTH3.request(reqObj);
+    var tldObj = {};
+    resp.data.forEach(function (tldInfo) {
+      if (tldInfo.enabled) {
+        tldObj[tldInfo.tld] = true;
+      }
+    });
+
+    tldCache[provider] = {
+      time: Date.now()
+    , tlds: tldObj
+    };
+    return tldObj;
+  }
+  async function getTlds(provider) {
+    // If we've never cached the results we need to return the promise that will fetch the result,
+    // otherwise we can return the cached value. If the cached value has "expired", we can still
+    // return the cached value we just want to update the cache in parellel (making sure we only
+    // update once).
+    if (!tldCache[provider]) {
+      tldCache[provider] = {
+        updating: true
+      , tlds: updateTldCache(provider)
+      };
+    }
+    if (!tldCache[provider].updating && Date.now() - tldCache[provider].time > 24 * 60 * 60 * 1000) {
+      tldCache[provider].updating = true;
+      updateTldCache(provider);
+    }
+
+    return tldCache[provider].tlds;
+  }
+
+  async function splitDomains(provider, domains) {
+    var tlds = await getTlds(provider);
+    return domains.map(function (domain) {
+      var split = domain.split('.');
+      var tldSegCnt = tlds[split.slice(-2).join('.')] ? 2 : 1;
+
+      // Currently assuming that the sld can't contain dots, and that the tld can have at
+      // most one dot. Not 100% sure this is a valid assumption, but exceptions should be
+      // rare even if the assumption isn't valid.
+      return {
+        tld: split.slice(-tldSegCnt).join('.')
+      , sld: split.slice(-tldSegCnt - 1, -tldSegCnt).join('.')
+      , sub: split.slice(0, -tldSegCnt - 1).join('.')
+      };
+    });
+  }
+
+  return {
+    getSession
+  , iterateAllModules
+  , getTlds
+  , splitDomains
+  };
+};

lib/domain-utils.js

@@ -0,0 +1,30 @@
+'use strict';
+
+module.exports.match = function (pattern, domainname) {
+  // Everything matches '*'
+  if (pattern === '*') {
+    return true;
+  }
+
+  if (/^\*./.test(pattern)) {
+    // get rid of the leading "*." to more easily check the servername against it
+    pattern = pattern.slice(2);
+    return pattern === domainname.slice(-pattern.length);
+  }
+
+  // pattern doesn't contains any wildcards, so exact match is required
+  return pattern === domainname;
+};
+
+module.exports.separatePort = function (fullHost) {
+  var match = /^(.*?)(:\d+)?$/.exec(fullHost);
+
+  if (match[2]) {
+    match[2] = match[2].replace(':', '');
+  }
+
+  return {
+    host: match[1]
+  , port: match[2]
+  };
+};

lib/goldilocks.js

@@ -1,253 +0,0 @@
-'use strict';
-
-module.exports.create = function (deps, config) {
-  console.log('config', config);
-
-  //var PromiseA = global.Promise;
-  var PromiseA = require('bluebird');
-  var listeners = require('./servers').listeners;
-  var modules;
-
-  function loadModules() {
-    modules = {};
-
-    modules.tls = require('./modules/tls').create(deps, config, netHandler);
-    modules.http = require('./modules/http.js').create(deps, config, modules.tls.middleware);
-  }
-
-  // opts = { servername, encrypted, peek, data, remoteAddress, remotePort }
-  function peek(conn, firstChunk, opts) {
-    if (!modules) {
-      loadModules();
-    }
-
-    opts.firstChunk = firstChunk;
-    conn.__opts = opts;
-    // TODO port/service-based routing can do here
-
-    // TLS byte 1 is handshake and byte 6 is client hello
-    if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) {
-      modules.tls.emit('connection', conn);
-      return;
-    }
-
-    // This doesn't work with TLS, but now that we know this isn't a TLS connection we can
-    // unshift the first chunk back onto the connection for future use. The unshift should
-    // happen after any listeners are attached to it but before any new data comes in.
-    if (!opts.hyperPeek) {
-      process.nextTick(function () {
-        conn.unshift(firstChunk);
-      });
-    }
-
-    // Connection is not TLS, check for HTTP next.
-    if (firstChunk[0] > 32 && firstChunk[0] < 127) {
-      var firstStr = firstChunk.toString();
-      if (/HTTP\//i.test(firstStr)) {
-        modules.http.emit('connection', conn);
-        return;
-      }
-    }
-
-    console.warn('failed to identify protocol from first chunk', firstChunk);
-    conn.close();
-  }
-  function netHandler(conn, opts) {
-    opts = opts || {};
-    console.log('[netHandler]', conn.localAddress, conn.localPort, opts.encrypted);
-
-    // XXX PEEK COMMENT XXX
-    // TODO we can have our cake and eat it too
-    // we can skip the need to wrap the TLS connection twice
-    // because we've already peeked at the data,
-    // but this needs to be handled better before we enable that
-    // (because it creates new edge cases)
-    if (opts.hyperPeek) {
-      console.log('hyperpeek');
-      peek(conn, opts.firstChunk, opts);
-      return;
-    }
-
-    conn.once('data', function (chunk) {
-      peek(conn, chunk, opts);
-    });
-  }
-
-  function dnsListener(msg) {
-    var dgram = require('dgram');
-    var socket = dgram.createSocket('udp4');
-    socket.send(msg, config.dns.proxy.port, config.dns.proxy.address || '127.0.0.1');
-  }
-
-  function createTcpForwarder(mod) {
-    var destination = mod.address.split(':');
-    var connected = false;
-
-    return function (conn) {
-      var newConn = deps.net.createConnection({
-        port: destination[1]
-      , host: destination[0] || '127.0.0.1'
-
-      , remoteFamily:  conn.remoteFamily
-      , remoteAddress: conn.remoteAddress
-      , remotePort:    conn.remotePort
-      }, function () {
-        connected = true;
-
-        newConn.pipe(conn);
-        conn.pipe(newConn);
-      });
-
-      // Not sure how to effectively report this to the user or client, but we need to listen
-      // for the event to prevent it from crashing us.
-      newConn.on('error', function (err) {
-        if (connected) {
-          console.error('TCP forward remote error', err);
-          conn.end();
-        } else {
-          console.log('TCP forward connection error', err);
-          require('./proxy-err-resp').sendBadGateway(conn, err, config.debug);
-        }
-      });
-      conn.on('error', function (err) {
-        console.error('TCP forward client error', err);
-        newConn.end();
-      });
-
-    };
-  }
-
-  deps.tunnel = deps.tunnel || {};
-  deps.tunnel.net = {
-    createConnection: function (opts, cb) {
-      console.log('[gl.tunnel] creating connection');
-
-      // here "reader" means the socket that looks like the connection being accepted
-      // here "writer" means the remote-looking part of the socket that driving the connection
-      var writer;
-      var wrapOpts = {};
-      var rawTls = opts.tls || (0x16 === opts.data[0]) && (0x01 === opts.data[5]);
-
-      function usePair(err, reader) {
-        if (err) {
-          process.nextTick(function () {
-            writer.emit('error', err);
-          });
-          return;
-        }
-
-        // this has the normal net/tcp stuff plus our custom stuff
-        // opts = { address, port,
-        //          hostname, servername, tls, encrypted, data, localAddress, localPort, remoteAddress, remotePort, remoteFamily }
-        Object.keys(opts).forEach(function (key) {
-          wrapOpts[key] = opts[key];
-          try {
-            reader[key] = opts[key];
-          } catch(e) {
-            // can't set real socket getters, like remoteAddr
-          }
-        });
-
-        // A few more extra specialty options
-        wrapOpts.localAddress = wrapOpts.localAddress || '127.0.0.2'; // TODO use the tunnel's external address
-        wrapOpts.localPort = wrapOpts.localPort || 'tunnel-0';
-        try {
-          reader._remoteAddress = wrapOpts.remoteAddress;
-          reader._remotePort = wrapOpts.remotePort;
-          reader._remoteFamily = wrapOpts.remoteFamily;
-          reader._localAddress = wrapOpts.localAddress;
-          reader._localPort = wrapOpts.localPort;
-          reader._localFamily = wrapOpts.localFamily;
-        } catch(e) {
-        }
-
-        netHandler(reader, wrapOpts);
-
-        process.nextTick(function () {
-          //opts.data = wrapOpts.data;
-
-          // this cb will cause the stream to emit its (actually) first data event
-          // (even though it already gave a peek into that first data chunk)
-          console.log('[tunnel] callback, data should begin to flow');
-          cb();
-        });
-      }
-
-      wrapOpts.firstChunk = opts.data;
-      wrapOpts.hyperPeek = !!opts.data;
-      // encrypted meaning is *terminated* TLS
-      // tls meaning is *raw* TLS
-      if (rawTls) {
-        // TLS sockets must actually use a socket with a file descriptor
-        // https://nodejs.org/api/net.html#net_class_net_socket
-
-        writer = require('socket-pair').create(function (err, other) {
-          usePair(err, other);
-        });
-      }
-      else {
-        // stream-pair can only be used by TCP sockets, not tls
-        writer = require('stream-pair').create();
-        usePair(null, writer.other);
-      }
-
-      return writer;
-    }
-  };
-
-  var listenPromises = [];
-  var tcpPortMap = {};
-  function addPorts(bindList) {
-    if (!bindList) {
-      return;
-    }
-    if (Array.isArray(bindList)) {
-      bindList.forEach(function (port) {
-        tcpPortMap[port] = true;
-      });
-    }
-    else {
-      tcpPortMap[bindList] = true;
-    }
-  }
-
-  addPorts(config.tcp.bind);
-  (config.tcp.modules || []).forEach(function (mod) {
-    if (mod.name === 'forward') {
-      var forwarder = createTcpForwarder(mod);
-      mod.ports.forEach(function (port) {
-        if (!tcpPortMap[port]) {
-          console.log("forwarding port", port, "that wasn't specified in bind");
-        } else {
-          delete tcpPortMap[port];
-        }
-        listenPromises.push(listeners.tcp.add(port, forwarder));
-      });
-    }
-    else {
-      console.warn('unknown TCP module specified', mod);
-    }
-  });
-
-  // Even though these ports were specified in different places we treat any TCP
-  // connections we haven't been told to just forward exactly as is equal so that
-  // we can potentially use the same ports for different protocols.
-  addPorts(config.tls.bind);
-  addPorts(config.http.bind);
-
-  Object.keys(tcpPortMap).forEach(function (port) {
-    listenPromises.push(listeners.tcp.add(port, netHandler));
-  });
-
-  if (config.dns.bind) {
-    if (Array.isArray(config.dns.bind)) {
-      config.dns.bind.map(function (port) {
-        listenPromises.push(listeners.udp.add(port, dnsListener));
-      });
-    } else {
-      listenPromises.push(listeners.udp.add(config.dns.bind, dnsListener));
-    }
-  }
-
-  return PromiseA.all(listenPromises);
-};

lib/match-domain.js

@@ -1,17 +0,0 @@
-'use strict';
-
-module.exports.match = function (pattern, servername) {
-  // Everything matches '*'
-  if (pattern === '*') {
-    return true;
-  }
-
-  if (/^\*./.test(pattern)) {
-    // get rid of the leading "*." to more easily check the servername against it
-    pattern = pattern.slice(2);
-    return pattern === servername.slice(-pattern.length);
-  }
-
-  // pattern doesn't contains any wildcards, so exact match is required
-  return pattern === servername;
-};

lib/match-ips.js

@@ -1,117 +0,0 @@
-'use strict';
-
-var PromiseA = require('bluebird');
-
-module.exports.match = function (servername, opts) {
-  return PromiseA.promisify(require('ipify'))().then(function (externalIp) {
-    var dns = PromiseA.promisifyAll(require('dns'));
-
-    opts.externalIps = [ { address: externalIp, family: 'IPv4' } ];
-    opts.ifaces = require('./local-ip.js').find({ externals: opts.externalIps });
-    opts.externalIfaces = Object.keys(opts.ifaces).reduce(function (all, iname) {
-      var iface = opts.ifaces[iname];
-
-      iface.ipv4.forEach(function (addr) {
-        if (addr.external) {
-          addr.iface = iname;
-          all.push(addr);
-        }
-      });
-      iface.ipv6.forEach(function (addr) {
-        if (addr.external) {
-          addr.iface = iname;
-          all.push(addr);
-        }
-      });
-
-      return all;
-    }, []).filter(Boolean);
-
-    function resolveIps(hostname) {
-      var allIps = [];
-
-      return PromiseA.all([
-        dns.resolve4Async(hostname).then(function (records) {
-            records.forEach(function (ip) {
-              allIps.push({
-                address: ip
-              , family: 'IPv4'
-              });
-            });
-          }, function () {})
-        , dns.resolve6Async(hostname).then(function (records) {
-            records.forEach(function (ip) {
-              allIps.push({
-                address: ip
-              , family: 'IPv6'
-              });
-            });
-          }, function () {})
-      ]).then(function () {
-        return allIps;
-      });
-    }
-
-    function resolveIpsAndCnames(hostname) {
-      return PromiseA.all([
-        resolveIps(hostname)
-      , dns.resolveCnameAsync(hostname).then(function (records) {
-          return PromiseA.all(records.map(function (hostname) {
-            return resolveIps(hostname);
-          })).then(function (allIps) {
-            return allIps.reduce(function (all, ips) {
-              return all.concat(ips);
-            }, []);
-          });
-        }, function () {
-          return [];
-        })
-      ]).then(function (ips) {
-        return ips.reduce(function (all, set) {
-          return all.concat(set);
-        }, []);
-      });
-    }
-
-    return resolveIpsAndCnames(servername).then(function (allIps) {
-      var matchingIps = [];
-
-      if (!allIps.length) {
-        console.warn("Could not resolve '" + servername + "'");
-      }
-
-      // { address, family }
-      allIps.some(function (ip) {
-        function match(addr) {
-          if (ip.address === addr.address) {
-            matchingIps.push(addr);
-          }
-        }
-
-        opts.externalIps.forEach(match);
-        // opts.externalIfaces.forEach(match);
-
-        Object.keys(opts.ifaces).forEach(function (iname) {
-          var iface = opts.ifaces[iname];
-
-          iface.ipv4.forEach(match);
-          iface.ipv6.forEach(match);
-        });
-
-        return matchingIps.length;
-      });
-
-      matchingIps.externalIps = {
-        ipv4: [
-          { address: externalIp
-          , family: 'IPv4'
-          }
-        ]
-      , ipv6: [
-        ]
-      };
-      matchingIps.matchingIps = matchingIps;
-      return matchingIps;
-    });
-  });
-};

lib/mdns.js

@@ -0,0 +1,203 @@
+'use strict';
+
+var PromiseA = require('bluebird');
+var queryName = '_cloud._tcp.local';
+var dnsSuite = require('dns-suite');
+
+function createResponse(name, ownerIds, packet, ttl, mainPort) {
+  var rpacket = {
+    header: {
+      id: packet.header.id
+    , qr: 1
+    , opcode: 0
+    , aa: 1
+    , tc: 0
+    , rd: 0
+    , ra: 0
+    , res1:  0
+    , res2:  0
+    , res3:  0
+    , rcode: 0
+  , }
+  , question: packet.question
+  , answer: []
+  , authority: []
+  , additional: []
+  , edns_options: []
+  };
+
+  rpacket.answer.push({
+    name: queryName
+  , typeName: 'PTR'
+  , ttl: ttl
+  , className: 'IN'
+  , data: name + '.' + queryName
+  });
+
+  var ifaces = require('./local-ip').find();
+  Object.keys(ifaces).forEach(function (iname) {
+    var iface = ifaces[iname];
+
+    iface.ipv4.forEach(function (addr) {
+      rpacket.additional.push({
+        name: name + '.local'
+      , typeName: 'A'
+      , ttl: ttl
+      , className: 'IN'
+      , address: addr.address
+      });
+    });
+
+    iface.ipv6.forEach(function (addr) {
+      rpacket.additional.push({
+        name: name + '.local'
+      , typeName: 'AAAA'
+      , ttl: ttl
+      , className: 'IN'
+      , address: addr.address
+      });
+    });
+  });
+
+  rpacket.additional.push({
+    name: name + '.' + queryName
+  , typeName: 'SRV'
+  , ttl: ttl
+  , className: 'IN'
+  , priority: 1
+  , weight: 0
+  , port: mainPort
+  , target: name + ".local"
+  });
+  rpacket.additional.push({
+    name: name + '._device-info.' + queryName
+  , typeName: 'TXT'
+  , ttl: ttl
+  , className: 'IN'
+  , data: ["model=CloudHome1,1", "dappsvers=1"]
+  });
+  ownerIds.forEach(function (id) {
+    rpacket.additional.push({
+      name: name + '._owner-id.' + queryName
+    , typeName: 'TXT'
+    , ttl: ttl
+    , className: 'IN'
+    , data: [id]
+    });
+  });
+
+  return dnsSuite.DNSPacket.write(rpacket);
+}
+
+module.exports.create = function (deps, config) {
+  var socket;
+  var nextBroadcast = -1;
+
+  function handlePacket(message, rinfo) {
+    // console.log('Received %d bytes from %s:%d', message.length, rinfo.address, rinfo.port);
+
+    var packet;
+    try {
+      packet = dnsSuite.DNSPacket.parse(message);
+    }
+    catch (er) {
+      // `dns-suite` actually errors on a lot of the packets floating around in our network,
+      // so don't bother logging any errors. (We still use `dns-suite` because unlike `dns-js`
+      // it can successfully craft the one packet we want to send.)
+      return;
+    }
+
+    // Only respond to queries.
+    if (packet.header.qr !== 0) {  return; }
+    // Only respond if they were asking for cloud devices.
+    if (packet.question.length !== 1)           { return; }
+    if (packet.question[0].name !== queryName)  { return; }
+    if (packet.question[0].typeName !== 'PTR')  { return; }
+    if (packet.question[0].className !== 'IN' ) { return; }
+
+    var proms = [
+      deps.storage.mdnsId.get()
+    , deps.storage.owners.all().then(function (owners) {
+        // The ID is the sha256 hash of the PPID, which shouldn't be reversible and therefore
+        // should be safe to expose without needing authentication.
+        return owners.map(function (owner) {
+          return owner.id;
+        });
+      })
+    ];
+
+    PromiseA.all(proms).then(function (results) {
+      var resp = createResponse(results[0], results[1], packet, config.mdns.ttl, deps.tcp.mainPort);
+      var now = Date.now();
+      if (now > nextBroadcast) {
+        socket.send(resp, config.mdns.port, config.mdns.broadcast);
+        nextBroadcast = now + config.mdns.ttl * 1000;
+      } else {
+        socket.send(resp, rinfo.port, rinfo.address);
+      }
+    });
+  }
+
+  function start() {
+    socket = require('dgram').createSocket({ type: 'udp4', reuseAddr: true });
+    socket.on('message', handlePacket);
+
+    return new Promise(function (resolve, reject) {
+      socket.once('error', reject);
+
+      socket.bind(config.mdns.port, function () {
+        var addr = this.address();
+        console.log('bound on UDP %s:%d for mDNS', addr.address, addr.port);
+
+        socket.setBroadcast(true);
+        socket.addMembership(config.mdns.broadcast);
+        // This is supposed to be a local device discovery mechanism, so we shouldn't
+        // need to hop through any gateways. This helps with security by making it
+        // much more difficult for someone to use us as part of a DDoS attack by
+        // spoofing the UDP address a request came from.
+        socket.setTTL(1);
+
+        socket.removeListener('error', reject);
+        resolve();
+      });
+    });
+  }
+  function stop() {
+    return new Promise(function (resolve, reject) {
+      socket.once('error', reject);
+
+      socket.close(function () {
+        socket.removeListener('error', reject);
+        socket = null;
+        resolve();
+      });
+    });
+  }
+
+  function updateConf() {
+    var promise;
+    if (config.mdns.disabled) {
+      if (socket) {
+        promise = stop();
+      }
+    } else {
+      if (!socket) {
+        promise = start();
+      } else if (socket.address().port !== config.mdns.port) {
+        promise = stop().then(start);
+      } else {
+        // Can't check membership, so just add the current broadcast address to make sure
+        // it's set. If it's already set it will throw an exception (at least on linux).
+        try {
+          socket.addMembership(config.mdns.broadcast);
+        } catch (e) {}
+        promise = Promise.resolve();
+      }
+    }
+  }
+  updateConf();
+
+  return {
+    updateConf
+  };
+};

lib/modules/admin.js

@@ -1,64 +0,0 @@
-module.exports.create = function (deps, conf) {
-  'use strict';
-
-  var path = require('path');
-  //var defaultServername = 'localhost.daplie.me';
-  //var defaultWebRoot = '.';
-  var assetsPath = path.join(__dirname, '..', '..', 'packages', 'assets');
-  var opts = {};
-
-  opts.global = opts.global || {};
-  opts.sites = opts.sites || [];
-  opts.sites._map = {};
-
-  // argv.sites
-
-  opts.groups = [];
-
-  // 'packages', 'assets', 'com.daplie.goldilocks'
-  opts.global = {
-    modules: [ // TODO uh-oh we've got a mixed bag of modules (various types), a true map
-      { $id: 'greenlock', email: opts.email, tos: opts.tos }
-    , { $id: 'rvpn', email: opts.email, tos: opts.tos }
-    //, { $id: 'content', content: content }
-    , { $id: 'livereload', on: opts.livereload }
-    , { $id: 'app', path: opts.expressApp }
-    ]
-  , paths: [
-      { $id: '/assets/', modules: [ { $id: 'serve', paths: [ assetsPath ] } ] }
-      // TODO figure this b out
-    , { $id: '/.well-known/', modules: [
-        { $id: 'serve', paths: [ path.join(assetsPath, 'well-known') ] }
-      ] }
-    ]
-  };
-  opts.defaults = {
-    modules: []
-  , paths: [
-      /*
-      { $id: '/', modules: [
-        { $id: 'serve', paths: [ defaultWebRoot ] }
-      , { $id: 'indexes', paths: [ defaultWebRoot ] }
-      ] }
-      */
-    ]
-  };
-  opts.sites.push({
-    // greenlock: {}
-    $id: 'localhost.alpha.daplie.me'
-  , paths: [
-      { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] }
-    , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] }
-    ]
-  });
-  opts.sites.push({
-    $id: 'localhost.daplie.invalid'
-  , paths: [
-      { $id: '/', modules: [ { $id: 'serve', paths: [ path.resolve(__dirname, '..', '..', 'admin', 'public') ] } ] }
-    , { $id: '/api/', modules: [ { $id: 'app', path: path.join(__dirname, 'admin') } ] }
-    ]
-  });
-
-  /* device, addresses, cwd, http */
-  return require('../app.js')(deps, conf, opts);
-};

lib/modules/http.js

@@ -1,217 +0,0 @@
-'use strict';
-
-module.exports.create = function (deps, conf, greenlockMiddleware) {
-  var express = require('express');
-  var app = express();
-  var adminApp = require('./admin').create(deps, conf);
-  var domainMatches = require('../match-domain').match;
-  var proxyRoutes = [];
-
-  var adminDomains = [
-    /\blocalhost\.admin\./
-  , /\blocalhost\.alpha\./
-  , /\badmin\.localhost\./
-  , /\balpha\.localhost\./
-  ];
-
-  function verifyHost(fullHost) {
-    var host = /^(.*?)(:\d+)?$/.exec(fullHost)[1];
-
-    if (host === 'localhost') {
-      return fullHost.replace(host, 'localhost.daplie.me');
-    }
-
-    // Test for IPv4 and IPv6 addresses. These patterns will match some invalid addresses,
-    // but since those still won't be valid domains that won't really be a problem.
-    if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(host) || /^\[[0-9a-fA-F:]+\]$/.test(host)) {
-      if (!conf.http.primaryDomain) {
-        (conf.http.modules || []).some(function (mod) {
-          return mod.domains.some(function (domain) {
-            if (domain[0] !== '*') {
-              conf.http.primaryDomain = domain;
-              return true;
-            }
-          });
-        });
-      }
-      return fullHost.replace(host, conf.http.primaryDomain || host);
-    }
-
-    return fullHost;
-  }
-
-  // We handle both HTTPS and HTTP traffic on the same ports, and we want to redirect
-  // any unencrypted requests to the same port they came from unless it came in on
-  // the default HTTP port, in which case there wont be a port specified in the host.
-  var redirecters = {};
-  function redirectHttps(req, res, next) {
-    if (conf.http.allowInsecure) {
-      next();
-      return;
-    }
-
-    var port = (/:(\d+)$/.exec(req.headers.host) || [])[1];
-    if (!redirecters[port]) {
-      redirecters[port] = require('redirect-https')({
-        port: port
-      , trustProxy: conf.http.trustProxy
-      });
-    }
-
-    // localhost and IP addresses cannot have real SSL certs (and don't contain any useful
-    // info for redirection either), so we direct some hosts to either localhost.daplie.me
-    // or the "primary domain" ie the first manually specified domain.
-    req.headers.host = verifyHost(req.headers.host);
-
-    redirecters[port](req, res, next);
-  }
-
-  function handleAdmin(req, res, next) {
-    var admin = adminDomains.some(function (re) {
-      return re.test(req.headers.host);
-    });
-
-    if (admin) {
-      adminApp(req, res);
-    } else {
-      next();
-    }
-  }
-
-  function respond404(req, res) {
-    res.writeHead(404);
-    res.end('Not Found');
-  }
-
-  function createProxyRoute(mod) {
-    // This is the easiest way to override the createConnections function the proxy
-    // module uses, but take note the since we don't have control over where this is
-    // called the extra options availabled will be different.
-    var agent = new require('http').Agent({});
-    agent.createConnection = deps.net.createConnection;
-
-    var proxy = require('http-proxy').createProxyServer({
-      agent: agent
-    , target: 'http://' + mod.address
-    , xfwd: true
-    , toProxy: true
-    });
-
-    // We want to override the default value for some headers with the extra information we
-    // have available to us in the opts object attached to the connection.
-    proxy.on('proxyReq', function (proxyReq, req) {
-      var conn = req.connection;
-      var opts = conn.__opts;
-      proxyReq.setHeader('X-Forwarded-For', opts.remoteAddress || conn.remoteAddress);
-    });
-
-    proxy.on('error', function (err, req, res) {
-      console.log(err);
-      res.statusCode = 502;
-      res.setHeader('Content-Type', 'text/html');
-      res.setHeader('Connection', 'close');
-      res.end(require('../proxy-err-resp').getRespBody(err, conf.debug));
-    });
-
-    return {
-      web: function (req, res, next) {
-        var hostname = req.headers.host.split(':')[0];
-        var relevant = mod.domains.some(function (pattern) {
-          return domainMatches(pattern, hostname);
-        });
-
-        if (relevant) {
-          proxy.web(req, res);
-        } else {
-          next();
-        }
-      }
-    , ws: function (req, socket, head, next) {
-        var hostname = req.headers.host.split(':')[0];
-        var relevant = mod.domains.some(function (pattern) {
-          return domainMatches(pattern, hostname);
-        });
-
-        if (relevant) {
-          proxy.ws(req, socket, head);
-        } else {
-          next();
-        }
-      }
-    };
-  }
-
-  function createStaticRoute(mod) {
-    var getStaticApp, staticApp;
-    if (/:hostname/.test(mod.root)) {
-      staticApp = {};
-      getStaticApp = function (hostname) {
-        if (!staticApp[hostname]) {
-          staticApp[hostname] = express.static(mod.root.replace(':hostname', hostname));
-        }
-        return staticApp[hostname];
-      };
-    }
-    else {
-      staticApp = express.static(mod.root);
-      getStaticApp = function () {
-        return staticApp;
-      };
-    }
-
-    return function (req, res, next) {
-      var hostname = req.headers.host.split(':')[0];
-      var relevant = mod.domains.some(function (pattern) {
-        return domainMatches(pattern, hostname);
-      });
-
-      if (relevant) {
-        getStaticApp(hostname)(req, res, next);
-      } else {
-        next();
-      }
-    };
-  }
-
-  app.use(greenlockMiddleware);
-  app.use(redirectHttps);
-  app.use(handleAdmin);
-
-  (conf.http.modules || []).forEach(function (mod) {
-    if (mod.name === 'proxy') {
-      var proxyRoute = createProxyRoute(mod);
-      proxyRoutes.push(proxyRoute);
-      app.use(proxyRoute.web);
-    }
-    else if (mod.name === 'static') {
-      app.use(createStaticRoute(mod));
-    }
-    else {
-      console.warn('unknown HTTP module', mod);
-    }
-  });
-
-  app.use(respond404);
-
-  var server = require('http').createServer(function (req, res) {
-    app(req, res)
-  });
-
-  server.on('upgrade', function (req, socket, head) {
-    if (!proxyRoutes.length) {
-      socket.end();
-    }
-
-    function proxyWs() {
-      var proxyRoute = proxyRoutes.shift();
-      if (!proxyRoute) {
-        socket.end();
-      }
-      proxyRoute.ws(req, socket, head, proxyWs);
-    }
-
-    proxyWs();
-  });
-
-  return server;
-};

lib/modules/tls.js

@@ -1,283 +0,0 @@
-'use strict';
-
-module.exports.create = function (deps, config, netHandler) {
-  var tls = require('tls');
-  var parseSni = require('sni');
-  var greenlock = require('greenlock');
-  var localhostCerts = require('localhost.daplie.me-certificates');
-  var domainMatches = require('../match-domain').match;
-
-  function extractSocketProp(socket, propName) {
-    // remoteAddress, remotePort... ugh... https://github.com/nodejs/node/issues/8854
-    return socket[propName]
-      || socket['_' + propName]
-      || socket._handle._parent.owner.stream[propName]
-      ;
-  }
-
-  function wrapSocket(socket, opts) {
-    var myDuplex = require('tunnel-packer').Stream.create(socket);
-    myDuplex.remoteFamily = opts.remoteFamily || myDuplex.remoteFamily;
-    myDuplex.remoteAddress = opts.remoteAddress || myDuplex.remoteAddress;
-    myDuplex.remotePort = opts.remotePort || myDuplex.remotePort;
-
-    socket.on('data', function (chunk) {
-      console.log('[' + Date.now() + '] tls socket data', chunk.byteLength);
-      myDuplex.push(chunk);
-    });
-    socket.on('error', function (err) {
-      console.error('[error] httpsTunnel (Admin) TODO close');
-      console.error(err);
-      myDuplex.emit('error', err);
-    });
-    socket.on('close', function () {
-      myDuplex.end();
-    });
-
-    process.nextTick(function () {
-      // this must happen after the socket is emitted to the next in the chain,
-      // but before any more data comes in via the network
-      socket.unshift(opts.firstChunk);
-    });
-
-    return myDuplex;
-  }
-
-  var le = greenlock.create({
-    // server: 'staging'
-    server: 'https://acme-v01.api.letsencrypt.org/directory'
-
-  , challenges: {
-      'http-01': require('le-challenge-fs').create({ webrootPath: '/tmp/acme-challenges', debug: config.debug })
-    , 'tls-sni-01': require('le-challenge-sni').create({ debug: config.debug })
-      // TODO dns-01
-      //, 'dns-01': require('le-challenge-ddns').create()
-    }
-
-  , store: require('le-store-certbot').create({ webrootPath: '/tmp/acme-challenges' })
-
-  , approveDomains: function (opts, certs, cb) {
-      // This is where you check your database and associated
-      // email addresses with domains and agreements and such
-
-      // The domains being approved for the first time are listed in opts.domains
-      // Certs being renewed are listed in certs.altnames
-      if (certs) {
-        // TODO make sure the same options are used for renewal as for registration?
-        opts.domains = certs.altnames;
-
-        cb(null, { options: opts, certs: certs });
-        return;
-      }
-
-      function complete(optsOverride) {
-        Object.keys(optsOverride).forEach(function (key) {
-          opts[key] = optsOverride[key];
-        });
-
-        cb(null, { options: opts, certs: certs });
-      }
-
-
-      // check config for domain name
-      if (-1 !== (config.tls.servernames || []).indexOf(opts.domain)) {
-        // TODO how to handle SANs?
-        // TODO fetch domain-specific email
-        // TODO fetch domain-specific acmeDirectory
-        // NOTE: you can also change other options such as `challengeType` and `challenge`
-        // opts.challengeType = 'http-01';
-        // opts.challenge = require('le-challenge-fs').create({}); // TODO this doesn't actually work yet
-        complete({
-          email: config.tls.email
-        , agreeTos: true
-        , server: config.tls.acmeDirectoryUrl || le.server
-        , challengeType: config.tls.challengeType || 'http-01'
-        });
-        return;
-      }
-
-      // TODO ask http module (and potentially all other modules) about what domains it can
-      // handle. We can allow any domains that other modules will handle after we terminate TLS.
-      cb(new Error('domain is not allowed'));
-      // if (!modules.http) {
-      //   modules.http = require('./modules/http.js').create(deps, config);
-      // }
-      // modules.http.checkServername(opts.domain).then(function (stuff) {
-      //   if (!stuff || !stuff.domains) {
-      //     // TODO once precheck is implemented we can just let it pass if it passes, yknow?
-      //     cb(new Error('domain is not allowed'));
-      //     return;
-      //   }
-
-      //   complete({
-      //     domain: stuff.domain || stuff.domains[0]
-      //   , domains: stuff.domains
-      //   , email: stuff.email || program.email
-      //   , server: stuff.acmeDirectoryUrl || program.acmeDirectoryUrl
-      //   , challengeType: stuff.challengeType || program.challengeType
-      //   , challenge: stuff.challenge
-      //   });
-      //   return;
-      // }, cb);
-    }
-  });
-  le.tlsOptions = le.tlsOptions || le.httpsOptions;
-
-  var secureContexts = {};
-  var terminatorOpts = require('localhost.daplie.me-certificates').merge({});
-  terminatorOpts.SNICallback = function (sni, cb) {
-    console.log("[tlsOptions.SNICallback] SNI: '" + sni + "'");
-
-    var tlsOptions;
-
-    // Static Certs
-    if (/.*localhost.*\.daplie\.me/.test(sni.toLowerCase())) {
-      // TODO implement
-      if (!secureContexts[sni]) {
-        tlsOptions = localhostCerts.mergeTlsOptions(sni, {});
-      }
-      if (tlsOptions) {
-        secureContexts[sni] = tls.createSecureContext(tlsOptions);
-      }
-      if (secureContexts[sni]) {
-        console.log('Got static secure context:', sni, secureContexts[sni]);
-        cb(null, secureContexts[sni]);
-        return;
-      }
-    }
-
-    le.tlsOptions.SNICallback(sni, cb);
-  };
-
-  var terminateServer = tls.createServer(terminatorOpts, function (socket) {
-    console.log('(pre-terminated) tls connection, addr:', socket.remoteAddress);
-
-    netHandler(socket, {
-      servername: socket.servername
-    , encrypted: true
-      // remoteAddress... ugh... https://github.com/nodejs/node/issues/8854
-    , remoteAddress: extractSocketProp(socket, 'remoteAddress')
-    , remotePort:    extractSocketProp(socket, 'remotePort')
-    , remoteFamily:  extractSocketProp(socket, 'remoteFamily')
-    });
-  });
-
-  function proxy(socket, opts, mod) {
-    var destination = mod.address.split(':');
-    var connected = false;
-
-    var newConn = deps.net.createConnection({
-        port: destination[1]
-      , host: destination[0] || '127.0.0.1'
-
-      , servername: opts.servername
-      , data: opts.firstChunk
-      , remoteFamily:  opts.family  || extractSocketProp(socket, 'remoteFamily')
-      , remoteAddress: opts.address || extractSocketProp(socket, 'remoteAddress')
-      , remotePort:    opts.port    || extractSocketProp(socket, 'remotePort')
-    }, function () {
-      connected = true;
-      if (!opts.hyperPeek) {
-        newConn.write(opts.firstChunk);
-      }
-      newConn.pipe(socket);
-      socket.pipe(newConn);
-    });
-
-    // Not sure how to effectively report this to the user or client, but we need to listen
-    // for the event to prevent it from crashing us.
-    newConn.on('error', function (err) {
-      if (connected) {
-        console.error('TLS proxy remote error', err);
-        socket.end();
-      } else {
-        console.log('TLS proxy connection error', err);
-        var tlsOpts = localhostCerts.mergeTlsOptions('localhost.daplie.me', {isServer: true});
-        var decrypted;
-        if (opts.hyperPeek) {
-          decrypted = new tls.TLSSocket(socket, tlsOpts);
-        } else {
-          decrypted = new tls.TLSSocket(wrapSocket(socket, opts), tlsOpts);
-        }
-        require('../proxy-err-resp').sendBadGateway(decrypted, err, config.debug);
-      }
-    });
-    socket.on('error', function (err) {
-      console.error('TLS proxy client error', err);
-      newConn.end();
-    });
-  }
-
-  function terminate(socket, opts) {
-    console.log(
-      '[tls-terminate]'
-    , opts.localAddress || socket.localAddress +':'+ opts.localPort || socket.localPort
-    , 'servername=' + opts.servername
-    , opts.remoteAddress || socket.remoteAddress
-    );
-
-    if (opts.hyperPeek) {
-      // This connection was peeked at using a method that doesn't interferre with the TLS
-      // server's ability to handle it properly. Currently the only way this happens is
-      // with tunnel connections where we have the first chunk of data before creating the
-      // new connection (thus removing need to get data off the new connection).
-      terminateServer.emit('connection', socket);
-    }
-    else {
-      // The hyperPeek flag wasn't set, so we had to read data off of this connection, which
-      // means we can no longer use it directly in the TLS server.
-      // See https://github.com/nodejs/node/issues/8752 (node's internal networking layer == 💩 sometimes)
-      terminateServer.emit('connection', wrapSocket(socket, opts));
-    }
-  }
-
-  function handleConn(socket, opts) {
-    opts.servername = (parseSni(opts.firstChunk)||'').toLowerCase() || 'localhost.invalid';
-    // needs to wind up in one of 2 states:
-    // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket)
-    // 2. Terminated (goes on to a particular module or route, including the admin interface)
-    // 3. Closed (we don't recognize the SNI servername as something we actually want to handle)
-
-    // We always want to terminate is the SNI matches the challenge pattern, unless a client
-    // on the south side has temporarily claimed a particular challenge. For the time being
-    // we don't have a way for the south-side to communicate with us, so that part isn't done.
-    if (domainMatches('*.acme-challenge.invalid', opts.servername)) {
-      terminate(socket, opts);
-      return;
-    }
-
-    var handled = (config.tls.modules || []).some(function (mod) {
-      var relevant = mod.domains.some(function (pattern) {
-        return domainMatches(pattern, opts.servername);
-      });
-      if (!relevant) {
-        return false;
-      }
-
-      if (mod.name === 'proxy') {
-        proxy(socket, opts, mod);
-      }
-      else {
-        console.error('saw unknown TLS module', mod);
-        return false;
-      }
-
-      return true;
-    });
-
-    // TODO: figure out all of the domains that the other modules intend to handle, and only
-    // terminate those ones, closing connections for all others.
-    if (!handled) {
-      terminate(socket, opts);
-    }
-  }
-
-  return {
-    emit: function (type, socket) {
-      if (type === 'connection') {
-        handleConn(socket, socket.__opts);
-      }
-    }
-  , middleware: le.middleware()
-  };
-};

lib/proxy-err-resp.js

@@ -1,32 +0,0 @@
-'use strict';
-
-function getRespBody(err, debug) {
-  if (debug) {
-    return err.toString();
-  }
-
-  if (err.code === 'ECONNREFUSED') {
-    return 'The connection was refused. Most likely the service being connected to '
-      + 'has stopped running or the configuration is wrong.';
-  }
-
-  return 'Bad Gateway: ' + err.code;
-}
-
-function sendBadGateway(conn, err, debug) {
-  var body = getRespBody(err, debug);
-
-  conn.write([
-    'HTTP/1.1 502 Bad Gateway'
-  , 'Date: ' + (new Date()).toUTCString()
-  , 'Connection: close'
-  , 'Content-Type: text/html'
-  , 'Content-Length: ' + body.length
-  , ''
-  , body
-  ].join('\r\n'));
-  conn.end();
-}
-
-module.exports.getRespBody = getRespBody;
-module.exports.sendBadGateway = sendBadGateway;

lib/servers.js

@@ -10,31 +10,27 @@ module.exports.addTcpListener = function (port, handler) {
 
     if (stat) {
       if (stat._closing) {
-        module.exports.destroyTcpListener(port);
-      }
-      else if (handler !== stat.handler) {
-
-        // we'll replace the current listener
+        stat.server.destroy();
+      } else {
+        // We're already listening on the port, so we only have 2 options. We can either
+        // replace the handler or reject with an error. (Though neither is really needed
+        // if the handlers are the same). Until there is reason to do otherwise we are
+        // opting for the replacement.
         stat.handler = handler;
         resolve();
         return;
       }
-      else {
-        // this exact listener is already open
-        resolve();
-        return;
-      }
     }
 
     var enableDestroy = require('server-destroy');
     var net = require('net');
     var resolved;
-    var server = net.createServer();
+    var server = net.createServer({allowHalfOpen: true});
 
     stat = serversMap[port] = {
       server: server
     , handler: handler
-    , _closing: null
+    , _closing: false
     };
 
     // Add .destroy so we can close all open connections. Better if added before listen
@@ -46,20 +42,18 @@ module.exports.addTcpListener = function (port, handler) {
       conn.__proto = 'tcp';
       stat.handler(conn);
     });
-    server.on('error', function (e) {
+    server.on('close', function () {
+      console.log('TCP server on port %d closed', port);
       delete serversMap[port];
-
+    });
+    server.on('error', function (e) {
       if (!resolved) {
         reject(e);
-        return;
-      }
-
-      if (handler.onError) {
+      } else if (handler.onError) {
         handler.onError(e);
-        return;
+      } else {
+        throw e;
       }
-
-      throw e;
     });
 
     server.listen(port, function () {
@@ -68,36 +62,39 @@ module.exports.addTcpListener = function (port, handler) {
     });
   });
 };
-module.exports.closeTcpListener = function (port) {
+module.exports.closeTcpListener = function (port, timeout) {
   return new PromiseA(function (resolve) {
     var stat = serversMap[port];
     if (!stat) {
       resolve();
       return;
     }
-    stat.server.on('close', function () {
-      // once the clients close too
-      delete serversMap[port];
-      if (stat._closing) {
-        stat._closing(); // resolve
-        stat._closing = null;
-      }
-      stat = null;
+    stat._closing = true;
+
+    var timeoutId;
+    if (timeout) {
+      timeoutId = setTimeout(() => stat.server.destroy(), timeout);
+    }
+
+    stat.server.once('close', function () {
+      clearTimeout(timeoutId);
+      resolve();
     });
-    stat._closing = resolve;
     stat.server.close();
   });
 };
 module.exports.destroyTcpListener = function (port) {
   var stat = serversMap[port];
-  delete serversMap[port];
-  stat.server.destroy();
-  if (stat._closing) {
-    stat._closing();
-    stat._closing = null;
+  if (stat) {
+    stat.server.destroy();
   }
-  stat = null;
 };
+module.exports.listTcpListeners = function () {
+  return Object.keys(serversMap).map(Number).filter(function (port) {
+    return port && !serversMap[port]._closing;
+  });
+};
+
 
 module.exports.addUdpListener = function (port, handler) {
   return new PromiseA(function (resolve, reject) {
@@ -111,7 +108,7 @@ module.exports.addUdpListener = function (port, handler) {
     }
 
     var dgram = require('dgram');
-    var server = dgram.createSocket('udp4');
+    var server = dgram.createSocket({type: 'udp4', reuseAddr: true});
     var resolved = false;
 
     stat = dgramMap[port] = {
@@ -162,6 +159,9 @@ module.exports.closeUdpListener = function (port) {
     stat.server.close();
   });
 };
+module.exports.listUdpListeners = function () {
+  return Object.keys(dgramMap).map(Number).filter(Boolean);
+};
 
 
 module.exports.listeners = {
@@ -169,9 +169,11 @@ module.exports.listeners = {
     add: module.exports.addTcpListener
   , close: module.exports.closeTcpListener
   , destroy: module.exports.destroyTcpListener
+  , list: module.exports.listTcpListeners
   }
 , udp: {
     add: module.exports.addUdpListener
   , close: module.exports.closeUdpListener
+  , list: module.exports.listUdpListeners
   }
 };

lib/socks5-server.js

@@ -0,0 +1,91 @@
+'use strict';
+
+module.exports.create = function (deps, config) {
+  var PromiseA = require('bluebird');
+  var server;
+
+  function curState() {
+    var addr = server && server.address();
+    if (!addr) {
+      return PromiseA.resolve({running: false});
+    }
+    return PromiseA.resolve({
+      running: true
+    , port: addr.port
+    });
+  }
+
+  function start(port) {
+    if (server) {
+      return curState();
+    }
+
+    server = require('socksv5').createServer(function (info, accept) {
+      accept();
+    });
+
+    // It would be nice if we could use `server-destroy` here, but we can't because
+    // the socksv5 library will not give us access to any sockets it actually
+    // handles, so we have no way of keeping track of them or closing them.
+    server.on('close', function () {
+      server = null;
+    });
+
+    server.useAuth(require('socksv5').auth.None());
+
+    return new PromiseA(function (resolve, reject) {
+      server.on('error', function (err) {
+        if (!port && err.code === 'EADDRINUSE') {
+          server.listen(0);
+        } else {
+          server = null;
+          reject(err);
+        }
+      });
+      server.listen(port || 1080, function () {
+        resolve(curState());
+      });
+    });
+  }
+
+  function stop() {
+    if (!server) {
+      return curState();
+    }
+    return new PromiseA(function (resolve, reject) {
+      server.close(function (err) {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(curState());
+        }
+      });
+    });
+  }
+
+  var configEnabled = false;
+  function updateConf() {
+    var wanted = config.socks5 && config.socks5.enabled;
+
+    if (configEnabled && !wanted) {
+      stop().catch(function (err) {
+        console.error('failed to stop socks5 proxy on config change', err);
+      });
+      configEnabled = false;
+    }
+    if (wanted && !configEnabled) {
+      start(config.socks5.port).catch(function (err) {
+        console.error('failed to start Socks5 proxy', err);
+      });
+      configEnabled = true;
+    }
+  }
+  process.nextTick(updateConf);
+
+  return {
+    curState
+  , start
+  , stop
+  , updateConf
+  };
+};

lib/storage.js

@@ -0,0 +1,225 @@
+'use strict';
+
+var PromiseA = require('bluebird');
+var path = require('path');
+var fs = PromiseA.promisifyAll(require('fs'));
+var jwt = require('jsonwebtoken');
+var crypto = require('crypto');
+
+module.exports.create = function (deps, conf) {
+  var hrIds = require('human-readable-ids').humanReadableIds;
+  var scmp = require('scmp');
+  var storageDir = path.join(__dirname, '..', 'var');
+
+  function read(fileName) {
+    return fs.readFileAsync(path.join(storageDir, fileName))
+    .then(JSON.parse, function (err) {
+      if (err.code === 'ENOENT') {
+        return {};
+      }
+      throw err;
+    });
+  }
+  function write(fileName, obj) {
+    return fs.mkdirAsync(storageDir).catch(function (err) {
+      if (err.code !== 'EEXIST') {
+        console.error('failed to mkdir', storageDir, err.toString());
+      }
+    }).then(function () {
+      return fs.writeFileAsync(path.join(storageDir, fileName), JSON.stringify(obj), 'utf8');
+    });
+  }
+
+  var owners = {
+    _filename: 'owners.json'
+  , all: function () {
+      return read(this._filename).then(function (owners) {
+        return Object.keys(owners).map(function (id) {
+          var owner = owners[id];
+          owner.id = id;
+          return owner;
+        });
+      });
+    }
+  , get: function (id) {
+      // While we could directly read the owners file and access the id directly from
+      // the resulting object I'm not sure of the details of how the object key lookup
+      // works or whether that would expose us to timing attacks.
+      // See https://codahale.com/a-lesson-in-timing-attacks/
+      return this.all().then(function (owners) {
+        return owners.filter(function (owner) {
+          return scmp(id, owner.id);
+        })[0];
+      });
+    }
+  , exists: function (id) {
+      return this.get(id).then(function (owner) {
+        return !!owner;
+      });
+    }
+  , set: function (id, obj) {
+      var self = this;
+      return read(self._filename).then(function (owners) {
+        obj.id = id;
+        owners[id] = obj;
+        return write(self._filename, owners);
+      });
+    }
+  };
+
+  var confCb;
+  var config = {
+    save: function (changes) {
+      deps.messenger.send({
+        type: 'com.daplie.goldilocks/config'
+      , changes: changes
+      });
+
+      return new deps.PromiseA(function (resolve, reject) {
+        var timeoutId = setTimeout(function () {
+          reject(new Error('Did not receive config update from main process in a reasonable time'));
+          confCb = null;
+        }, 15*1000);
+
+        confCb = function (config) {
+          confCb = null;
+          clearTimeout(timeoutId);
+          resolve(config);
+        };
+      });
+    }
+  };
+  function updateConf(config) {
+    if (confCb) {
+      confCb(config);
+    }
+  }
+
+  var userTokens = {
+    _filename: 'user-tokens.json'
+  , _cache: {}
+  , _convertToken: function convertToken(id, token) {
+      // convert the token into something that looks more like what OAuth3 uses internally
+      // as sessions so we can use it with OAuth3. We don't use OAuth3's internal session
+      // storage because it effectively only supports storing tokens based on provider URI.
+      // We also use the token as the `access_token` instead of `refresh_token` because the
+      // refresh functionality is closely tied to the storage.
+      var decoded = jwt.decode(token);
+      if (!decoded) {
+        return null;
+      }
+      return {
+        id:           id
+      , access_token: token
+      , token:        decoded
+      , provider_uri: decoded.iss || decoded.issuer || decoded.provider_uri
+      , client_uri:   decoded.azp
+      , scope:        decoded.scp || decoded.scope || decoded.grants
+      };
+    }
+  , all: function allUserTokens() {
+      var self = this;
+      if (self._cacheComplete) {
+        return deps.PromiseA.resolve(Object.values(self._cache));
+      }
+
+      return read(self._filename).then(function (tokens) {
+        // We will read every single token into our cache, so it will be complete once we finish
+        // creating the result (it's set out of order so we can directly return the result).
+        self._cacheComplete = true;
+
+        return Object.keys(tokens).map(function (id) {
+          self._cache[id] = self._convertToken(id, tokens[id]);
+          return self._cache[id];
+        });
+      });
+    }
+  , get: function getUserToken(id) {
+      var self = this;
+      if (self._cache.hasOwnProperty(id) || self._cacheComplete) {
+        return deps.PromiseA.resolve(self._cache[id] || null);
+      }
+
+      return read(self._filename).then(function (tokens) {
+        self._cache[id] = self._convertToken(id, tokens[id]);
+        return self._cache[id];
+      });
+    }
+  , save: function saveUserToken(newToken) {
+      var self = this;
+      return read(self._filename).then(function (tokens) {
+        var rawToken;
+        if (typeof newToken === 'string') {
+          rawToken = newToken;
+        } else {
+          rawToken = newToken.refresh_token || newToken.access_token;
+        }
+        if (typeof rawToken !== 'string') {
+          throw new Error('cannot save invalid session: missing refresh_token and access_token');
+        }
+
+        var decoded = jwt.decode(rawToken);
+        var idHash = crypto.createHash('sha256');
+        idHash.update(decoded.sub || decoded.ppid || decoded.appScopedId || '');
+        idHash.update(decoded.iss || decoded.issuer || '');
+        idHash.update(decoded.aud || decoded.audience || '');
+
+        var scope = decoded.scope || decoded.scp || decoded.grants || '';
+        idHash.update(scope.split(/[,\s]+/mg).sort().join(','));
+
+        var id = idHash.digest('hex');
+        tokens[id] = rawToken;
+        return write(self._filename, tokens).then(function () {
+          // Delete the current cache so that if this is an update it will refresh
+          // the cache once we read the ID.
+          delete self._cache[id];
+          return self.get(id);
+        });
+      });
+    }
+  , remove: function removeUserToken(id) {
+      var self = this;
+      return read(self._filename).then(function (tokens) {
+        var present = delete tokens[id];
+        if (!present) {
+          return present;
+        }
+
+        return write(self._filename, tokens).then(function () {
+          delete self._cache[id];
+          return true;
+        });
+      });
+    }
+  };
+
+  var mdnsId = {
+    _filename: 'mdns-id'
+  , get: function () {
+      var self = this;
+      return read("mdns-id").then(function (result) {
+        if (typeof result !== 'string') {
+          throw new Error('mDNS ID not present');
+        }
+        return result;
+      }).catch(function () {
+        return self.set(hrIds.random());
+      });
+    }
+
+  , set: function (value) {
+      var self = this;
+      return write(self._filename, value).then(function () {
+        return self.get();
+      });
+    }
+  };
+
+  return {
+    owners: owners
+  , config: config
+  , updateConf: updateConf
+  , tokens: userTokens
+  , mdnsId: mdnsId
+  };
+};

lib/tcp/http.js

@@ -0,0 +1,543 @@
+'use strict';
+
+module.exports.create = function (deps, conf, tcpMods) {
+  var PromiseA = require('bluebird');
+  var statAsync = PromiseA.promisify(require('fs').stat);
+  var domainMatches = require('../domain-utils').match;
+  var separatePort = require('../domain-utils').separatePort;
+
+  function parseHeaders(conn, opts) {
+    // There should already be a `firstChunk` on the opts, but because we might sometimes
+    // need more than that to get all the headers it's easier to always read the data off
+    // the connection and put it back later if we need to.
+    opts.firstChunk = Buffer.alloc(0);
+
+    // First we make sure we have all of the headers.
+    return new PromiseA(function (resolve, reject) {
+      if (opts.firstChunk.includes('\r\n\r\n')) {
+        resolve(opts.firstChunk.toString());
+        return;
+      }
+
+      var errored = false;
+      function handleErr(err) {
+        errored = true;
+        reject(err);
+      }
+      conn.once('error', handleErr);
+
+      function handleChunk(chunk) {
+        if (!errored) {
+          opts.firstChunk = Buffer.concat([opts.firstChunk, chunk]);
+          if (!opts.firstChunk.includes('\r\n\r\n')) {
+            conn.once('data', handleChunk);
+            return;
+          }
+
+          conn.removeListener('error', handleErr);
+          conn.pause();
+          resolve(opts.firstChunk.toString());
+        }
+      }
+      conn.once('data', handleChunk);
+    }).then(function (firstStr) {
+      var headerSection = firstStr.split('\r\n\r\n')[0];
+      var lines = headerSection.split('\r\n');
+      var result = {};
+
+      lines.slice(1).forEach(function (line) {
+        var match = /([^:]*?)\s*:\s*(.*)/.exec(line);
+        if (match) {
+          result[match[1].toLowerCase()] = match[2];
+        } else {
+          console.error('HTTP header line does not match pattern', line);
+        }
+      });
+
+      var match = /^([a-zA-Z]+)\s+(\S+)\s+HTTP/.exec(lines[0]);
+      if (!match) {
+        throw new Error('first line of "HTTP" does not match pattern: '+lines[0]);
+      }
+      result.method = match[1].toUpperCase();
+      result.url = match[2];
+
+      return result;
+    });
+  }
+
+  function hostMatchesDomains(req, domainList) {
+    var host = separatePort((req.headers || req).host).host.toLowerCase();
+
+    return domainList.some(function (pattern) {
+      return domainMatches(pattern, host);
+    });
+  }
+
+  function determinePrimaryHost() {
+    var result;
+    if (Array.isArray(conf.domains)) {
+      conf.domains.some(function (dom) {
+        if (!dom.modules || !dom.modules.http) {
+          return false;
+        }
+        return dom.names.some(function (domain) {
+          if (domain[0] !== '*') {
+            result = domain;
+            return true;
+          }
+        });
+      });
+    }
+    if (result) {
+      return result;
+    }
+
+    if (Array.isArray(conf.http.modules)) {
+      conf.http.modules.some(function (mod) {
+        return mod.domains.some(function (domain) {
+          if (domain[0] !== '*') {
+            result = domain;
+            return true;
+          }
+        });
+      });
+    }
+    return result;
+  }
+
+  // We handle both HTTPS and HTTP traffic on the same ports, and we want to redirect
+  // any unencrypted requests to the same port they came from unless it came in on
+  // the default HTTP port, in which case there wont be a port specified in the host.
+  var redirecters = {};
+  var ipv4Re = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
+  var ipv6Re = /^\[[0-9a-fA-F:]+\]$/;
+  function redirectHttps(req, res) {
+    var host = separatePort(req.headers.host);
+
+    if (!redirecters[host.port]) {
+      redirecters[host.port] = require('redirect-https')({ port: host.port });
+    }
+
+    // localhost and IP addresses cannot have real SSL certs (and don't contain any useful
+    // info for redirection either), so we direct some hosts to either localhost.daplie.me
+    // or the "primary domain" ie the first manually specified domain.
+    if (host.host === 'localhost') {
+      req.headers.host = 'localhost.daplie.me' + (host.port ? ':'+host.port : '');
+    }
+    // Test for IPv4 and IPv6 addresses. These patterns will match some invalid addresses,
+    // but since those still won't be valid domains that won't really be a problem.
+    if (ipv4Re.test(host.host) || ipv6Re.test(host.host)) {
+      var dest;
+      if (conf.http.primaryDomain) {
+        dest = conf.http.primaryDomain;
+      } else {
+        dest = determinePrimaryHost();
+      }
+      if (dest) {
+        req.headers.host = dest + (host.port ? ':'+host.port : '');
+      }
+    }
+
+    redirecters[host.port](req, res);
+  }
+
+  function emitConnection(server, conn, opts) {
+    server.emit('connection', conn);
+
+    // We need to put back whatever data we read off to determine the connection was HTTP
+    // and to parse the headers. Must be done after data handlers added but before any new
+    // data comes in.
+    process.nextTick(function () {
+      conn.unshift(opts.firstChunk);
+      conn.resume();
+    });
+
+    // Convenience return for all the check* functions.
+    return true;
+  }
+
+  var acmeServer;
+  function checkAcme(conn, opts, headers) {
+    if (headers.url.indexOf('/.well-known/acme-challenge/') !== 0) {
+      return false;
+    }
+
+    if (deps.stunneld.isClientDomain(separatePort(headers.host).host)) {
+      deps.stunneld.handleClientConn(conn);
+      process.nextTick(function () {
+        conn.unshift(opts.firstChunk);
+        conn.resume();
+      });
+      return true;
+    }
+
+    if (!acmeServer) {
+      acmeServer = require('http').createServer(tcpMods.tls.middleware);
+    }
+    return emitConnection(acmeServer, conn, opts);
+  }
+
+  function checkLoopback(conn, opts, headers) {
+    if (headers.url.indexOf('/.well-known/cloud-challenge/') !== 0) {
+      return false;
+    }
+    return emitConnection(deps.ddns.loopbackServer, conn, opts);
+  }
+
+  var httpsRedirectServer;
+  function checkHttps(conn, opts, headers) {
+    if (conf.http.allowInsecure || conn.encrypted) {
+      return false;
+    }
+    if (conf.http.trustProxy && 'https' === headers['x-forwarded-proto']) {
+      return false;
+    }
+
+    if (!httpsRedirectServer) {
+      httpsRedirectServer = require('http').createServer(redirectHttps);
+    }
+    return emitConnection(httpsRedirectServer, conn, opts);
+  }
+
+  var adminDomains;
+  var adminServer;
+  function checkAdmin(conn, opts, headers) {
+    var host = separatePort(headers.host).host;
+
+    if (!adminDomains) {
+      adminDomains = require('../admin').adminDomains;
+    }
+    if (adminDomains.indexOf(host) !== -1) {
+      if (!adminServer) {
+        adminServer = require('../admin').create(deps, conf);
+      }
+      return emitConnection(adminServer, conn, opts);
+    }
+
+    if (deps.stunneld.isAdminDomain(host)) {
+      deps.stunneld.handleAdminConn(conn);
+      process.nextTick(function () {
+        conn.unshift(opts.firstChunk);
+        conn.resume();
+      });
+      return true;
+    }
+
+    return false;
+  }
+
+  var proxyServer;
+  function createProxyServer() {
+    var http = require('http');
+    var agent = new http.Agent();
+    agent.createConnection = deps.net.createConnection;
+
+    var proxy = require('http-proxy').createProxyServer({
+      agent: agent
+    , toProxy: true
+    });
+
+    proxy.on('error', function (err, req, res) {
+      res.statusCode = 502;
+      res.setHeader('Connection', 'close');
+      res.setHeader('Content-Type', 'text/html');
+      res.end(tcpMods.proxy.getRespBody(err, conf.debug));
+    });
+
+    proxyServer = http.createServer(function (req, res) {
+      proxy.web(req, res, req.connection.proxyOpts);
+    });
+    proxyServer.on('upgrade', function (req, socket, head) {
+      proxy.ws(req, socket, head, socket.proxyOpts);
+    });
+  }
+  function proxyRequest(mod, conn, opts, xHeaders) {
+    if (!proxyServer) {
+      createProxyServer();
+    }
+
+    conn.proxyOpts = {
+      target: 'http://'+(mod.address || (mod.host || 'localhost')+':'+mod.port)
+    , headers: xHeaders
+    };
+    return emitConnection(proxyServer, conn, opts);
+  }
+
+  function proxyWebsocket(mod, conn, opts, headers, xHeaders) {
+    var index = opts.firstChunk.indexOf('\r\n\r\n');
+    var body = opts.firstChunk.slice(index);
+
+    var head = opts.firstChunk.slice(0, index).toString();
+    var headLines = head.split('\r\n');
+    // First strip any existing `X-Forwarded-*` headers (for security purposes?)
+    headLines = headLines.filter(function (line) {
+      return !/^x-forwarded/i.test(line);
+    });
+    // Then add our own `X-Forwarded` headers at the end.
+    Object.keys(xHeaders).forEach(function (key) {
+      headLines.push(key + ': ' +xHeaders[key]);
+    });
+    // Then convert all of the head lines back into a header buffer.
+    head = Buffer.from(headLines.join('\r\n'));
+
+    opts.firstChunk = Buffer.concat([head, body]);
+
+    var newConnOpts = separatePort(mod.address || '');
+    newConnOpts.port = newConnOpts.port || mod.port;
+    newConnOpts.host = newConnOpts.host || mod.host || 'localhost';
+    newConnOpts.servername = separatePort(headers.host).host;
+    newConnOpts.data = opts.firstChunk;
+
+    newConnOpts.remoteFamily  = opts.family  || conn.remoteFamily;
+    newConnOpts.remoteAddress = opts.address || conn.remoteAddress;
+    newConnOpts.remotePort    = opts.port    || conn.remotePort;
+
+    tcpMods.proxy(conn, newConnOpts, opts.firstChunk);
+  }
+
+  function checkProxy(mod, conn, opts, headers) {
+    var xHeaders = {};
+    // Then add our own `X-Forwarded` headers at the end.
+    if (conf.http.trustProxy && headers['x-forwarded-proto']) {
+      xHeaders['X-Forwarded-Proto'] = headers['x-forwarded-proto'];
+    } else {
+      xHeaders['X-Forwarded-Proto'] = conn.encrypted ? 'https' : 'http';
+    }
+    var proxyChain = (headers['x-forwarded-for'] || '').split(/ *, */).filter(Boolean);
+    proxyChain.push(opts.remoteAddress || opts.address || conn.remoteAddress);
+    xHeaders['X-Forwarded-For'] = proxyChain.join(', ');
+    xHeaders['X-Forwarded-Host'] = headers.host;
+
+    if ((headers.connection || '').toLowerCase() === 'upgrade') {
+      proxyWebsocket(mod, conn, opts, headers, xHeaders);
+    } else {
+      proxyRequest(mod, conn, opts, xHeaders);
+    }
+    return true;
+  }
+
+  function checkRedirect(mod, conn, opts, headers) {
+    if (!mod.fromRe || mod.fromRe.origSrc !== mod.from) {
+      // Escape any characters that (can) have special meaning in regular expression
+      // but that aren't the special characters we have interest in.
+      var from = mod.from.replace(/[-\/\\^$+?.()|[\]{}]/g, '\\$&');
+      // Then modify the characters we are interested in so they do what we want in
+      // the regular expression after being compiled.
+      from = from.replace(/\*/g, '(.*)');
+      var fromRe = new RegExp('^' + from + '/?$');
+      fromRe.origSrc = mod.from;
+      // We don't want this property showing up in the actual config file or the API,
+      // so we define it this way so it's not enumberable.
+      Object.defineProperty(mod, 'fromRe', {value: fromRe, configurable: true});
+    }
+
+    var match = mod.fromRe.exec(headers.url);
+    if (!match) {
+      return false;
+    }
+
+    var to = mod.to;
+    match.slice(1).forEach(function (globMatch, index) {
+      to = to.replace(':'+(index+1), globMatch);
+    });
+    var status = mod.status || 301;
+    var code = require('http').STATUS_CODES[status] || 'Unknown';
+
+    conn.end([
+      'HTTP/1.1 ' + status + ' ' + code
+    , 'Date: ' + (new Date()).toUTCString()
+    , 'Location: ' + to
+    , 'Connection: close'
+    , 'Content-Length: 0'
+    , ''
+    , ''
+    ].join('\r\n'));
+    return true;
+  }
+
+  var staticServer;
+  var staticHandlers = {};
+  var indexHandlers = {};
+  function serveStatic(req, res) {
+    var rootDir = req.connection.rootDir;
+    var modOpts = req.connection.modOpts;
+
+    if (!staticHandlers[rootDir]) {
+      staticHandlers[rootDir] = require('express').static(rootDir, {
+        dotfiles: modOpts.dotfiles
+      , fallthrough: false
+      , redirect: modOpts.redirect
+      , index: modOpts.index
+      });
+    }
+
+    staticHandlers[rootDir](req, res, function (err) {
+      function doFinal() {
+        if (err) {
+          res.statusCode = err.statusCode;
+        } else {
+          res.statusCode = 404;
+        }
+        res.setHeader('Content-Type', 'text/html');
+
+        if (res.statusCode === 404) {
+          res.end('File Not Found');
+        } else {
+          res.end(require('http').STATUS_CODES[res.statusCode]);
+        }
+      }
+
+      var handlerHandle = rootDir
+        + (modOpts.hidden||'')
+        + (modOpts.icons||'')
+        + (modOpts.stylesheet||'')
+        + (modOpts.template||'')
+        + (modOpts.view||'')
+        ;
+
+      function pathMatchesUrl(pathname) {
+        if (req.url === pathname) {
+          return true;
+        }
+        if (0 === req.url.replace(/\/?$/, '/').indexOf(pathname.replace(/\/?$/, '/'))) {
+          return true;
+        }
+      }
+      if (!modOpts.indexes || ('*' !== modOpts.indexes[0] && !modOpts.indexes.some(pathMatchesUrl))) {
+        doFinal();
+        return;
+      }
+
+      if (!indexHandlers[handlerHandle]) {
+        // https://www.npmjs.com/package/serve-index
+        indexHandlers[handlerHandle] = require('serve-index')(rootDir, {
+          hidden: modOpts.hidden
+        , icons: modOpts.icons
+        , stylesheet: modOpts.stylesheet
+        , template: modOpts.template
+        , view: modOpts.view
+        });
+      }
+
+      indexHandlers[handlerHandle](req, res, function (_err) {
+        err = _err || err;
+
+        doFinal();
+      });
+
+    });
+  }
+  function checkStatic(modOpts, conn, opts, headers) {
+    var rootDir = modOpts.root.replace(':hostname', separatePort(headers.host).host);
+    return statAsync(rootDir)
+      .then(function (stats) {
+        if (!stats || !stats.isDirectory()) {
+          return false;
+        }
+
+        if (!staticServer) {
+          staticServer = require('http').createServer(serveStatic);
+        }
+        conn.rootDir = rootDir;
+        conn.modOpts = modOpts;
+        return emitConnection(staticServer, conn, opts);
+      })
+      .catch(function (err) {
+        if (err.code !== 'ENOENT') {
+          console.warn('errored stating', rootDir, 'for serving static files', err);
+        }
+        return false;
+      })
+      ;
+  }
+
+  // The function signature is as follows
+  // function module(moduleOptions, tcpConnection, connectionOptions, headers) { ... }
+  var moduleChecks = {
+    proxy:    checkProxy
+  , redirect: checkRedirect
+  , static:   checkStatic
+  };
+
+  function handleConnection(conn) {
+    var opts = conn.__opts;
+    parseHeaders(conn, opts)
+      .then(function (headers) {
+        if (checkAcme(conn, opts, headers))  { return; }
+        if (checkLoopback(conn, opts, headers))  { return; }
+        if (checkHttps(conn, opts, headers)) { return; }
+        if (checkAdmin(conn, opts, headers)) { return; }
+
+        var prom = PromiseA.resolve(false);
+        (conf.domains || []).forEach(function (dom) {
+          prom = prom.then(function (handled) {
+            if (handled) {
+              return handled;
+            }
+            if (!dom.modules || !dom.modules.http) {
+              return false;
+            }
+            if (!hostMatchesDomains(headers, dom.names)) {
+              return false;
+            }
+
+            var subProm = PromiseA.resolve(false);
+            dom.modules.http.forEach(function (mod) {
+              if (moduleChecks[mod.type]) {
+                subProm = subProm.then(function (handled) {
+                  if (handled) { return handled; }
+                  return moduleChecks[mod.type](mod, conn, opts, headers);
+                });
+              } else {
+                console.warn('unknown HTTP module under domains', dom.names.join(','), mod);
+              }
+            });
+            return subProm;
+          });
+        });
+        (conf.http.modules || []).forEach(function (mod) {
+          prom = prom.then(function (handled) {
+            if (handled) {
+              return handled;
+            }
+            if (!hostMatchesDomains(headers, mod.domains)) {
+              return false;
+            }
+
+            if (moduleChecks[mod.type]) {
+              return moduleChecks[mod.type](mod, conn, opts, headers);
+            }
+            console.warn('unknown HTTP module found', mod);
+          });
+        });
+
+
+        prom.then(function (handled) {
+          // XXX TODO SECURITY html escape
+          var host = (headers.host || '[no host header]').replace(/</, '&lt;');
+          // TODO specify filepath of config file or database connection, etc
+          var msg = "Bad Gateway: Goldilocks accepted '" + host + "' but no module (neither static nor proxy) was designated to handle it. Check your config file.";
+          if (!handled) {
+            conn.end([
+              'HTTP/1.1 502 Bad Gateway'
+            , 'Date: ' + (new Date()).toUTCString()
+            , 'Content-Type: text/html'
+            , 'Content-Length: ' + msg.length
+            , 'Connection: close'
+            , ''
+            , msg
+            ].join('\r\n'));
+          }
+        });
+      })
+      ;
+  }
+
+  return {
+    emit: function (type, value) {
+      if (type === 'connection') {
+        handleConnection(value);
+      }
+    }
+  };
+};

lib/tcp/index.js

@@ -0,0 +1,242 @@
+'use strict';
+
+module.exports.create = function (deps, config) {
+  console.log('config', config);
+
+  var listeners = require('../servers').listeners.tcp;
+  var domainUtils = require('../domain-utils');
+  var modules;
+
+  var addrProperties = [
+    'remoteAddress'
+  , 'remotePort'
+  , 'remoteFamily'
+  , 'localAddress'
+  , 'localPort'
+  , 'localFamily'
+  ];
+
+  function nameMatchesDomains(name, domainList) {
+    return domainList.some(function (pattern) {
+      return domainUtils.match(pattern, name);
+    });
+  }
+
+  function proxy(mod, conn, opts) {
+    // First thing we need to add to the connection options is where to proxy the connection to
+    var newConnOpts = domainUtils.separatePort(mod.address || '');
+    newConnOpts.port = newConnOpts.port || mod.port;
+    newConnOpts.host = newConnOpts.host || mod.host || 'localhost';
+
+    // Then we add all of the connection address information. We need to prefix all of the
+    // properties with '_' so we can provide the information to any connection `createConnection`
+    // implementation but not have the default implementation try to bind the same local port.
+    addrProperties.forEach(function (name) {
+      newConnOpts['_' + name] = opts[name] || opts['_'+name] || conn[name] || conn['_'+name];
+    });
+
+    modules.proxy(conn, newConnOpts);
+    return true;
+  }
+
+  function checkTcpProxy(conn, opts) {
+    var proxied = false;
+
+    // TCP Proxying (ie routing based on domain name [vs local port]) only works for
+    // TLS wrapped connections, so if the opts don't give us a servername or don't tell us
+    // this is the decrypted side of a TLS connection we can't handle it here.
+    if (!opts.servername || !opts.encrypted) { return proxied; }
+
+    proxied = config.domains.some(function (dom) {
+      if (!dom.modules || !Array.isArray(dom.modules.tcp)) { return false; }
+      if (!nameMatchesDomains(opts.servername, dom.names)) { return false; }
+
+      return dom.modules.tcp.some(function (mod) {
+        if (mod.type !== 'proxy') { return false; }
+
+        return proxy(mod, conn, opts);
+      });
+    });
+
+    proxied = proxied || config.tcp.modules.some(function (mod) {
+      if (mod.type !== 'proxy') { return false; }
+      if (!nameMatchesDomains(opts.servername, mod.domains)) { return false; }
+
+      return proxy(mod, conn, opts);
+    });
+
+    return proxied;
+  }
+
+  function checkTcpForward(conn, opts) {
+    // TCP forwarding (ie routing connections based on local port) requires the local port
+    if (!conn.localPort) { return false; }
+
+    return config.tcp.modules.some(function (mod) {
+      if (mod.type !== 'forward')                { return false; }
+      if (mod.ports.indexOf(conn.localPort) < 0) { return false; }
+
+      return proxy(mod, conn, opts);
+    });
+  }
+
+  // opts = { servername, encrypted, peek, data, remoteAddress, remotePort }
+  function peek(conn, firstChunk, opts) {
+    opts.firstChunk = firstChunk;
+    conn.__opts = opts;
+    // TODO port/service-based routing can do here
+
+    // TLS byte 1 is handshake and byte 6 is client hello
+    if (0x16 === firstChunk[0]/* && 0x01 === firstChunk[5]*/) {
+      modules.tls.emit('connection', conn);
+      return;
+    }
+
+    // This doesn't work with TLS, but now that we know this isn't a TLS connection we can
+    // unshift the first chunk back onto the connection for future use. The unshift should
+    // happen after any listeners are attached to it but before any new data comes in.
+    if (!opts.hyperPeek) {
+      process.nextTick(function () {
+        conn.unshift(firstChunk);
+      });
+    }
+
+    // Connection is not TLS, check for HTTP next.
+    if (firstChunk[0] > 32 && firstChunk[0] < 127) {
+      var firstStr = firstChunk.toString();
+      if (/HTTP\//i.test(firstStr)) {
+        modules.http.emit('connection', conn);
+        return;
+      }
+    }
+
+    console.warn('failed to identify protocol from first chunk', firstChunk);
+    conn.destroy();
+  }
+  function tcpHandler(conn, opts) {
+    function getProp(name) {
+      return opts[name] || opts['_'+name] || conn[name] || conn['_'+name];
+    }
+    opts = opts || {};
+    var logName = getProp('remoteAddress') + ':' + getProp('remotePort') + ' -> ' +
+                  getProp('localAddress')  + ':' + getProp('localPort');
+    console.log('[tcpHandler]', logName, 'connection started - encrypted: ' + (opts.encrypted || false));
+
+    var start = Date.now();
+    conn.on('timeout', function () {
+      console.log('[tcpHandler]', logName, 'connection timed out', (Date.now()-start)/1000);
+    });
+    conn.on('end', function () {
+      console.log('[tcpHandler]', logName, 'connection ended', (Date.now()-start)/1000);
+    });
+    conn.on('close', function () {
+      console.log('[tcpHandler]', logName, 'connection closed', (Date.now()-start)/1000);
+    });
+
+    if (checkTcpForward(conn, opts)) { return; }
+    if (checkTcpProxy(conn, opts))   { return; }
+
+    // XXX PEEK COMMENT XXX
+    // TODO we can have our cake and eat it too
+    // we can skip the need to wrap the TLS connection twice
+    // because we've already peeked at the data,
+    // but this needs to be handled better before we enable that
+    // (because it creates new edge cases)
+    if (opts.hyperPeek) {
+      console.log('hyperpeek');
+      peek(conn, opts.firstChunk, opts);
+      return;
+    }
+
+    function onError(err) {
+      console.error('[error] socket errored peeking -', err);
+      conn.destroy();
+    }
+    conn.once('error', onError);
+    conn.once('data', function (chunk) {
+      conn.removeListener('error', onError);
+      peek(conn, chunk, opts);
+    });
+  }
+
+  process.nextTick(function () {
+    modules = {};
+    modules.tcpHandler = tcpHandler;
+    modules.proxy = require('./proxy-conn').create(deps, config);
+    modules.tls   = require('./tls').create(deps, config, modules);
+    modules.http  = require('./http').create(deps, config, modules);
+  });
+
+  function updateListeners() {
+    var current = listeners.list();
+    var wanted = config.tcp.bind;
+
+    if (!Array.isArray(wanted)) { wanted = []; }
+    wanted = wanted.map(Number).filter((port) => port > 0 && port < 65356);
+
+    var closeProms = current.filter(function (port) {
+      return wanted.indexOf(port) < 0;
+    }).map(function (port) {
+      return listeners.close(port, 1000);
+    });
+
+    // We don't really need to filter here since listening on the same port with the
+    // same handler function twice is basically a no-op.
+    var openProms = wanted.map(function (port) {
+      return listeners.add(port, tcpHandler);
+    });
+
+    return Promise.all(closeProms.concat(openProms));
+  }
+
+  var mainPort;
+  function updateConf() {
+    updateListeners().catch(function (err) {
+      console.error('Error updating TCP listeners to match bind configuration');
+      console.error(err);
+    });
+
+    var unforwarded = {};
+    config.tcp.bind.forEach(function (port) {
+      unforwarded[port] = true;
+    });
+
+    config.tcp.modules.forEach(function (mod) {
+      if (['forward', 'proxy'].indexOf(mod.type) < 0) {
+        console.warn('unknown TCP module type specified', JSON.stringify(mod));
+      }
+      if (mod.type !== 'forward') { return; }
+
+      mod.ports.forEach(function (port) {
+        if (!unforwarded[port]) {
+          console.warn('trying to forward TCP port ' + port + ' multiple times or it is unbound');
+        } else {
+          delete unforwarded[port];
+        }
+      });
+    });
+
+    // Not really sure what we can reasonably do to prevent this. At least not without making
+    // our configuration validation more complicated.
+    if (!Object.keys(unforwarded).length) {
+      console.warn('no bound TCP ports are not being forwarded, admin interface will be inaccessible');
+    }
+
+    // If we are listening on port 443 make that the main port we respond to mDNS queries with
+    // otherwise choose the lowest number port we are bound to but not forwarding.
+    if (unforwarded['443']) {
+      mainPort = 443;
+    } else {
+      mainPort = Object.keys(unforwarded).map(Number).sort((a, b) => a - b)[0];
+    }
+  }
+  updateConf();
+
+  var result =  {
+    updateConf
+  , handler: tcpHandler
+  };
+  Object.defineProperty(result, 'mainPort', {enumerable: true, get: () => mainPort});
+
+  return result;
+};

lib/tcp/proxy-conn.js

@@ -0,0 +1,81 @@
+'use strict';
+
+function getRespBody(err, debug) {
+  if (debug) {
+    return err.toString();
+  }
+
+  if (err.code === 'ECONNREFUSED') {
+    return 'The connection was refused. Most likely the service being connected to '
+      + 'has stopped running or the configuration is wrong.';
+  }
+
+  return 'Bad Gateway: ' + err.code;
+}
+
+function sendBadGateway(conn, err, debug) {
+  var body = getRespBody(err, debug);
+
+  conn.write([
+    'HTTP/1.1 502 Bad Gateway'
+  , 'Date: ' + (new Date()).toUTCString()
+  , 'Connection: close'
+  , 'Content-Type: text/html'
+  , 'Content-Length: ' + body.length
+  , ''
+  , body
+  ].join('\r\n'));
+  conn.end();
+}
+
+module.exports.getRespBody = getRespBody;
+module.exports.sendBadGateway = sendBadGateway;
+
+module.exports.create = function (deps, config) {
+  function proxy(conn, newConnOpts, firstChunk, decrypt) {
+    var connected = false;
+    newConnOpts.allowHalfOpen = true;
+    var newConn = deps.net.createConnection(newConnOpts, function () {
+      connected = true;
+
+      if (firstChunk) {
+        newConn.write(firstChunk);
+      }
+
+      newConn.pipe(conn);
+      conn.pipe(newConn);
+    });
+
+    // Listening for this largely to prevent uncaught exceptions.
+    conn.on('error', function (err) {
+      console.log('proxy client error', err);
+    });
+    newConn.on('error', function (err) {
+      if (connected) {
+        // Not sure how to report this to a user or a client. We can assume that some data
+        // has already been exchanged, so we can't really be sure what we can send in addition
+        // that wouldn't result in a parse error.
+        console.log('proxy remote error', err);
+      } else {
+        console.log('proxy connection error', err);
+        if (decrypt) {
+          sendBadGateway(decrypt(conn), err, config.debug);
+        } else {
+          sendBadGateway(conn, err, config.debug);
+        }
+      }
+    });
+
+    // Make sure that once one side closes, no I/O activity will happen on the other side.
+    conn.on('close', function () {
+      newConn.destroy();
+    });
+    newConn.on('close', function () {
+      conn.destroy();
+    });
+  }
+
+  proxy.getRespBody = getRespBody;
+  proxy.sendBadGateway = sendBadGateway;
+  return proxy;
+};

lib/tcp/tls.js

@@ -0,0 +1,349 @@
+'use strict';
+
+module.exports.create = function (deps, config, tcpMods) {
+  var path = require('path');
+  var tls = require('tls');
+  var parseSni = require('sni');
+  var greenlock = require('greenlock');
+  var localhostCerts = require('localhost.daplie.me-certificates');
+  var domainMatches = require('../domain-utils').match;
+
+  function extractSocketProp(socket, propName) {
+    // remoteAddress, remotePort... ugh... https://github.com/nodejs/node/issues/8854
+    var altName = '_' + propName;
+    var value = socket[propName] || socket[altName];
+    try {
+      value = value || socket._handle._parent.owner.stream[propName];
+      value = value || socket._handle._parent.owner.stream[altName];
+    } catch (e) {}
+
+    try {
+      value = value || socket._handle._parentWrap[propName];
+      value = value || socket._handle._parentWrap[altName];
+      value = value || socket._handle._parentWrap._handle.owner.stream[propName];
+      value = value || socket._handle._parentWrap._handle.owner.stream[altName];
+    } catch (e) {}
+
+    return value || '';
+  }
+
+  function nameMatchesDomains(name, domainList) {
+    return domainList.some(function (pattern) {
+      return domainMatches(pattern, name);
+    });
+  }
+
+  var addressNames = [
+    'remoteAddress'
+  , 'remotePort'
+  , 'remoteFamily'
+  , 'localAddress'
+  , 'localPort'
+  ];
+  function wrapSocket(socket, opts, cb) {
+    var reader = require('socket-pair').create(function (err, writer) {
+      if (typeof cb === 'function') {
+        process.nextTick(cb);
+      }
+      if (err) {
+        reader.emit('error', err);
+        return;
+      }
+
+      writer.write(opts.firstChunk);
+      socket.pipe(writer);
+      writer.pipe(socket);
+
+      socket.on('error', function (err) {
+        console.log('wrapped TLS socket error', err);
+        reader.emit('error', err);
+      });
+      writer.on('error', function (err) {
+        console.error('socket-pair writer error', err);
+        // If the writer had an error the reader probably did too, and I don't think we'll
+        // get much out of emitting this on the original socket, so logging is enough.
+      });
+
+      socket.on('close', writer.destroy.bind(writer));
+      writer.on('close', socket.destroy.bind(socket));
+    });
+
+    // We can't set these properties the normal way because there is a getter without a setter,
+    // but we can use defineProperty. We reuse the descriptor even though we will be manipulating
+    // it because we will only ever set the value and we set it every time.
+    var descriptor = {enumerable: true, configurable: true, writable: true};
+    addressNames.forEach(function (name) {
+      descriptor.value = opts[name] || extractSocketProp(socket, name);
+      Object.defineProperty(reader, name, descriptor);
+    });
+
+    return reader;
+  }
+
+  var le = greenlock.create({
+    server: 'https://acme-v01.api.letsencrypt.org/directory'
+
+  , challenges: {
+      'http-01': require('le-challenge-fs').create({ debug: config.debug })
+    , 'tls-sni-01': require('le-challenge-sni').create({ debug: config.debug })
+    , 'dns-01': deps.ddns.challenge
+    }
+  , challengeType: 'http-01'
+
+  , store: require('le-store-certbot').create({
+      debug: config.debug
+    , configDir: path.join(require('os').homedir(), 'acme', 'etc')
+    , logDir: path.join(require('os').homedir(), 'acme', 'var', 'log')
+    , workDir: path.join(require('os').homedir(), 'acme', 'var', 'lib')
+    })
+
+  , approveDomains: function (opts, certs, cb) {
+      // This is where you check your database and associated
+      // email addresses with domains and agreements and such
+
+      // The domains being approved for the first time are listed in opts.domains
+      // Certs being renewed are listed in certs.altnames
+      if (certs) {
+        // TODO make sure the same options are used for renewal as for registration?
+        opts.domains = certs.altnames;
+        cb(null, { options: opts, certs: certs });
+        return;
+      }
+
+      function complete(optsOverride, domains) {
+        if (!cb) {
+          console.warn('tried to complete domain approval multiple times');
+          return;
+        }
+
+        // // We can't request certificates for wildcard domains, so filter any of those
+        // // out of this list and put the domain that triggered this in the list if needed.
+        // domains = (domains || []).filter(function (dom) { return dom[0] !== '*'; });
+        // if (domains.indexOf(opts.domain) < 0) {
+        //   domains.push(opts.domain);
+        // }
+        domains = [ opts.domain ];
+        // TODO: allow user to specify options for challenges or storage.
+
+        Object.assign(opts, optsOverride, { domains: domains, agreeTos: true });
+        cb(null, { options: opts, certs: certs });
+        cb = null;
+      }
+
+      var handled = false;
+      if (Array.isArray(config.domains)) {
+        handled = config.domains.some(function (dom) {
+          if (!dom.modules || !dom.modules.tls) {
+            return false;
+          }
+          if (!nameMatchesDomains(opts.domain, dom.names)) {
+            return false;
+          }
+
+          return dom.modules.tls.some(function (mod) {
+            if (mod.type !== 'acme') {
+              return false;
+            }
+            complete(mod, dom.names);
+            return true;
+          });
+        });
+      }
+      if (handled) {
+        return;
+      }
+
+      if (Array.isArray(config.tls.modules)) {
+        handled = config.tls.modules.some(function (mod) {
+          if (mod.type !== 'acme') {
+            return false;
+          }
+          if (!nameMatchesDomains(opts.domain, mod.domains)) {
+            return false;
+          }
+
+          complete(mod, mod.domains);
+          return true;
+        });
+      }
+      if (handled) {
+        return;
+      }
+
+      cb(new Error('domain is not allowed'));
+    }
+  });
+  le.tlsOptions = le.tlsOptions || le.httpsOptions;
+
+  var secureContexts = {};
+  var terminatorOpts = require('localhost.daplie.me-certificates').merge({});
+  terminatorOpts.SNICallback = function (sni, cb) {
+    sni = sni.toLowerCase();
+    console.log("[tlsOptions.SNICallback] SNI: '" + sni + "'");
+
+    var tlsOptions;
+
+    // Static Certs
+    if (/\.invalid$/.test(sni)) {
+      sni = 'localhost.daplie.me';
+    }
+    if (/.*localhost.*\.daplie\.me/.test(sni)) {
+      if (!secureContexts[sni]) {
+        tlsOptions = localhostCerts.mergeTlsOptions(sni, {});
+        if (tlsOptions) {
+          secureContexts[sni] = tls.createSecureContext(tlsOptions);
+        }
+      }
+      if (secureContexts[sni]) {
+        console.log('Got static secure context:', sni, secureContexts[sni]);
+        cb(null, secureContexts[sni]);
+        return;
+      }
+    }
+
+    le.tlsOptions.SNICallback(sni, cb);
+  };
+
+  var terminateServer = tls.createServer(terminatorOpts, function (socket) {
+    console.log('(post-terminated) tls connection, addr:', extractSocketProp(socket, 'remoteAddress'));
+
+    tcpMods.tcpHandler(socket, {
+      servername: socket.servername
+    , encrypted: true
+      // remoteAddress... ugh... https://github.com/nodejs/node/issues/8854
+    , remoteAddress: extractSocketProp(socket, 'remoteAddress')
+    , remotePort:    extractSocketProp(socket, 'remotePort')
+    , remoteFamily:  extractSocketProp(socket, 'remoteFamily')
+    });
+  });
+  terminateServer.on('error', function (err) {
+    console.log('[error] TLS termination server', err);
+  });
+
+  function proxy(socket, opts, mod) {
+    var newConnOpts = require('../domain-utils').separatePort(mod.address || '');
+    newConnOpts.port = newConnOpts.port || mod.port;
+    newConnOpts.host = newConnOpts.host || mod.host || 'localhost';
+    newConnOpts.servername = opts.servername;
+    newConnOpts.data = opts.firstChunk;
+
+    newConnOpts.remoteFamily  = opts.family  || extractSocketProp(socket, 'remoteFamily');
+    newConnOpts.remoteAddress = opts.address || extractSocketProp(socket, 'remoteAddress');
+    newConnOpts.remotePort    = opts.port    || extractSocketProp(socket, 'remotePort');
+
+    tcpMods.proxy(socket, newConnOpts, opts.firstChunk, function () {
+      // This function is called in the event of a connection error and should decrypt
+      // the socket so the proxy module can send a 502 HTTP response.
+      var tlsOpts = localhostCerts.mergeTlsOptions('localhost.daplie.me', {isServer: true});
+      if (opts.hyperPeek) {
+        return new tls.TLSSocket(socket, tlsOpts);
+      } else {
+        return new tls.TLSSocket(wrapSocket(socket, opts), tlsOpts);
+      }
+    });
+    return true;
+  }
+
+  function terminate(socket, opts) {
+    console.log(
+      '[tls-terminate]'
+    , opts.localAddress || socket.localAddress +':'+ opts.localPort || socket.localPort
+    , 'servername=' + opts.servername
+    , opts.remoteAddress || socket.remoteAddress
+    );
+
+    var wrapped;
+    // We can't emit the connection to the TLS server until we know the connection is fully
+    // opened, otherwise it might hang open when the decrypted side is destroyed.
+    // https://github.com/nodejs/node/issues/14605
+    function emitSock() {
+      terminateServer.emit('connection', wrapped);
+    }
+    if (opts.hyperPeek) {
+      // This connection was peeked at using a method that doesn't interferre with the TLS
+      // server's ability to handle it properly. Currently the only way this happens is
+      // with tunnel connections where we have the first chunk of data before creating the
+      // new connection (thus removing need to get data off the new connection).
+      wrapped = socket;
+      process.nextTick(emitSock);
+    }
+    else {
+      // The hyperPeek flag wasn't set, so we had to read data off of this connection, which
+      // means we can no longer use it directly in the TLS server.
+      // See https://github.com/nodejs/node/issues/8752 (node's internal networking layer == 💩 sometimes)
+      wrapped = wrapSocket(socket, opts, emitSock);
+    }
+  }
+
+  function handleConn(socket, opts) {
+    opts.servername = (parseSni(opts.firstChunk)||'').toLowerCase() || 'localhost.invalid';
+    // needs to wind up in one of 2 states:
+    // 1. SNI-based Proxy / Tunnel (we don't even need to put it through the tlsSocket)
+    // 2. Terminated (goes on to a particular module or route, including the admin interface)
+    // 3. Closed (we don't recognize the SNI servername as something we actually want to handle)
+
+    // We always want to terminate is the SNI matches the challenge pattern, unless a client
+    // on the south side has temporarily claimed a particular challenge. For the time being
+    // we don't have a way for the south-side to communicate with us, so that part isn't done.
+    if (domainMatches('*.acme-challenge.invalid', opts.servername)) {
+      terminate(socket, opts);
+      return;
+    }
+
+    if (deps.stunneld.isClientDomain(opts.servername)) {
+      deps.stunneld.handleClientConn(socket);
+      if (!opts.hyperPeek) {
+        process.nextTick(function () {
+          socket.unshift(opts.firstChunk);
+        });
+      }
+      return;
+    }
+
+    function checkModule(mod) {
+      if (mod.type === 'proxy') {
+        return proxy(socket, opts, mod);
+      }
+      if (mod.type !== 'acme') {
+        console.error('saw unknown TLS module', mod);
+      }
+    }
+
+    var handled = (config.domains || []).some(function (dom) {
+      if (!dom.modules || !dom.modules.tls) {
+        return false;
+      }
+      if (!nameMatchesDomains(opts.servername, dom.names)) {
+        return false;
+      }
+
+      return dom.modules.tls.some(checkModule);
+    });
+    if (handled) {
+      return;
+    }
+
+    handled = (config.tls.modules || []).some(function (mod) {
+      if (!nameMatchesDomains(opts.servername, mod.domains)) {
+        return false;
+      }
+      return checkModule(mod);
+    });
+    if (handled) {
+      return;
+    }
+
+    // TODO: figure out all of the domains that the other modules intend to handle, and only
+    // terminate those ones, closing connections for all others.
+    terminate(socket, opts);
+  }
+
+  return {
+    emit: function (type, socket) {
+      if (type === 'connection') {
+        handleConn(socket, socket.__opts);
+      }
+    }
+  , middleware: le.middleware()
+  };
+};

lib/tunnel-server-manager.js

@@ -0,0 +1,131 @@
+'use strict';
+
+function httpsTunnel(servername, conn) {
+  console.error('tunnel server received encrypted connection to', servername);
+  conn.end();
+}
+function handleHttp(servername, conn) {
+  console.error('tunnel server received un-encrypted connection to', servername);
+  conn.end([
+    'HTTP/1.1 404 Not Found'
+  , 'Date: ' + (new Date()).toUTCString()
+  , 'Connection: close'
+  , 'Content-Type: text/html'
+  , 'Content-Length: 9'
+  , ''
+  , 'Not Found'
+  ].join('\r\n'));
+}
+function rejectNonWebsocket(req, res) {
+  // status code 426 = Upgrade Required
+  res.statusCode = 426;
+  res.setHeader('Content-Type', 'application/json');
+  res.send({error: { message: 'Only websockets accepted for tunnel server' }});
+}
+
+var defaultConfig = {
+  servernames: []
+, secret: null
+};
+var tunnelFuncs = {
+  // These functions should not be called because connections to the admin domains
+  // should already be decrypted, and connections to non-client domains should never
+  // be given to us in the first place.
+  httpsTunnel:  httpsTunnel
+, httpsInvalid: httpsTunnel
+  // These function should not be called because ACME challenges should be handled
+  // before admin domain connections are given to us, and the only non-encrypted
+  // client connections that should be given to us are ACME challenges.
+, handleHttp:         handleHttp
+, handleInsecureHttp: handleHttp
+};
+
+module.exports.create = function (deps, config) {
+  var equal = require('deep-equal');
+  var enableDestroy = require('server-destroy');
+  var currentOpts = Object.assign({}, defaultConfig);
+
+  var httpServer, wsServer, stunneld;
+  function start() {
+    if (httpServer || wsServer || stunneld) {
+      throw new Error('trying to start already started tunnel server');
+    }
+    httpServer = require('http').createServer(rejectNonWebsocket);
+    enableDestroy(httpServer);
+
+    wsServer = new (require('ws').Server)({ server: httpServer });
+
+    var tunnelOpts = Object.assign({}, tunnelFuncs, currentOpts);
+    stunneld = require('stunneld').create(tunnelOpts);
+    wsServer.on('connection', stunneld.ws);
+  }
+
+  function stop() {
+    if (!httpServer || !wsServer || !stunneld) {
+      throw new Error('trying to stop unstarted tunnel server (or it got into semi-initialized state');
+    }
+    wsServer.close();
+    wsServer = null;
+    httpServer.destroy();
+    httpServer = null;
+    // Nothing to close here, just need to set it to null to allow it to be garbage-collected.
+    stunneld = null;
+  }
+
+  function updateConf() {
+    var newOpts = Object.assign({}, defaultConfig, config.tunnelServer);
+    if (!Array.isArray(newOpts.servernames)) {
+      newOpts.servernames = [];
+    }
+    var trimmedOpts = {
+      servernames: newOpts.servernames.slice().sort()
+    , secret:      newOpts.secret
+    };
+
+    if (equal(trimmedOpts, currentOpts)) {
+      return;
+    }
+    currentOpts = trimmedOpts;
+
+    // Stop what's currently running, then if we are still supposed to be running then we
+    // can start it again with the updated options. It might be possible to make use of
+    // the existing http and ws servers when the config changes, but I'm not sure what
+    // state the actions needed to close all existing connections would put them in.
+    if (httpServer || wsServer || stunneld) {
+      stop();
+    }
+    if (currentOpts.servernames.length && currentOpts.secret) {
+      start();
+    }
+  }
+  process.nextTick(updateConf);
+
+  return {
+    isAdminDomain: function (domain) {
+      return currentOpts.servernames.indexOf(domain) !== -1;
+    }
+  , handleAdminConn: function (conn) {
+      if (!httpServer) {
+        console.error(new Error('handleAdminConn called with no active tunnel server'));
+        conn.end();
+      } else {
+        return httpServer.emit('connection', conn);
+      }
+    }
+
+  , isClientDomain: function (domain) {
+      if (!stunneld) { return false; }
+      return stunneld.isClientDomain(domain);
+    }
+  , handleClientConn: function (conn) {
+      if (!stunneld) {
+        console.error(new Error('handleClientConn called with no active tunnel server'));
+        conn.end();
+      } else {
+        return stunneld.tcp(conn);
+      }
+    }
+
+  , updateConf
+  };
+};

lib/tunnel.js

@@ -1,144 +0,0 @@
-'use strict';
-
-module.exports.create = function (opts, servers) {
-  // servers = { plainserver, server }
-  var Oauth3 = require('oauth3-cli');
-  var Tunnel = require('daplie-tunnel').create({
-    Oauth3: Oauth3
-  , PromiseA: opts.PromiseA
-  , CLI: {
-      init: function (rs, ws/*, state, options*/) {
-        // noop
-        return ws;
-      }
-    }
-  }).Tunnel;
-  var stunnel = require('stunnel');
-  var killcount = 0;
-
-  /*
-  var Dup = {
-    write: function (chunk, encoding, cb) {
-      this.__my_socket.push(chunk, encoding);
-      cb();
-    }
-  , read: function (size) {
-      var x = this.__my_socket.read(size);
-      if (x) { this.push(x); }
-    }
-  , setTimeout: function () {
-      console.log('TODO implement setTimeout on Duplex');
-    }
-  };
-
-  var httpServer = require('http').createServer(function (req, res) {
-    console.log('req.socket.encrypted', req.socket.encrypted);
-    res.end('Hello, tunneled World!');
-  });
-
-  var tlsServer = require('tls').createServer(opts.httpsOptions, function (tlsSocket) {
-    console.log('tls connection');
-    // things get a little messed up here
-    httpServer.emit('connection', tlsSocket);
-
-    // try again
-    //servers.server.emit('connection', tlsSocket);
-  });
-  */
-
-  process.on('SIGINT', function () {
-    killcount += 1;
-    console.log('[quit] closing http and https servers');
-    if (killcount >= 3) {
-      process.exit(1);
-    }
-    if (servers.server) {
-      servers.server.close();
-    }
-    if (servers.insecureServer) {
-      servers.insecureServer.close();
-    }
-  });
-
-  return Tunnel.token({
-    refreshToken: opts.refreshToken
-  , email: opts.email
-  , domains: opts.sites.map(function (site) {
-      return site.name;
-    })
-  , device: { hostname: opts.devicename || opts.device }
-  }).then(function (result) {
-    // { jwt, tunnelUrl }
-    var locals = [];
-    opts.sites.map(function (site) {
-      locals.push({
-        protocol: 'https'
-      , hostname: site.name
-      , port: opts.port
-      });
-      locals.push({
-        protocol: 'http'
-      , hostname: site.name
-      , port: opts.insecurePort || opts.port
-      });
-    });
-    return stunnel.connect({
-      token: result.jwt
-    , stunneld: result.tunnelUrl
-      // XXX TODO BUG // this is just for testing
-    , insecure: /*opts.insecure*/ true
-    , locals: locals
-      // a simple passthru is proving to not be so simple
-    , net: require('net') /*
-      {
-        createConnection: function (info, cb) {
-          // data is the hello packet / first chunk
-          // info = { data, servername, port, host, remoteAddress: { family, address, port } }
-
-          var myDuplex = new (require('stream').Duplex)();
-          var myDuplex2 = new (require('stream').Duplex)();
-          // duplex = { write, push, end, events: [ 'readable', 'data', 'error', 'end' ] };
-
-          myDuplex2.__my_socket = myDuplex;
-          myDuplex.__my_socket = myDuplex2;
-
-          myDuplex2._write = Dup.write;
-          myDuplex2._read = Dup.read;
-
-          myDuplex._write = Dup.write;
-          myDuplex._read = Dup.read;
-
-          myDuplex.remoteFamily = info.remoteFamily;
-          myDuplex.remoteAddress = info.remoteAddress;
-          myDuplex.remotePort = info.remotePort;
-
-          // socket.local{Family,Address,Port}
-          myDuplex.localFamily = 'IPv4';
-          myDuplex.localAddress = '127.0.01';
-          myDuplex.localPort = info.port;
-
-          myDuplex.setTimeout = Dup.setTimeout;
-
-          // this doesn't seem to work so well
-          //servers.server.emit('connection', myDuplex);
-
-          // try a little more manual wrapping / unwrapping
-          var firstByte = info.data[0];
-          if (firstByte < 32 || firstByte >= 127) {
-            tlsServer.emit('connection', myDuplex);
-          }
-          else {
-            httpServer.emit('connection', myDuplex);
-          }
-
-          if (cb) {
-            process.nextTick(cb);
-          }
-
-          return myDuplex2;
-        }
-      }
-      //*/
-    });
-  });
-};

lib/udp.js

@@ -0,0 +1,57 @@
+'use strict';
+
+module.exports.create = function (deps, config) {
+  var listeners = require('./servers').listeners.udp;
+
+  function packetHandler(port, msg) {
+    if (!Array.isArray(config.udp.modules)) {
+      return;
+    }
+
+    var socket = require('dgram').createSocket('udp4');
+    config.udp.modules.forEach(function (mod) {
+      if (mod.type !== 'forward') {
+        // To avoid logging bad modules every time we get a UDP packet we assign a warned
+        // property to the module (non-enumerable so it won't be saved to the config or
+        // show up in the API).
+        if (!mod.warned) {
+          console.warn('found bad DNS module', mod);
+          Object.defineProperty(mod, 'warned', {value: true, enumerable: false});
+        }
+        return;
+      }
+      if (mod.ports.indexOf(port) < 0) {
+        return;
+      }
+
+      var dest = require('./domain-utils').separatePort(mod.address || '');
+      dest.port = dest.port || mod.port;
+      dest.host = dest.host || mod.host || 'localhost';
+      socket.send(msg, dest.port, dest.host);
+    });
+  }
+
+  function updateListeners() {
+    var current = listeners.list();
+    var wanted = config.udp.bind;
+
+    if (!Array.isArray(wanted)) { wanted = []; }
+    wanted = wanted.map(Number).filter((port) => port > 0 && port < 65356);
+
+    current.forEach(function (port) {
+      if (wanted.indexOf(port) < 0) {
+        listeners.close(port);
+      }
+    });
+    wanted.forEach(function (port) {
+      if (current.indexOf(port) < 0) {
+        listeners.add(port, packetHandler.bind(port));
+      }
+    });
+  }
+
+  updateListeners();
+  return {
+    updateConf: updateListeners
+  };
+};

lib/worker.js

@@ -1,13 +1,64 @@
 'use strict';
 
-// TODO needs some sort of config-sync
-process.on('message', function (conf) {
+var config;
+var modules;
+
+// Everything that uses the config should be reading it when relevant rather than
+// just at the beginning, so we keep the reference for the main object and just
+// change all of its properties to match the new config.
+function update(conf) {
+  var newKeys = Object.keys(conf);
+
+  Object.keys(config).forEach(function (key) {
+    if (newKeys.indexOf(key) < 0) {
+      delete config[key];
+    } else {
+      config[key] = conf[key];
+    }
+  });
+
+  console.log('config update', JSON.stringify(config));
+  Object.values(modules).forEach(function (mod) {
+    if (typeof mod.updateConf === 'function') {
+      mod.updateConf(config);
+    }
+  });
+}
+
+function create(conf) {
+  var PromiseA = require('bluebird');
+  var OAUTH3 = require('../packages/assets/org.oauth3');
+  require('../packages/assets/org.oauth3/oauth3.domains.js');
+  require('../packages/assets/org.oauth3/oauth3.dns.js');
+  require('../packages/assets/org.oauth3/oauth3.tunnel.js');
+  OAUTH3._hooks = require('../packages/assets/org.oauth3/oauth3.node.storage.js');
+
+  config = conf;
   var deps = {
     messenger: process
+  , PromiseA: PromiseA
+  , OAUTH3: OAUTH3
+  , request: PromiseA.promisify(require('request'))
+  , recase: require('recase').create({})
     // Note that if a custom createConnections is used it will be called with different
     // sets of custom options based on what is actually being proxied. Most notably the
     // HTTP proxying connection creation is not something we currently control.
   , net: require('net')
   };
-  require('./goldilocks.js').create(deps, conf);
-});
+
+  modules = {
+    storage:  require('./storage').create(deps, conf)
+  , socks5:   require('./socks5-server').create(deps, conf)
+  , ddns:     require('./ddns').create(deps, conf)
+  , mdns:     require('./mdns').create(deps, conf)
+  , udp:      require('./udp').create(deps, conf)
+  , tcp:      require('./tcp').create(deps, conf)
+  , stunneld: require('./tunnel-server-manager').create(deps, config)
+  };
+  Object.assign(deps, modules);
+
+  process.removeListener('message', create);
+  process.on('message', update);
+}
+
+process.on('message', create);

package.json

@@ -1,14 +1,14 @@
 {
   "name": "goldilocks",
-  "version": "1.0.0-placeholder",
+  "version": "1.1.6",
   "description": "The node.js webserver that's just right, Greenlock (HTTPS/TLS/SSL via ACME/Let's Encrypt) and tunneling (RVPN) included.",
   "main": "bin/goldilocks.js",
   "repository": {
     "type": "git",
-    "url": "git@git.daplie.com:Daplie/goldilocks.js.git"
+    "url": "git.coolaj86.com:coolaj86/goldilocks.js.git"
   },
-  "author": "AJ ONeal <aj@daplie.com> (https://daplie.com/)",
-  "license": "SEE LICENSE IN LICENSE.txt",
+  "author": "AJ ONeal <coolaj86@gmail.com> (https://coolaj86.com/)",
+  "license": "(MIT OR Apache-2.0)",
   "scripts": {
     "test": "node bin/goldilocks.js -p 8443 -d /tmp/"
   },
@@ -34,38 +34,42 @@
     "server"
   ],
   "bugs": {
-    "url": "https://git.daplie.com/Daplie/server-https/issues"
+    "url": "https://git.coolaj86.com/coolaj86/goldilocks.js/issues"
   },
-  "homepage": "https://git.daplie.com/Daplie/goldilocks.js#readme",
+  "homepage": "https://git.coolaj86.com/coolaj86/goldilocks.js",
   "dependencies": {
     "bluebird": "^3.4.6",
-    "body-parser": "git+https://github.com/expressjs/body-parser.git#1.16.1",
+    "body-parser": "1",
     "commander": "^2.9.0",
-    "daplie-tunnel": "git+https://git.daplie.com/Daplie/daplie-cli-tunnel.git#master",
-    "ddns-cli": "git+https://git.daplie.com/Daplie/node-ddns-client.git#master",
-    "express": "git+https://github.com/expressjs/express.git#4.x",
+    "deep-equal": "^1.0.1",
+    "dns-suite": "1",
+    "express": "4",
     "finalhandler": "^0.4.0",
-    "greenlock": "git+https://git.daplie.com/Daplie/node-greenlock.git#master",
-    "greenlock-express": "git+https://git.daplie.com/Daplie/greenlock-express.git#master",
+    "greenlock": "2.1",
     "http-proxy": "^1.16.2",
-    "httpolyglot": "^0.1.1",
-    "ipaddr.js": "git+https://github.com/whitequark/ipaddr.js.git#v1.3.0",
-    "ipify": "^1.1.0",
+    "human-readable-ids": "1",
+    "ipaddr.js": "v1.3",
     "js-yaml": "^3.8.3",
-    "le-challenge-ddns": "git+https://git.daplie.com/Daplie/le-challenge-ddns.git#master",
-    "le-challenge-fs": "git+https://git.daplie.com/Daplie/le-challenge-webroot.git#master",
+    "jsonschema": "^1.2.0",
+    "jsonwebtoken": "^7.4.0",
+    "le-challenge-fs": "2",
     "le-challenge-sni": "^2.0.1",
-    "livereload": "^0.6.0",
-    "localhost.daplie.me-certificates": "^1.3.0",
-    "minimist": "^1.1.1",
-    "oauth3-cli": "git+https://git.daplie.com/OAuth3/oauth3-cli.git#master",
-    "recase": "git+https://git.daplie.com/coolaj86/recase-js.git#v1.0.4",
+    "le-store-certbot": "2",
+    "localhost.daplie.me-certificates": "^1.3.5",
+    "network": "^0.4.0",
+    "recase": "v1.0.4",
     "redirect-https": "^1.1.0",
-    "scmp": "git+https://github.com/freewil/scmp.git#1.x",
+    "request": "^2.81.0",
+    "scmp": "1",
     "serve-index": "^1.7.0",
     "serve-static": "^1.10.0",
     "server-destroy": "^1.0.1",
-    "stream-pair": "^1.0.3",
-    "stunnel": "git+https://git.daplie.com/Daplie/node-tunnel-client.git#v1"
+    "sni": "^1.0.0",
+    "socket-pair": "^1.0.3",
+    "socksv5": "0.0.6",
+    "stunnel": "1.0",
+    "stunneld": "0.9",
+    "tunnel-packer": "^1.3.0",
+    "ws": "^2.3.1"
   }
 }

packages/apis/com.daplie.goldilocks/index.js

@@ -1,185 +0,0 @@
-'use strict';
-
-module.exports.dependencies = [ 'OAUTH3', 'storage.owners', 'options.device' ];
-module.exports.create = function (deps, conf) {
-  var scmp = require('scmp');
-  var crypto = require('crypto');
-  var jwt = require('jsonwebtoken');
-  var bodyParser = require('body-parser');
-  var jsonParser = bodyParser.json({
-    inflate: true, limit: '100kb', reviver: null, strict: true /* type, verify */
-  });
-
-  var api = deps.api;
-
-  /*
-  var owners;
-  deps.storage.owners.on('set', function (_owners) {
-    owners = _owners;
-  });
-  */
-
-  function isAuthorized(req, res, fn) {
-    var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, ''));
-    if (!auth) {
-      res.setHeader('Content-Type', 'application/json;');
-      res.end(JSON.stringify({ error: { message: "no token", code: 'E_NO_TOKEN', uri: undefined } }));
-      return;
-    }
-
-    var id = crypto.createHash('sha256').update(auth.sub).digest('hex');
-    return deps.storage.owners.exists(id).then(function (exists) {
-      if (!exists) {
-        res.setHeader('Content-Type', 'application/json;');
-        res.end(JSON.stringify({ error: { message: "not authorized", code: 'E_NO_AUTHZ', uri: undefined } }));
-        return;
-      }
-
-      req.userId = id;
-      fn();
-    });
-  }
-
-  return {
-    init: function (req, res) {
-      jsonParser(req, res, function () {
-
-      return deps.PromiseA.resolve().then(function () {
-
-        console.log('req.body', req.body);
-        var auth = jwt.decode((req.headers.authorization||'').replace(/^bearer\s+/i, ''));
-        var token = jwt.decode(req.body.access_token);
-        var refresh = jwt.decode(req.body.refresh_token);
-        auth.sub = auth.sub || auth.acx.id;
-        token.sub = token.sub || token.acx.id;
-        refresh.sub = refresh.sub || refresh.acx.id;
-
-        // TODO validate token with issuer, but as-is the sub is already a secret
-        var id = crypto.createHash('sha256').update(auth.sub).digest('hex');
-        var tid = crypto.createHash('sha256').update(token.sub).digest('hex');
-        var rid = crypto.createHash('sha256').update(refresh.sub).digest('hex');
-        var session = {
-          access_token: req.body.access_token
-        , token: token
-        , refresh_token: req.body.refresh_token
-        , refresh: refresh
-        };
-
-        console.log('ids', id, tid, rid);
-
-        if (req.body.ip_url) {
-          // TODO set options / GunDB
-          conf.ip_url = req.body.ip_url;
-        }
-
-        return deps.storage.owners.all().then(function (results) {
-          console.log('results', results);
-          var err;
-
-          // There is no owner yet. First come, first serve.
-          if (!results || !results.length) {
-            if (tid !== id || rid !== id) {
-              err = new Error(
-                "When creating an owner the Authorization Bearer and Token and Refresh must all match"
-              );
-              return deps.PromiseA.reject(err);
-            }
-            console.log('no owner, creating');
-            return deps.storage.owners.set(id, session);
-          }
-          console.log('has results');
-
-          // There are onwers. Is this one of them?
-          if (!results.some(function (token) {
-            return scmp(id, token.id);
-          })) {
-            err = new Error("Authorization token does not belong to an existing owner.");
-            return deps.PromiseA.reject(err);
-          }
-          console.log('has correct owner');
-
-          // We're adding an owner, unless it already exists
-          if (!results.some(function (token) {
-            return scmp(tid, token.id);
-          })) {
-            console.log('adds new owner with existing owner');
-            return deps.storage.owners.set(id, session);
-          }
-        }).then(function () {
-          res.setHeader('Content-Type', 'application/json;');
-          res.end(JSON.stringify({ success: true }));
-        });
-      }, function (err) {
-        res.setHeader('Content-Type', 'application/json;');
-        res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } }));
-      });
-
-      });
-    }
-  , tunnel: function (req, res) {
-      isAuthorized(req, res, function () {
-        jsonParser(req, res, function () {
-
-          console.log('req.body', req.body);
-
-          return deps.storage.owners.get(req.userId).then(function (session) {
-            session.token.id = req.userId;
-            return api.tunnel(deps, session).then(function () {
-              res.setHeader('Content-Type', 'application/json;');
-              res.end(JSON.stringify({ success: true }));
-            }, function (err) {
-              res.setHeader('Content-Type', 'application/json;');
-              res.end(JSON.stringify({ error: { message: err.message, code: err.code, uri: err.uri } }));
-            });
-          });
-        });
-      });
-    }
-  , config: function (req, res) {
-      isAuthorized(req, res, function () {
-        if ('POST' !== req.method) {
-          res.setHeader('Content-Type', 'application/json;');
-          res.end(JSON.stringify(deps.recase.snakeCopy(conf)));
-          return;
-        }
-
-        jsonParser(req, res, function () {
-
-          console.log('req.body', req.body);
-
-          deps.storage.config.merge(req.body);
-          deps.storage.config.save();
-        });
-      });
-    }
-  , request: function (req, res) {
-      jsonParser(req, res, function () {
-      isAuthorized(req, res, function () {
-
-        deps.request({
-          method: req.body.method || 'GET'
-        , url: req.body.url
-        , headers: req.body.headers
-        , body: req.body.data
-        }).then(function (resp) {
-          if (resp.body instanceof Buffer || 'string' === typeof resp.body) {
-            resp.body = JSON.parse(resp.body);
-          }
-
-          return {
-            statusCode: resp.statusCode
-          , status: resp.status
-          , headers: resp.headers
-          , body: resp.body
-          , data: resp.data
-          };
-        }).then(function (result) {
-          res.send(result);
-        });
-
-      });
-      });
-    }
-  , _api: api
-  };
-};

packages/apis/com.daplie.goldilocks/test.js

@@ -1,23 +0,0 @@
-'use strict';
-
-var api = require('./index.js').api;
-var OAUTH3 = require('../../assets/org.oauth3/');
-// these all auto-register
-require('../../assets/org.oauth3/oauth3.domains.js');
-require('../../assets/org.oauth3/oauth3.dns.js');
-require('../../assets/org.oauth3/oauth3.tunnel.js');
-OAUTH3._hooks = require('../../assets/org.oauth3/oauth3.node.storage.js');
-
-api.tunnel(
-  {
-    OAUTH3: OAUTH3
-  , options: {
-      device: {
-        hostname: 'test.local'
-      , id: ''
-      }
-    }
-  }
-  // OAUTH3.hooks.session.get('oauth3.org').then(function (result) { console.log(result) });
-, require('./test.session.json')
-);

packages/assets/org.oauth3

@@ -1 +0,0 @@
-Subproject commit c4cc61992805469f86ecd4d74afc18cf6506155b

stages/01-serve.js

@@ -1,23 +0,0 @@
-'use strict';
-
-var https = require('httpolyglot');
-var httpsOptions = require('localhost.daplie.me-certificates').merge({});
-var httpsPort = 8443;
-var redirectApp = require('redirect-https')({
-  port: httpsPort
-});
-
-var server = https.createServer(httpsOptions);
-
-server.on('request', function (req, res) {
-  if (!req.socket.encrypted) {
-    redirectApp(req, res);
-    return;
-  }
-
-  res.end("Hello, Encrypted World!");
-});
-
-server.listen(httpsPort, function () {
-  console.log('https://' + 'localhost.daplie.me' + (443 === httpsPort ? ':' : ':' + httpsPort));
-});

terms.sh

@@ -1,3 +0,0 @@
-# adding TOS to TXT DNS Record
-daplie dns:set -n _terms._cloud.localhost.foo.daplie.me -t TXT -a '{"url":"oauth3.org/tos/draft","explicit":true}' --ttl 3600
-daplie dns:set -n _terms._cloud.localhost.alpha.daplie.me -t TXT -a '{"url":"oauth3.org/tos/draft","explicit":true}' --ttl 3600

test-chain.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-node serve.js \
-  --port 8443 \
-  --key node_modules/localhost.daplie.me-certificates/privkey.pem \
-  --cert node_modules/localhost.daplie.me-certificates/fullchain.pem \
-  --root node_modules/localhost.daplie.me-certificates/root.pem \
-  -c "$(cat node_modules/localhost.daplie.me-certificates/root.pem)" &
-
-PID=$!
-
-sleep 1
-curl -s --insecure http://localhost.daplie.me:8443 > ./root.pem
-curl -s https://localhost.daplie.me:8443 --cacert ./root.pem
-
-rm ./root.pem
-kill $PID 2>/dev/null

uninstall.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# something or other about android and tmux using PREFIX
-#: "${PREFIX:=''}"
-MY_ROOT=""
-if [ -z "${PREFIX-}" ]; then
-  MY_ROOT=""
-else
-  MY_ROOT="$PREFIX"
-fi
-# Not every platform has or needs sudo, gotta save them O(1)s...
-sudo_cmd=""
-((EUID)) && [[ -z "$ANDROID_ROOT" ]] && sudo_cmd="sudo"
-
-# you don't want any oopsies when an rm -rf is involved...
-set -e
-set -u
-
-my_app_name=goldilocks
-my_app_pkg_name=com.daplie.goldilocks.web
-
-my_app_etc_config="etc/${my_app_name}/${my_app_name}.yml"
-my_app_systemd_service="etc/systemd/system/${my_app_name}.service"
-my_app_systemd_tmpfiles="etc/tmpfiles.d/${my_app_name}.conf"
-my_app_launchd_service="Library/LaunchDaemons/${my_app_pkg_name}.plist"
-my_app_upstart_service="etc/init.d/${my_app_name}.conf"
-
-
-$sudo_cmd rm -f /usr/local/bin/$my_app_name
-$sudo_cmd rm -f /usr/local/bin/uninstall-$my_app_name
-$sudo_cmd rm -rf /usr/local/lib/node_modules/$my_app_name
-
-$sudo_cmd rm -f "$MY_ROOT/$my_app_etc_config"
-$sudo_cmd rmdir -p $(dirname "$MY_ROOT/$my_app_etc_config") 2>/dev/null || true
-$sudo_cmd rm -f "$MY_ROOT/$my_app_systemd_service"
-$sudo_cmd rm -f "$MY_ROOT/$my_app_systemd_tmpfiles"
-$sudo_cmd rm -f "$MY_ROOT/$my_app_launchd_service"
-$sudo_cmd rm -f "$MY_ROOT/$my_app_upstart_service"
-
-$sudo_cmd rm -rf /opt/$my_app_name
-$sudo_cmd rm -rf /var/log/$my_app_name
-
-# TODO flag for --purge
-#rm -rf /etc/goldilocks
-
-# TODO trap uninstall function
-
-echo "uninstall complete: $my_app_name"

update-packages.sh

@@ -1,22 +0,0 @@
-pushd packages/assets
-
-git clone https://git.daplie.com/Daplie/oauth3.js.git org.oauth3
-pushd org.oauth3
-git checkout master
-git pull
-popd
-
-mkdir -p com.jquery
-pushd com.jquery
-wget 'https://code.jquery.com/jquery-3.1.1.js' -O jquery-3.1.1.js
-popd
-
-mkdir -p com.google
-pushd com.google
-wget 'https://ajax.googleapis.com/ajax/libs/angularjs/1.6.2/angular.min.js' -O angular.1.6.2.min.js
-popd
-
-mkdir -p well-known
-pushd well-known
-ln -snf ../org.oauth3/well-known/oauth3 ./oauth3
-popd