chimney (log cleanup)

bin/telebit-remote.js

@@ -6,10 +6,18 @@ var pkg = require('../package.json');
 var os = require('os');
 
 //var url = require('url');
+var fs = require('fs');
 var path = require('path');
 var http = require('http');
 //var https = require('https');
 var YAML = require('js-yaml');
+var TOML = require('toml');
+var TPLS = TOML.parse(fs.readFileSync(path.join(__dirname, "../lib/en-us.toml"), 'utf8'));
+/*
+if ('function' !== typeof TOML.stringify) {
+  TOML.stringify = require('json2toml');
+}
+*/
 var recase = require('recase').create({});
 var camelCopy = recase.camelCopy.bind(recase);
 //var snakeCopy = recase.snakeCopy.bind(recase);
@@ -20,6 +28,9 @@ var common = require('../lib/cli-common.js');
 var argv = process.argv.slice(2);
 
 var argIndex = argv.indexOf('--config');
+if (-1 === argIndex) {
+  argIndex = argv.indexOf('-c');
+}
 var confpath;
 var useTty;
 var state = {};
@@ -35,50 +46,13 @@ if (-1 !== argIndex) {
 }
 
 function help() {
-  //console.info('');
+  console.info(TPLS.remote.help.main.replace(/{version}/g, pkg.version));
-  //console.info('Telebit Remote v' + pkg.version);
-  console.info('');
-  console.info('Usage:');
-  console.info('');
-  console.info('\ttelebit [--config <path>] <module> <module-options>');
-  console.info('');
-  console.info('Examples:');
-  console.info('');
-  //console.info('\ttelebit init                            # bootstrap the config files');
-  //console.info('');
-  console.info('\ttelebit status                          # whether enabled or disabled');
-  console.info('\ttelebit enable                          # allow incoming connections');
-  console.info('\ttelebit disable                         # disallow incoming connections');
-  console.info('');
-  console.info('\ttelebit list                            # list rules for servernames and ports');
-  console.info('');
-  console.info('\ttelebit http none                       # remove all https handlers');
-  console.info('\ttelebit http 3000                       # forward all https traffic to port 3000');
-  console.info('\ttelebit http /module/path               # load a node module to handle all https traffic');
-  console.info('');
-  console.info('\ttelebit http none example.com           # remove https handler from example.com');
-  console.info('\ttelebit http 3001 sub.example.com       # forward https traffic for sub.example.com to port 3001');
-  console.info('\ttelebit http /module/path sub           # forward https traffic for sub.example.com to port 3001');
-  console.info('');
-  console.info('\ttelebit tcp none                        # remove all tcp handlers');
-  console.info('\ttelebit tcp 5050                        # forward all tcp to port 5050');
-  console.info('\ttelebit tcp /module/path                # handle all tcp with a node module');
-  console.info('');
-  console.info('\ttelebit tcp none 6565                   # remove tcp handler from external port 6565');
-  console.info('\ttelebit tcp 5050 6565                   # forward external port 6565 to local 5050');
-  console.info('\ttelebit tcp /module/path 6565           # handle external port 6565 with a node module');
-  console.info('');
-  console.info('Config:');
-  console.info('');
-  console.info('\tSee https://git.coolaj86.com/coolaj86/telebit.js');
-  console.info('');
-  console.info('');
 }
 
 var verstr = [ pkg.name + ' remote v' + pkg.version ];
 if (!confpath) {
   confpath = path.join(os.homedir(), '.config/telebit/telebit.yml');
-  verstr.push('(--config "' + confpath + '")');
+  verstr.push('(--config \'' + confpath.replace(new RegExp('^' + os.homedir()), '~') + '\')');
 }
 
 if (-1 !== argv.indexOf('-h') || -1 !== argv.indexOf('--help')) {
@@ -435,6 +409,7 @@ var utils = {
           console.info(body.message);
         } else if ("CONFIG" === body.code) {
           delete body.code;
+          //console.info(TOML.stringify(body));
           console.info(YAML.safeDump(body));
         } else {
           if ('http' === body.module) {
@@ -447,7 +422,7 @@ var utils = {
           } else if ('tcp' === body.module) {
               console.info('> Forwarding ' + state.config.relay + ':' + body.remote + ' => localhost:' + body.local);
           } else if ('ssh' === body.module) {
-              console.info('> Forwarding ' + state.config.relay + ' -p ' + body.remote + ' => localhost:' + body.local);
+              //console.info('> Forwarding ' + state.config.relay + ' -p ' + JSON.stringify(body) + ' => localhost:' + body.local);
               console.info('> Forwarding ssh+https (openssl proxy) => localhost:' + body.local);
           } else {
             console.info(JSON.stringify(body, null, 2));
@@ -604,7 +579,7 @@ function getToken(err, state) {
         if (state._useTty) {
           setTimeout(function () {
             console.info("Some fun things to try first:\n");
-            console.info("    ~/telebit http 3000");
+            console.info("    ~/telebit http ~/public");
             console.info("    ~/telebit tcp 5050");
             console.info("    ~/telebit ssh auto");
             console.info();
@@ -670,8 +645,12 @@ function handleConfig(err, config) {
   //console.log('CONFIG');
   //console.log(config);
   state.config = config;
-  var verstr = [ pkg.name + ' daemon v' + state.config.version ];
+  var verstrd = [ pkg.name + ' daemon v' + state.config.version ];
-  console.info(verstr.join(' '));
+  if (state.config.version && state.config.version !== pkg.version) {
+    console.info(verstr.join(' '), verstrd.join(' '));
+  } else {
+    console.info(verstr.join(' '));
+  }
 
   if (err) { console.error(err); process.exit(101); return; }
 
@@ -714,20 +693,20 @@ function handleConfig(err, config) {
 }
 
 function parseConfig(err, text) {
-
-  console.info("");
-  console.info(verstr.join(' '));
-
   try {
     state._clientConfig = JSON.parse(text || '{}');
   } catch(e1) {
     try {
       state._clientConfig = YAML.safeLoad(text || '{}');
     } catch(e2) {
-      console.error(e1.message);
+      try {
-      console.error(e2.message);
+        state._clientConfig = TOML.parse(text || '');
-      process.exit(1);
+      } catch(e3) {
-      return;
+        console.error(e1.message);
+        console.error(e2.message);
+        process.exit(1);
+        return;
+      }
     }
   }
 
@@ -790,6 +769,9 @@ var parsers = {
       answers[parts[0]] = parts[1];
     });
 
+    if (answers.relay) {
+      console.info("using --relay " + answers.relay);
+    }
     // things that aren't straight-forward copy-over
     if (!answers.advanced && !answers.relay) {
       answers.relay = 'telebit.cloud';
@@ -830,6 +812,6 @@ var parsers = {
   }
 };
 
-require('fs').readFile(confpath, 'utf8', parseConfig);
+fs.readFile(confpath, 'utf8', parseConfig);
 
 }());

bin/telebit.js

@@ -2,11 +2,35 @@
 (function () {
 'use strict';
 
+//
 // node telebit daemon arg1 arg2
+//
 if ('daemon' === process.argv[2]) {
   require('./telebitd.js');
-} else {
+  return;
-  require('./telebit-remote.js');
 }
 
+//
+// sclient proxies
+//
+if ('sclient' === process.argv[2]) {
+  process.argv.splice(1,1);
+  return;
+}
+if ('rsync' === process.argv[2]) {
+  require('sclient/bin/sclient.js');
+  return;
+}
+if ('ssh' === process.argv[2] && /[\w-]+\.[a-z]{2,}/i.test(process.argv[3])) {
+  process.argv.splice(1,1,'sclient');
+  process.argv.splice(2,1,'ssh');
+  require('sclient/bin/sclient.js');
+  return;
+}
+
+//
+// telebit remote
+//
+require('./telebit-remote.js');
+
 }());

bin/telebitd.js

@@ -2,6 +2,13 @@
 (function () {
 'use strict';
 
+var PromiseA;
+try {
+  PromiseA = require('bluebird');
+} catch(e) {
+  PromiseA = global.Promise;
+}
+
 var pkg = require('../package.json');
 
 var url = require('url');
@@ -10,11 +17,16 @@ var os = require('os');
 var fs = require('fs');
 var common = require('../lib/cli-common.js');
 var http = require('http');
+var TOML = require('toml');
 var YAML = require('js-yaml');
 var recase = require('recase').create({});
 var camelCopy = recase.camelCopy.bind(recase);
 var snakeCopy = recase.snakeCopy.bind(recase);
-var state = { homedir: os.homedir(), servernames: {}, ports: {} };
+var TPLS = TOML.parse(fs.readFileSync(path.join(__dirname, "../lib/en-us.toml"), 'utf8'));
+
+var TelebitRemote = require('../').TelebitRemote;
+
+var state = { homedir: os.homedir(), servernames: {}, ports: {}, keepAlive: { state: false } };
 
 var argv = process.argv.slice(2);
 
@@ -32,20 +44,7 @@ if (-1 !== confIndex) {
 var cancelUpdater = require('../lib/updater')(pkg);
 
 function help() {
-  console.info('');
+  console.info(TPLS.daemon.help.main.replace(/{version}/g, pkg.version));
-  console.info('Telebit Daemon v' + pkg.version);
-  console.info('');
-  console.info('Usage:');
-  console.info('');
-  console.info('\ttelebitd --config <path>');
-  console.info('\tex: telebitd --config ~/.config/telebit/telebitd.yml');
-  console.info('');
-  console.info('');
-  console.info('Config:');
-  console.info('');
-  console.info('\tSee https://git.coolaj86.com/coolaj86/telebit.js');
-  console.info('');
-  console.info('');
 }
 
 var verstr = [ pkg.name + ' daemon v' + pkg.version ];
@@ -66,21 +65,19 @@ if (!confpath || /^--/.test(confpath)) {
   help();
   process.exit(1);
 }
-var tokenpath = path.join(path.dirname(confpath), 'access_token.txt');
+
+state._confpath = confpath;
+var tokenpath = path.join(path.dirname(state._confpath), 'access_token.txt');
 var token;
 try {
   token = fs.readFileSync(tokenpath, 'ascii').trim();
-  console.log('[DEBUG] access_token', typeof token, token);
+  //console.log('[DEBUG] access_token', typeof token, token);
 } catch(e) {
   // ignore
 }
 var controlServer;
-var tun;
+var myRemote;
 
-var controllers = {};
-function saveConfig(cb) {
-  fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), cb);
-}
 function getServername(servernames, sub) {
   if (state.servernames[sub]) {
     return sub;
@@ -105,6 +102,11 @@ function getServername(servernames, sub) {
     }
   })[0];
 }
+
+function saveConfig(cb) {
+  fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), cb);
+}
+var controllers = {};
 controllers.http = function (req, res, opts) {
   function getAppname(pathname) {
     // port number
@@ -352,22 +354,19 @@ function serveControlsHelper() {
       res.end(JSON.stringify(dumpy));
     }
 
-    if (/\b(config)\b/.test(opts.pathname) && /get/i.test(req.method)) {
+    function getConfigOnly() {
       var resp = JSON.parse(JSON.stringify(state.config));
       resp.version = pkg.version;
       res.setHeader('Content-Type', 'application/json');
       res.end(JSON.stringify(resp));
-      return;
     }
 
     //
     // without proper config
     //
-    function saveAndReport(err, _tun) {
+    function saveAndReport() {
       console.log('[DEBUG] saveAndReport config write', confpath);
       console.log(YAML.safeDump(snakeCopy(state.config)));
-      if (err) { throw err; }
-      tun = _tun;
       fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), function (err) {
         if (err) {
           res.statusCode = 500;
@@ -380,7 +379,8 @@ function serveControlsHelper() {
         listSuccess();
       });
     }
-    if (/\b(init|config)\b/.test(opts.pathname)) {
+
+    function initOrConfig() {
       var conf = {};
       if (!opts.body) {
         res.statusCode = 422;
@@ -458,7 +458,7 @@ function serveControlsHelper() {
       }
 
       if (!state.config.relay || !state.config.email || !state.config.agreeTos) {
-        console.log('aborting for some reason');
+        console.warn('missing config');
         res.statusCode = 400;
 
         res.setHeader('Content-Type', 'application/json');
@@ -474,66 +474,42 @@ function serveControlsHelper() {
         return;
       }
 
-      if (tun) {
+      // init also means enable
-        console.log('ending existing tunnel, starting anew');
+      delete state.config.disable;
-        tun.end(function () {
+      safeStartTelebitRemote(true).then(saveAndReport).catch(handleError);
-          console.log('success ending');
-          rawTunnel(saveAndReport);
-        });
-        tun = null;
-        setTimeout(function () {
-          if (!tun) {
-            console.log('failed to end, but starting anyway');
-            rawTunnel(saveAndReport);
-          }
-        }, 3000);
-      } else {
-        console.log('no tunnel, starting anew');
-        rawTunnel(saveAndReport);
-      }
-      return;
     }
 
-    if (/restart/.test(opts.pathname)) {
+    function restart() {
-      tun.end();
+      // failsafe
-      res.setHeader('Content-Type', 'application/json');
+      setTimeout(function () {
-      res.end(JSON.stringify({ success: true }));
+        res.setHeader('Content-Type', 'application/json');
+        res.end(JSON.stringify({ success: true }));
+        setTimeout(function () {
+          process.exit(33);
+        }, 500);
+      }, 5 * 1000);
+
+      if (myRemote) { myRemote.end(); }
       controlServer.close(function () {
-        // TODO closeAll other things
+        res.setHeader('Content-Type', 'application/json');
-        process.nextTick(function () {
+        res.end(JSON.stringify({ success: true }));
+
+        setTimeout(function () {
           // system daemon will restart the process
           process.exit(22); // use non-success exit code
-        });
+        }, 500);
       });
-      return;
     }
 
-    //
+    function invalidConfig() {
-    // Check for proper config
-    //
-    if (!state.config.relay || !state.config.email || !state.config.agreeTos) {
       res.statusCode = 400;
       res.setHeader('Content-Type', 'application/json');
       res.end(JSON.stringify({
         error: { code: "E_CONFIG", message: "Invalid config file. Please run 'telebit init'" }
       }));
-      return;
     }
 
-    //
+    function saveAndCommit() {
-    // With proper config
-    //
-    if (/http/.test(opts.pathname)) {
-      controllers.http(req, res, opts);
-      return;
-    }
-
-    if (/tcp/.test(opts.pathname)) {
-      controllers.tcp(req, res, opts);
-      return;
-    }
-
-    if (/save|commit/.test(opts.pathname)) {
       state.config.servernames = state.servernames;
       state.config.ports = state.ports;
       fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), function (err) {
@@ -547,69 +523,114 @@ function serveControlsHelper() {
         }
         listSuccess();
       });
-      return;
     }
 
-    if (/ssh/.test(opts.pathname)) {
+    function handleError(err) {
-      controllers.ssh(req, res, opts);
+      res.statusCode = 500;
-      return;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(JSON.stringify({
+        error: { message: err.message, code: err.code }
+      }));
     }
 
-    if (/enable/.test(opts.pathname)) {
+    function enable() {
       delete state.config.disable;// = undefined;
-      if (tun) {
+      state.keepAlive.state = true;
-        listSuccess();
+
-        return;
+      fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), function (err) {
-      }
+        if (err) {
-      rawTunnel(function (err, _tun) {
+          err.message = "Could not save config file. Perhaps you're user doesn't have permission?";
-        if (err) { throw err; }
+          handleError(err);
-        tun = _tun;
+          return;
-        fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), function (err) {
+        }
-          if (err) {
+        // TODO XXX myRemote.active
-            res.statusCode = 500;
+        if (myRemote) {
-            res.setHeader('Content-Type', 'application/json');
-            res.end(JSON.stringify({
-              error: { message: "Could not save config file. Perhaps you're user doesn't have permission?" }
-            }));
-            return;
-          }
           listSuccess();
-        });
+          return;
+        }
+        safeStartTelebitRemote(true).then(listSuccess).catch(handleError);
       });
-      return;
     }
 
-    if (/disable/.test(opts.pathname)) {
+    function disable() {
       state.config.disable = true;
-      if (tun) { tun.end(); tun = null; }
+      state.keepAlive.state = false;
+
+      if (myRemote) { myRemote.end(); myRemote = null; }
       fs.writeFile(confpath, YAML.safeDump(snakeCopy(state.config)), function (err) {
         res.setHeader('Content-Type', 'application/json');
         if (err) {
-          res.statusCode = 500;
+          err.message = "Could not save config file. Perhaps you're user doesn't have permission?";
-          res.end(JSON.stringify({
+          handleError(err);
-            "error":{"message":"Could not save config file. Perhaps you're not running as root?"}
-          }));
           return;
         }
         res.end('{"success":true}');
       });
-      return;
     }
 
-    if (/status/.test(opts.pathname)) {
+    function getStatus() {
       res.setHeader('Content-Type', 'application/json');
       res.end(JSON.stringify(
         { status: (state.config.disable ? 'disabled' : 'enabled')
         , ready: ((state.config.relay && (state.config.token || state.config.agreeTos)) ? true : false)
-        , active: !!tun
+        , active: !!myRemote
         , connected: 'maybe (todo)'
         , version: pkg.version
         , servernames: state.servernames
         }
       ));
-      return;
     }
 
+    if (/\b(config)\b/.test(opts.pathname) && /get/i.test(req.method)) {
+      getConfigOnly();
+      return;
+    }
+    if (/\b(init|config)\b/.test(opts.pathname)) {
+      initOrConfig();
+      return;
+    }
+    if (/restart/.test(opts.pathname)) {
+      restart();
+      return;
+    }
+    //
+    // Check for proper config
+    //
+    if (!state.config.relay || !state.config.email || !state.config.agreeTos) {
+      invalidConfig();
+      return;
+    }
+    //
+    // With proper config
+    //
+    if (/http/.test(opts.pathname)) {
+      controllers.http(req, res, opts);
+      return;
+    }
+    if (/tcp/.test(opts.pathname)) {
+      controllers.tcp(req, res, opts);
+      return;
+    }
+    if (/save|commit/.test(opts.pathname)) {
+      saveAndCommit();
+      return;
+    }
+    if (/ssh/.test(opts.pathname)) {
+      controllers.ssh(req, res, opts);
+      return;
+    }
+    if (/enable/.test(opts.pathname)) {
+      enable();
+      return;
+    }
+    if (/disable/.test(opts.pathname)) {
+      disable();
+      return;
+    }
+    if (/status/.test(opts.pathname)) {
+      getStatus();
+      return;
+    }
     if (/list/.test(opts.pathname)) {
       listSuccess();
       return;
@@ -618,6 +639,7 @@ function serveControlsHelper() {
     res.setHeader('Content-Type', 'application/json');
     res.end(JSON.stringify({"error":{"message":"unrecognized rpc"}}));
   });
+
   if (fs.existsSync(state._ipc.path)) {
     fs.unlinkSync(state._ipc.path);
   }
@@ -653,27 +675,23 @@ function serveControlsHelper() {
 }
 
 function serveControls() {
+  serveControlsHelper();
+
   if (state.config.disable) {
     console.info("[info] starting disabled");
     return;
   }
 
-  if (state.config.relay && (state.config.token || state.config.pretoken)) {
+  // This will remain in a disconnect state and wait for an init
-    console.info("[info] connecting with stored token");
+  if (!(state.config.relay && (state.config.token || state.config.pretoken))) {
-    rawTunnel(function (err, _tun) {
-      if (err) { throw err; }
-      if (_tun) { tun = _tun; }
-      setTimeout(function () {
-        // TODO attach handler to tunnel
-        serveControlsHelper();
-      }, 150);
-    });
-    return;
-  } else {
     console.info("[info] waiting for init/authentication (missing relay and/or token)");
+    return;
   }
 
-  serveControlsHelper();
+  console.info("[info] connecting with stored token");
+  return safeStartTelebitRemote().catch(function (/*err*/) {
+    // ignore, it'll keep looping anyway
+  });
 }
 
 function parseConfig(err, text) {
@@ -735,95 +753,258 @@ function parseConfig(err, text) {
   }
 }
 
-function rawTunnel(rawCb) {
+function approveDomains(opts, certs, cb) {
-  if (state.config.disable || !state.config.relay || !(state.config.token || state.config.agreeTos)) {
+  // Even though it's being tunneled by a trusted source
-    rawCb(null, null);
+  // we need to make sure we don't get rate-limit spammed
+  // with wildcard domains
+  // TODO: finish implementing dynamic dns for wildcard certs
+  if (getServername(state.servernames, opts.domains[0])) {
+    opts.email = state.greenlockConf.email || state.config.email;
+    opts.agreeTos = state.greenlockConf.agree || state.greenlockConf.agreeTos || state.config.agreeTos;
+    cb(null, { options: opts, certs: certs });
     return;
   }
 
+  cb(new Error("servername not found in allowed list"));
+}
+
+function greenlockHelper(state) {
+  // TODO Check undefined vs false for greenlock config
+  state.greenlockConf = state.config.greenlock || {};
+  state.greenlockConfig = {
+    version: state.greenlockConf.version || 'draft-11'
+  , server: state.greenlockConf.server || 'https://acme-v02.api.letsencrypt.org/directory'
+  , communityMember: state.greenlockConf.communityMember || state.config.communityMember
+  , telemetry: state.greenlockConf.telemetry || state.config.telemetry
+  , configDir: state.greenlockConf.configDir
+      || (state.config.root && path.join(state.config.root, 'etc/acme'))
+      || path.join(os.homedir(), '.config/telebit/acme')
+  // TODO, store: require(state.greenlockConf.store.name || 'le-store-certbot').create(state.greenlockConf.store.options || {})
+  , approveDomains: approveDomains
+  };
+  state.insecure = state.config.relay_ignore_invalid_certificates;
+}
+
+function promiseTimeout(t) {
+  return new PromiseA(function (resolve) {
+    setTimeout(resolve, t);
+  });
+}
+
+var promiseWss = PromiseA.promisify(function (state, fn) {
+  return common.api.wss(state, fn);
+});
+
+var trPromise;
+function safeStartTelebitRemote(forceOn) {
+  // whatever is currently going will not restart
+  state.keepAlive.state = false;
+  if (trPromise && !forceOn) { return trPromise; }
+
+  // if something is running, this will kill it
+  // (TODO option to use known-good active instead of restarting)
+  // this won't restart either
+  trPromise = rawStartTelebitRemote(state.keepAlive);
+  trPromise.then(function () {
+    //console.log("I'm RIGHT HERE!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
+    state.keepAlive.state = true;
+    trPromise = null;
+  }).catch(function () {
+    //console.log("I FAILED US ALL!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
+    // this will restart
+    state.keepAlive = { state: true };
+    trPromise = rawStartTelebitRemote(state.keepAlive);
+    trPromise.then(function () {
+      trPromise = null;
+    }).catch(function () {
+      //console.log('DEBUG state.keepAlive turned off and remote quit');
+      trPromise = null;
+    });
+  });
+  return trPromise;
+}
+
+function rawStartTelebitRemote(keepAlive) {
+  var err;
+  var exiting = false;
+  var localRemote = myRemote;
+  myRemote = null;
+  if (localRemote) { /*console.log('DEBUG destroy() existing');*/ localRemote.destroy(); }
+
+  function safeReload(delay) {
+    if (exiting) {
+      // return a junk promise as the prior call
+      // already passed flow-control to the next promise
+      // (this is a second or delayed error or close event)
+      return PromiseA.resolve();
+    }
+    exiting = true;
+    // TODO state.keepAlive?
+    return promiseTimeout(delay).then(function () {
+      return rawStartTelebitRemote(keepAlive);
+    });
+  }
+
+  if (state.config.disable) {
+    //console.log('DEBUG disabled or incapable');
+    err = new Error("connecting is disabled");
+    err.code = 'EDISABLED';
+    return PromiseA.reject(err);
+  }
+
+  if (!(state.config.token || state.config.agreeTos)) {
+    //console.log('DEBUG Must agreeTos to generate preauth');
+    err = new Error("Must either supply token (for auth) or agreeTos (for preauth)");
+    err.code = 'ENOAGREE';
+    return PromiseA.reject(err);
+  }
+
   state.relay = state.config.relay;
   if (!state.relay) {
-    rawCb(new Error("'" + state._confpath + "' is missing 'relay'"));
+    //console.log('DEBUG no relay');
-    return;
+    err = new Error("'" + state._confpath + "' is missing 'relay'");
+    err.code = 'ENORELAY';
+    return PromiseA.reject(err);
   }
 
+  // TODO: we need some form of pre-authorization before connecting,
+  // otherwise we'll get disconnected pretty quickly
   if (!(state.token || state.pretoken)) {
-    rawCb(null, null);
+    //console.log('DEBUG no token');
-    return;
+    err = new Error("no jwt token or preauthorization");
+    err.code = 'ENOAUTH';
+    return PromiseA.reject(err);
   }
 
-  if (tun) {
+  return PromiseA.resolve().then(function () {
-    rawCb(null, tun);
+    //console.log('DEBUG rawStartTelebitRemote');
-    return;
-  }
 
-  common.api.wss(state, function (err, wss) {
+    function startHelper() {
-    if (err) { rawCb(err); return; }
+      //console.log('DEBUG startHelper');
-    state.wss = wss;
+      greenlockHelper(state);
+      // Saves the token
+      // state.handlers.access_token({ jwt: token });
+      // Adds the token to the connection
+      // tun.append(token);
 
-    // Saves the token
+      //console.log("[DEBUG] token", typeof token, token);
-    // state.handlers.access_token({ jwt: token });
+      //state.sortingHat = state.config.sortingHat;
-    // Adds the token to the connection
+      // { relay, config, servernames, ports, sortingHat, net, insecure, token, handlers, greenlockConfig }
-    // tun.append(token);
 
-    state.greenlockConf = state.config.greenlock || {};
+      return new PromiseA(function (myResolve, myReject) {
-    state.sortingHat = state.config.sortingHat;
+        function reject(err) {
+          if (myReject) {
+            myReject(err);
+            myResolve = null;
+            myReject = null;
+          } else {
+            //console.log('DEBUG double rejection');
+          }
+        }
+        function resolve(val) {
+          //console.log('[DEBUG] pre-resolve');
+          if (myResolve) {
+            myResolve(val);
+            myResolve = null;
+            myReject = null;
+          } else {
+            //console.log('DEBUG double resolution');
+          }
+        }
 
-    // TODO sortingHat.print(); ?
+        function onConnect() {
-    // TODO Check undefined vs false for greenlock config
+          console.info('[connect] relay established');
-    var remote = require('../');
+          myRemote.removeListener('error', onConnectError);
-
+          myRemote.once('error', function () {
-    state.greenlockConfig = {
+            if (!keepAlive.state) {
-      version: state.greenlockConf.version || 'draft-11'
+              reject(err);
-    , server: state.greenlockConf.server || 'https://acme-v02.api.letsencrypt.org/directory'
+              return;
-    , communityMember: state.greenlockConf.communityMember || state.config.communityMember
+            }
-    , telemetry: state.greenlockConf.telemetry || state.config.telemetry
+            retryLoop();
-    , configDir: state.greenlockConf.configDir
+          });
-        || (state.config.root && path.join(state.config.root, 'etc/acme'))
+          resolve(myRemote);
-        || path.join(os.homedir(), '.config/telebit/acme')
-    // TODO, store: require(state.greenlockConf.store.name || 'le-store-certbot').create(state.greenlockConf.store.options || {})
-    , approveDomains: function (opts, certs, cb) {
-        // Certs being renewed are listed in certs.altnames
-        if (certs) {
-          opts.domains = certs.altnames;
-          cb(null, { options: opts, certs: certs });
           return;
         }
 
-        // Even though it's being tunneled by a trusted source
+        function onConnectError(err) {
-        // we need to make sure we don't get rate-limit spammed
+          myRemote = null;
-        // with wildcard domains
+          // Likely causes:
-        // TODO: finish implementing dynamic dns for wildcard certs
+          //   * DNS lookup failed (no Internet)
-        if (getServername(state.servernames, opts.domains[0])) {
+          //   * Rejected (bad authn)
-          opts.email = state.greenlockConf.email || state.config.email;
+          if ('ENOTFOUND' === err.code) {
-          opts.agreeTos = state.greenlockConf.agree || state.greenlockConf.agreeTos || state.config.agreeTos;
+            // DNS issue, probably network is disconnected
-          cb(null, { options: opts, certs: certs });
+            if (!keepAlive.state) {
+              reject(err);
+              return;
+            }
+            console.warn('[Warn] onConnectError: network error, will retry', err);
+            safeReload(10 * 1000).then(resolve).catch(reject);
+            return;
+          }
+          console.error('[Error] onConnectError: no retry (possibly bad auth)', err);
+          reject(err);
           return;
         }
 
-        //cb(new Error("servername not found in allowed list"));
+        function retryLoop() {
+          console.warn('[Warn] disconnected. Will retry?', keepAlive.state);
+          if (keepAlive.state) {
+            safeReload(10 * 1000).then(resolve).catch(reject);
+          }
+        }
+
+        myRemote = TelebitRemote.createConnection({
+          relay: state.relay
+        , wss: state.wss
+        , config: state.config
+        , otp: state.otp
+        , sortingHat: state.config.sortingHat
+        , net: state.net
+        , insecure: state.insecure
+        , token: state.token || state.pretoken // instance
+        , servernames: state.servernames
+        , ports: state.ports
+        , handlers: state.handlers
+        , greenlockConfig: state.greenlockConfig
+        }, onConnect);
+
+        myRemote.once('error', onConnectError);
+        myRemote.once('close', retryLoop);
+        myRemote.on('grant', state.handlers.grant);
+        myRemote.on('access_token', state.handlers.access_token);
+      });
+    }
+
+    if (state.wss) {
+      return startHelper();
+    }
+
+    // get the wss url
+    function retryWssLoop(err) {
+      if (!keepAlive.state) {
+        return PromiseA.reject(err);
       }
-    };
-    state.insecure = state.config.relay_ignore_invalid_certificates;
-    // { relay, config, servernames, ports, sortingHat, net, insecure, token, handlers, greenlockConfig }
 
-    console.log("[DEBUG] token", typeof token, token);
+      myRemote = null;
-    tun = remote.connect({
+      if (!err) {
-      relay: state.relay
+        return startHelper();
-    , wss: state.wss
+      }
-    , config: state.config
+
-    , otp: state.otp
+      if ('ENOTFOUND' === err.code) {
-    , sortingHat: state.sortingHat
+        // The internet is disconnected
-    , net: state.net
+        // try again, and again, and again
-    , insecure: state.insecure
+        return safeReload(2 * 1000);
-    , token: state.token || state.pretoken // instance
+      }
-    , servernames: state.servernames
+
-    , ports: state.ports
+      return PromiseA.reject(err);
-    , handlers: state.handlers
+    }
-    , greenlockConfig: state.greenlockConfig
+
+    return promiseWss(state).then(function (wss) {
+      state.wss = wss;
+      return startHelper();
+    }).catch(function (err) {
+      return retryWssLoop(err);
     });
-
-    rawCb(null, tun);
   });
 }
 
@@ -885,14 +1066,17 @@ state.handlers = {
 };
 
 function sigHandler() {
+  process.removeListener('SIGINT', sigHandler);
+
   console.info('Received kill signal. Attempting to exit cleanly...');
+  state.keepAlive.state = false;
 
   // We want to handle cleanup properly unless something is broken in our cleanup process
   // that prevents us from exitting, in which case we want the user to be able to send
   // the signal again and exit the way it normally would.
-  process.removeListener('SIGINT', sigHandler);
+  if (myRemote) {
-  if (tun) {
+    myRemote.end();
-    tun.end();
+    myRemote = null;
   }
   if (controlServer) {
     controlServer.close();

lib/en-us.toml

@@ -0,0 +1,87 @@
+[remote]
+[remote.help]
+
+main = "telebit remote v{version}
+
+Telebit is a tool for helping you access your devices and share your stuff.
+
+Usage:
+
+        telebit [flags] <command> [arguments]
+
+The flags are:
+
+        --config <path> specify config file (default is ~/.config/telebit/telebit.yml)
+
+The commands are:
+
+        status        show status and configuration info
+        enable        turn on remote access and sharing
+        disable       turn off remote access and sharing
+
+        http          access files, folders, and local apps via https (secure)
+        ssh (local)   enable remote access to this device with ssh-over-https
+        ssh (remote)  access devices via ssh-over-https (telebit, stunnel, openssl, etc)
+        tcp           forward tcp locally
+
+Use \"telebit help [command]\" for more information about a command.
+
+Additional help topics:
+
+        config        config file format and settings
+        ssh (proxy)   ssh over https and proxy commands
+        ftp           secure ftp file transfer between devices
+        rsync         rsync over https and proxy commands
+        vpn           home network access and private web browsing via socks5
+        daemon        telebit daemon secure background service
+        relay         telebit secure relay, hosted, and self-hosting options
+
+Copyright 2015-2018 https://telebit.cloud MPL-2.0 Licensed"
+
+http = "usage: telebit http <path/port/none> [subdomain]
+
+'telebit http' is the fastest way to share files, folders, and local apps.
+
+        http <DIR> [subdomain]          serve a file, folder, or node express app
+        ex: telebit http ~/Public pub   securely host ~/Public as pub.johndoe.telebit.io
+
+        http <PORT> [subdomain]         forward all https traffic to a local app
+        ex: telebit http 3000 app       publicize localhost:3000 as app.johndoe.telebit.io
+
+        http none [subdomain]           remove secure http access for (any or all) subdomain(s)
+        ex: telebit http none           remove all https access
+"
+
+tcp = "
+usage: telebit tcp <path/port/none>
+
+'telebit tcp' is provided for seemless connectivity to legacy apps
+
+        tcp <local> [remote]          forward tcp to <local> from <remote>
+        ex: telebit tcp 5050 6565     forward tcp port 6565 locally to port 5050
+
+        tcp <path> [remote]           show ftp-style directory listing
+        ex: telebit tcp ~/Public      show listing of ~/Public
+
+        tcp none [remote]             disable tcp access for [remote] port
+        ex: telebit tcp none 6565     remove access to port 6565
+
+See also sclient <https://telebit.cloud/sclient> for connecting to legacy apps
+with telebit-upscaled secure https access.
+"
+
+[daemon]
+[daemon.help]
+main = "telebit daemon v{version}
+
+Usage:
+
+        telebit daemon --config <path>
+        ex: telebit daemon --config ~/.config/telebit/telebitd.yml
+
+Additional help topics:
+
+        config    config file format and settings
+        remote    telebit cli remote control
+
+Copyright 2015-2018 https://telebit.cloud MPL-2.0 Licensed"

lib/remote.js

@@ -11,6 +11,7 @@ var WebSocket = require('ws');
 var sni = require('sni');
 var Packer = require('proxy-packer');
 var os = require('os');
+var EventEmitter = require('events').EventEmitter;
 
 function timeoutPromise(duration) {
   return new PromiseA(function (resolve) {
@@ -18,15 +19,24 @@ function timeoutPromise(duration) {
   });
 }
 
-function _connect(state) {
+function TelebitRemote(state) {
   // jshint latedef:false
-  var defaultHttpTimeout = (2 * 60);
+
-  var activityTimeout = state.activityTimeout || (defaultHttpTimeout - 5) * 1000;
+  if (!(this instanceof TelebitRemote)) {
+    return new TelebitRemote(state);
+  }
+  EventEmitter.call(this);
+  var me = this;
+  var priv = {};
+
+  //var defaultHttpTimeout = (2 * 60);
+  //var activityTimeout = state.activityTimeout || (defaultHttpTimeout - 5) * 1000;
+  var activityTimeout = 6 * 1000;
   var pongTimeout = state.pongTimeout || 10*1000;
   // Allow the tunnel client to be created with no token. This will prevent the connection from
   // being established initialy and allows the caller to use `.append` for the first token so
   // they can get a promise that will provide feedback about invalid tokens.
-  var tokens = [];
+  priv.tokens = [];
   var auth;
   if(!state.sortingHat) {
     state.sortingHat = "./sorting-hat.js";
@@ -35,7 +45,7 @@ function _connect(state) {
     if ('undefined' === state.token) {
       throw new Error("passed string 'undefined' as token");
     }
-    tokens.push(state.token);
+    priv.tokens.push(state.token);
   }
 
   var wstunneler;
@@ -43,19 +53,24 @@ function _connect(state) {
   var authsent = false;
   var initialConnect = true;
 
-  var localclients = {};
+  priv.localclients = {};
   var pausedClients = [];
   var clientHandlers = {
     add: function (conn, cid, tun) {
-      localclients[cid] = conn;
+      priv.localclients[cid] = conn;
-      console.info("[connect] new client '" + cid + "' for '" + tun.name + ":" + tun.serviceport + "' "
+      console.info("[connect] new client '" + tun.name + ":" + tun.serviceport + "' for  '" + cid + "'"
         + "(" + clientHandlers.count() + " clients)");
 
       conn.tunnelCid = cid;
-      conn.tunnelRead = tun.data.byteLength;
+      if (tun.data) {
-      conn.tunnelWritten    = 0;
+        conn.tunnelRead = tun.data.byteLength;
+      } else {
+        conn.tunnelRead = 0;
+      }
+      conn.tunnelWritten = 0;
 
       conn.on('data', function onLocalData(chunk) {
+        //var chunk = conn.read();
         if (conn.tunnelClosing) {
           console.warn("[onLocalData] received data for '"+cid+"' over socket after connection was ended");
           return;
@@ -67,8 +82,10 @@ function _connect(state) {
         // down the data we are getting to send over. We also want to pause all active connections
         // if any connections are paused to make things more fair so one connection doesn't get
         // stuff waiting for all other connections to finish because it tried writing near the border.
-        var bufSize = wsHandlers.sendMessage(Packer.pack(tun, chunk));
+        var bufSize = sendMessage(Packer.packHeader(tun, chunk));
-        if (pausedClients.length || bufSize > 1024*1024) {
+        // Sending 2 messages instead of copying the buffer
+        var bufSize2 = sendMessage(chunk);
+        if (pausedClients.length || (bufSize + bufSize2) > 1024*1024) {
           // console.log('[onLocalData] paused connection', cid, 'to allow websocket to catch up');
           conn.pause();
           pausedClients.push(conn);
@@ -77,32 +94,33 @@ function _connect(state) {
 
       var sentEnd = false;
       conn.on('end', function onLocalEnd() {
-        console.info("[onLocalEnd] connection '" + cid + "' ended, will probably close soon");
+        //console.info("[onLocalEnd] connection '" + cid + "' ended, will probably close soon");
         conn.tunnelClosing = true;
         if (!sentEnd) {
-          wsHandlers.sendMessage(Packer.pack(tun, null, 'end'));
+          sendMessage(Packer.packHeader(tun, null, 'end'));
           sentEnd = true;
         }
       });
       conn.on('error', function onLocalError(err) {
         console.info("[onLocalError] connection '" + cid + "' errored:", err);
         if (!sentEnd) {
-          wsHandlers.sendMessage(Packer.pack(tun, {message: err.message, code: err.code}, 'error'));
+          var packBody = true;
+          sendMessage(Packer.packHeader(tun, {message: err.message, code: err.code}, 'error', packBody));
           sentEnd = true;
         }
       });
       conn.on('close', function onLocalClose(hadErr) {
-        delete localclients[cid];
+        delete priv.localclients[cid];
         console.log('[onLocalClose] closed "' + cid + '" read:'+conn.tunnelRead+', wrote:'+conn.tunnelWritten+' (' + clientHandlers.count() + ' clients)');
         if (!sentEnd) {
-          wsHandlers.sendMessage(Packer.pack(tun, null, hadErr && 'error' || 'end'));
+          sendMessage(Packer.packHeader(tun, null, hadErr && 'error' || 'end'));
           sentEnd = true;
         }
       });
     }
 
   , write: function (cid, opts) {
-      var conn = localclients[cid];
+      var conn = priv.localclients[cid];
       if (!conn) {
         return false;
       }
@@ -119,11 +137,13 @@ function _connect(state) {
       conn.tunnelRead += opts.data.byteLength;
 
       if (!conn.remotePaused && conn.bufferSize > 1024*1024) {
-        wsHandlers.sendMessage(Packer.pack(opts, conn.tunnelRead, 'pause'));
+        var packBody = true;
+        sendMessage(Packer.packHeader(opts, conn.tunnelRead, 'pause', packBody));
         conn.remotePaused = true;
 
         conn.once('drain', function () {
-          wsHandlers.sendMessage(Packer.pack(opts, conn.tunnelRead, 'resume'));
+          var packBody = true;
+          sendMessage(Packer.packHeader(opts, conn.tunnelRead, 'resume', packBody));
           conn.remotePaused = false;
         });
       }
@@ -131,13 +151,13 @@ function _connect(state) {
     }
 
   , closeSingle: function (cid) {
-      if (!localclients[cid]) {
+      if (!priv.localclients[cid]) {
         return;
       }
 
-      console.log('[closeSingle]', cid);
+      //console.log('[closeSingle]', cid);
       PromiseA.resolve().then(function () {
-        var conn = localclients[cid];
+        var conn = priv.localclients[cid];
         conn.tunnelClosing = true;
         conn.end();
 
@@ -155,40 +175,49 @@ function _connect(state) {
           });
         });
       }).then(function () {
-        if (localclients[cid]) {
+        if (priv.localclients[cid]) {
           console.warn('[closeSingle]', cid, 'connection still present after calling `end`');
-          localclients[cid].destroy();
+          priv.localclients[cid].destroy();
           return timeoutPromise(500);
         }
       }).then(function () {
-        if (localclients[cid]) {
+        if (priv.localclients[cid]) {
           console.error('[closeSingle]', cid, 'connection still present after calling `destroy`');
-          delete localclients[cid];
+          delete priv.localclients[cid];
         }
       }).catch(function (err) {
         console.error('[closeSingle] failed to close connection', cid, err.toString());
-        delete localclients[cid];
+        delete priv.localclients[cid];
       });
     }
   , closeAll: function () {
       console.log('[closeAll]');
-      Object.keys(localclients).forEach(function (cid) {
+      Object.keys(priv.localclients).forEach(function (cid) {
         clientHandlers.closeSingle(cid);
       });
     }
 
   , count: function () {
-      return Object.keys(localclients).length;
+      return Object.keys(priv.localclients).length;
     }
   };
 
   var pendingCommands = {};
+  function sendMessage(msg) {
+    // There is a chance that this occurred after the websocket was told to close
+    // and before it finished, in which case we don't need to log the error.
+    if (wstunneler.readyState !== wstunneler.CLOSING) {
+        wstunneler.send(msg, {binary: true});
+        return wstunneler.bufferedAmount;
+    }
+  }
   function sendCommand(name) {
     var id = Math.ceil(1e9 * Math.random());
     var cmd = [id, name].concat(Array.prototype.slice.call(arguments, 1));
     if (state.debug) { console.log('[DEBUG] command sending', cmd); }
 
-    wsHandlers.sendMessage(Packer.pack(null, cmd, 'control'));
+    var packBody = true;
+    sendMessage(Packer.packHeader(null, cmd, 'control', packBody));
     setTimeout(function () {
       if (pendingCommands[id]) {
         console.warn('command', name, id, 'timed out');
@@ -211,24 +240,6 @@ function _connect(state) {
     });
   }
 
-  function sendAllTokens() {
-    if (auth) {
-      authsent = true;
-      sendCommand('auth', auth).catch(function (err) { console.error('1', err); });
-    }
-    tokens.forEach(function (jwtoken) {
-      if (state.debug) { console.log('[DEBUG] send token'); }
-      authsent = true;
-      sendCommand('add_token', jwtoken)
-        .catch(function (err) {
-          console.error('failed re-adding token', jwtoken, 'after reconnect', err);
-          // Not sure if we should do something like remove the token here. It worked
-          // once or it shouldn't have stayed in the list, so it's less certain why
-          // it would have failed here.
-        });
-    });
-  }
-
   function noHandler(cmd) {
     console.warn("[telebit] state.handlers['" + cmd[1] + "'] not set");
     console.warn(cmd[2]);
@@ -236,6 +247,23 @@ function _connect(state) {
 
   var connCallback;
 
+  function hyperPeek(tun) {
+    var m;
+    var str;
+    if (tun.data) {
+      if ('http' === tun.service) {
+        str = tun.data.toString();
+        m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im);
+        tun._name = tun._hostname = (m && m[1].toLowerCase() || '').split(':')[0];
+      }
+      else if ('https' === tun.service || 'tls' === tun.service) {
+        tun._name = tun._servername = sni(tun.data);
+      } else {
+        tun._name = '';
+      }
+    }
+  }
+
   var packerHandlers = {
     oncontrol: function (opts) {
       var cmd, err;
@@ -267,7 +295,21 @@ function _connect(state) {
 
       if (cmd[1] === 'hello') {
         if (state.debug) { console.log('[DEBUG] hello received'); }
-        sendAllTokens();
+        if (auth) {
+          authsent = true;
+          sendCommand('auth', auth).catch(function (err) { console.error('1', err); });
+        }
+        priv.tokens.forEach(function (jwtoken) {
+          if (state.debug) { console.log('[DEBUG] send token'); }
+          authsent = true;
+          sendCommand('add_token', jwtoken)
+            .catch(function (err) {
+              console.error('failed re-adding token', jwtoken, 'after reconnect', err);
+              // Not sure if we should do something like remove the token here. It worked
+              // once or it shouldn't have stayed in the list, so it's less certain why
+              // it would have failed here.
+            });
+        });
         if (connCallback) {
           connCallback();
         }
@@ -294,28 +336,19 @@ function _connect(state) {
         err = { message: 'unknown command "'+cmd[1]+'"', code: 'E_UNKNOWN_COMMAND' };
       }
 
-      wsHandlers.sendMessage(Packer.pack(null, [-cmd[0], err], 'control'));
+      var packBody = true;
+      sendMessage(Packer.packHeader(null, [-cmd[0], err], 'control', packBody));
     }
 
-  , onmessage: function (tun) {
+  , onconnection: function (tun) {
       var cid = tun._id = Packer.addrToId(tun);
-      var str;
-      var m;
 
-      if ('http' === tun.service) {
+      // this data should have been gathered already as part of the proxy protocol
-        str = tun.data.toString();
+      // but if it's available again here we can double check
-        m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im);
+      hyperPeek(tun);
-        tun._name = tun._hostname = (m && m[1].toLowerCase() || '').split(':')[0];
-      }
-      else if ('https' === tun.service || 'tls' === tun.service) {
-        tun._name = tun._servername = sni(tun.data);
-      } else {
-        tun._name = '';
-      }
 
-      if (clientHandlers.write(cid, tun)) { return; }
+      // TODO use readable streams instead
-
+      wstunneler._socket.pause();
-      wstunneler.pause();
       require(state.sortingHat).assign(state, tun, function (err, conn) {
         if (err) {
           err.message = err.message.replace(/:tun_id/, tun._id);
@@ -324,32 +357,46 @@ function _connect(state) {
         }
         clientHandlers.add(conn, cid, tun);
         if (tun.data) { conn.write(tun.data); }
-        wstunneler.resume();
+        wstunneler._socket.resume();
       });
     }
 
+  , onmessage: function (tun) {
+      var cid = tun._id = Packer.addrToId(tun);
+      var handled;
+
+      hyperPeek(tun);
+
+      handled = clientHandlers.write(cid, tun);
+
+      // quasi backwards compat
+      if (!handled) { console.log("[debug] did not get 'connection' event"); packerHandlers.onconnection(tun); }
+    }
+
   , onpause: function (opts) {
       var cid = Packer.addrToId(opts);
-      if (localclients[cid]) {
+      if (priv.localclients[cid]) {
-        console.log("[TunnelPause] pausing '"+cid+"', remote received", opts.data.toString(), 'of', localclients[cid].tunnelWritten, 'sent');
+        console.log("[TunnelPause] pausing '"+cid+"', remote received", opts.data.toString(), 'of', priv.localclients[cid].tunnelWritten, 'sent');
-        localclients[cid].manualPause = true;
+        priv.localclients[cid].manualPause = true;
-        localclients[cid].pause();
+        priv.localclients[cid].pause();
       } else {
         console.log('[TunnelPause] remote tried pausing finished connection', cid);
         // Often we have enough latency that we've finished sending before we're told to pause, so
         // don't worry about sending back errors, since we won't be sending data over anyway.
-        // wsHandlers.sendMessage(Packer.pack(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error'));
+        // var packBody = true;
+        // sendMessage(Packer.packHeader(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error', packBody));
       }
     }
   , onresume: function (opts) {
       var cid = Packer.addrToId(opts);
-      if (localclients[cid]) {
+      if (priv.localclients[cid]) {
-        console.log("[TunnelResume] resuming '"+cid+"', remote received", opts.data.toString(), 'of', localclients[cid].tunnelWritten, 'sent');
+        console.log("[TunnelResume] resuming '"+cid+"', remote received", opts.data.toString(), 'of', priv.localclients[cid].tunnelWritten, 'sent');
-        localclients[cid].manualPause = false;
+        priv.localclients[cid].manualPause = false;
-        localclients[cid].resume();
+        priv.localclients[cid].resume();
       } else {
         console.log('[TunnelResume] remote tried resuming finished connection', cid);
-        // wsHandlers.sendMessage(Packer.pack(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error'));
+        // var packBody = true;
+        // sendMessage(Packer.packHeader(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error', packBody));
       }
     }
 
@@ -366,56 +413,74 @@ function _connect(state) {
 
   , _onConnectError: function (cid, opts, err) {
       console.info("[_onConnectError] opening '" + cid + "' failed because " + err.message);
-      wsHandlers.sendMessage(Packer.pack(opts, null, 'error'));
+      sendMessage(Packer.packHeader(opts, null, 'error'));
     }
   };
 
-  var lastActivity;
+  priv.timeoutId = null;
-  var timeoutId;
+  priv.lastActivity = Date.now();
-  var wsHandlers = {
+  priv.refreshTimeout = function refreshTimeout() {
-    refreshTimeout: function () {
+    priv.lastActivity = Date.now();
-      lastActivity = Date.now();
+  };
+  priv.checkTimeout = function checkTimeout() {
+    if (!wstunneler) {
+      console.warn('checkTimeout called when websocket already closed');
+      return;
     }
-  , checkTimeout: function () {
+    // Determine how long the connection has been "silent", ie no activity.
-      if (!wstunneler) {
+    var silent = Date.now() - priv.lastActivity;
-        console.warn('checkTimeout called when websocket already closed');
-        return;
-      }
-      // Determine how long the connection has been "silent", ie no activity.
-      var silent = Date.now() - lastActivity;
 
-      // If we have had activity within the last activityTimeout then all we need to do is
+    // If we have had activity within the last activityTimeout then all we need to do is
-      // call this function again at the soonest time when the connection could be timed out.
+    // call this function again at the soonest time when the connection could be timed out.
-      if (silent < activityTimeout) {
+    if (silent < activityTimeout) {
-        timeoutId = setTimeout(wsHandlers.checkTimeout, activityTimeout-silent);
+      priv.timeoutId = setTimeout(priv.checkTimeout, activityTimeout-silent);
-      }
-
-      // Otherwise we check to see if the pong has also timed out, and if not we send a ping
-      // and call this function again when the pong will have timed out.
-      else if (silent < activityTimeout + pongTimeout) {
-        console.log('pinging tunnel server');
-        try {
-          wstunneler.ping();
-        } catch (err) {
-          console.warn('failed to ping tunnel server', err);
-        }
-        timeoutId = setTimeout(wsHandlers.checkTimeout, pongTimeout);
-      }
-
-      // Last case means the ping we sent before didn't get a response soon enough, so we
-      // need to close the websocket connection.
-      else {
-        console.log('connection timed out');
-        wstunneler.close(1000, 'connection timeout');
-      }
     }
 
-  , onOpen: function () {
+    // Otherwise we check to see if the pong has also timed out, and if not we send a ping
+    // and call this function again when the pong will have timed out.
+    else if (silent < activityTimeout + pongTimeout) {
+      //console.log('DEBUG: pinging tunnel server');
+      try {
+        wstunneler.ping();
+      } catch (err) {
+        console.warn('failed to ping tunnel server', err);
+      }
+      priv.timeoutId = setTimeout(priv.checkTimeout, pongTimeout);
+    }
+
+    // Last case means the ping we sent before didn't get a response soon enough, so we
+    // need to close the websocket connection.
+    else {
+      console.info('[info] closing due to connection timeout');
+      wstunneler.close(1000, 'connection timeout');
+    }
+  };
+
+  me.destroy = function destroy() {
+    console.info('[info] destroy()');
+    try {
+      //wstunneler.close(1000, 're-connect');
+      wstunneler._socket.destroy();
+    } catch(e) {
+      // ignore
+    }
+  };
+  me.connect = function connect() {
+    if (!priv.tokens.length && state.config.email) {
+      auth = TelebitRemote._tokenFromState(state);
+    }
+    priv.timeoutId = null;
+    var machine = Packer.create(packerHandlers);
+
+    console.info("[connect] '" + (state.wss || state.relay) + "'");
+    var tunnelUrl = (state.wss || state.relay).replace(/\/$/, '') + '/'; // + auth;
+    wstunneler = new WebSocket(tunnelUrl, { rejectUnauthorized: !state.insecure });
+    // XXXXXX
+    wstunneler.on('open', function () {
       console.info("[open] connected to '" + (state.wss || state.relay) + "'");
-      wsHandlers.refreshTimeout();
+      me.emit('connect');
-
+      priv.refreshTimeout();
-      timeoutId = setTimeout(wsHandlers.checkTimeout, activityTimeout);
+      priv.timeoutId = setTimeout(priv.checkTimeout, activityTimeout);
-
       wstunneler._socket.on('drain', function () {
         // the websocket library has it's own buffer apart from node's socket buffer, but that one
         // is much more difficult to watch, so we watch for the lower level buffer to drain and
@@ -432,22 +497,13 @@ function _connect(state) {
             conn.resume();
           }
         });
-
         pausedClients.length = 0;
       });
-
-      //Call either Open or Reconnect handlers.
-      if(state.handlers.onOpen && initialConnect) {
-        state.handlers.onOpen();
-      } else if (state.handlers.onReconnect && !initialConnect) {
-        state.handlers.onReconnect();
-      }
       initialConnect = false;
-    }
+    });
-
+    wstunneler.on('close', function () {
-  , onClose: function () {
+      console.info("[info] [closing] received close signal from relay");
-      clearTimeout(timeoutId);
+      clearTimeout(priv.timeoutId);
-      wstunneler = null;
       clientHandlers.closeAll();
 
       var error = new Error('websocket connection closed before response');
@@ -459,204 +515,69 @@ function _connect(state) {
         connCallback(error);
       }
 
-      if (!authenticated) {
+      me.emit('close');
-        if(state.handlers.onError) {
+    });
-          var err = new Error('Failed to connect on first attempt... check authentication');
+    wstunneler.on('error', function (err) {
-          state.handlers.onError(err);
+      me.emit('error', err);
-        }
+    });
-        if(state.handlers.onClose) {
-          state.handlers.onClose();
-        }
-        console.info('[close] failed on first attempt... check authentication.');
-        timeoutId = null;
-      }
-      else if (tokens.length) {
-        if(state.handlers.onDisconnect) {
-          state.handlers.onDisconnect();
-        }
-        console.info('[retry] disconnected and waiting...');
-        timeoutId = setTimeout(connect, 5000);
-      } else {
-        if(state.handlers.onClose) {
-          state.handlers.onClose();
-        }
-      }
-    }
-
-  , onError: function (err) {
-      if ('ENOTFOUND' === err.code) {
-        // DNS issue, probably network is disconnected
-        timeoutId = setTimeout(connect, 90 * 1000);
-        return;
-      }
-      console.error("[tunnel error] " + err.message);
-      console.error(err);
-      if (connCallback) {
-        connCallback(err);
-      }
-    }
-
-  , sendMessage: function (msg) {
-      if (wstunneler) {
-        try {
-          wstunneler.send(msg, {binary: true});
-          return wstunneler.bufferedAmount;
-        } catch (err) {
-          // There is a chance that this occurred after the websocket was told to close
-          // and before it finished, in which case we don't need to log the error.
-          if (wstunneler.readyState !== wstunneler.CLOSING) {
-            console.warn('[sendMessage] error sending websocket message', err);
-          }
-        }
-      }
-    }
-  };
-
-  function connect() {
-    if (wstunneler) {
-      console.warn('attempted to connect with connection already active');
-      return;
-    }
-    if (!tokens.length) {
-      if (state.config.email) {
-        auth = {
-          subject: state.config.email
-        , subject_scheme: 'mailto'
-          // TODO create domains list earlier
-        , scope: Object.keys(state.config.servernames || {}).join(',')
-        , otp: state.otp
-        , hostname: os.hostname()
-          // Used for User-Agent
-        , os_type: os.type()
-        , os_platform: os.platform()
-        , os_release: os.release()
-        , os_arch: os.arch()
-        };
-      }
-    }
-    timeoutId = null;
-    var machine = Packer.create(packerHandlers);
-
-    console.info("[connect] '" + (state.wss || state.relay) + "'");
-    var tunnelUrl = (state.wss || state.relay).replace(/\/$/, '') + '/'; // + auth;
-    wstunneler = new WebSocket(tunnelUrl, { rejectUnauthorized: !state.insecure });
-    wstunneler.on('open', wsHandlers.onOpen);
-    wstunneler.on('close', wsHandlers.onClose);
-    wstunneler.on('error', wsHandlers.onError);
 
     // Our library will automatically handle sending the pong respose to ping requests.
-    wstunneler.on('ping', wsHandlers.refreshTimeout);
+    wstunneler.on('ping', priv.refreshTimeout);
-    wstunneler.on('pong', wsHandlers.refreshTimeout);
+    wstunneler.on('pong', function () {
+      //console.log('DEBUG received pong');
+      priv.refreshTimeout();
+    });
     wstunneler.on('message', function (data, flags) {
-      wsHandlers.refreshTimeout();
+      priv.refreshTimeout();
       if (data.error || '{' === data[0]) {
         console.log(data);
         return;
       }
       machine.fns.addChunk(data, flags);
     });
-  }
+  };
-  connect();
+  me.end = function() {
-
+    priv.tokens.length = 0;
-  var connPromise;
+    if (priv.timeoutId) {
-  return {
+      clearTimeout(priv.timeoutId);
-    end: function(cb) {
+      priv.timeoutId = null;
-      tokens.length = 0;
-      if (timeoutId) {
-        clearTimeout(timeoutId);
-        timeoutId = null;
-      }
-
-      if (wstunneler) {
-        try {
-          wstunneler.close(cb);
-        } catch(e) {
-          console.error("[error] wstunneler.close()");
-          console.error(e);
-        }
-      }
-    }
-  , append: function (token) {
-      if (!token) {
-        throw new Error("attempted to append empty token");
-      }
-      if ('undefined' === token) {
-        throw new Error("attempted to append token as the string 'undefined'");
-      }
-      if (tokens.indexOf(token) >= 0) {
-        return PromiseA.resolve();
-      }
-      tokens.push(token);
-      var prom;
-      if (tokens.length === 1 && !wstunneler) {
-        // We just added the only token in the list, and the websocket connection isn't up
-        // so we need to restart the connection.
-        if (timeoutId) {
-          // Handle the case were the last token was removed and this token added between
-          // reconnect attempts to make sure we don't try openning multiple connections.
-          clearTimeout(timeoutId);
-          timeoutId = null;
-        }
-
-        // We want this case to behave as much like the other case as we can, but we don't have
-        // the same kind of reponses when we open brand new connections, so we have to rely on
-        // the 'hello' and the 'un-associated' error commands to determine if the token is good.
-        prom = connPromise = new PromiseA(function (resolve, reject) {
-          connCallback = function (err) {
-            connCallback = null;
-            connPromise = null;
-            if (err) {
-              reject(err);
-            } else {
-              resolve();
-            }
-          };
-        });
-        connect();
-      }
-      else if (connPromise) {
-        prom = connPromise.then(function () {
-          return sendCommand('add_token', token);
-        });
-      }
-      else {
-        prom = sendCommand('add_token', token);
-      }
-
-      prom.catch(function (err) {
-        console.error('adding token', token, 'failed:', err);
-        // Most probably an invalid token of some kind, so we don't really want to keep it.
-        tokens.splice(tokens.indexOf(token), 1);
-      });
-
-      return prom;
-    }
-  , clear: function (token) {
-      if (typeof token === 'undefined') {
-        token = '*';
-      }
-
-      if (token === '*') {
-        tokens.length = 0;
-      } else {
-        var index = tokens.indexOf(token);
-        if (index < 0) {
-          return PromiseA.resolve();
-        }
-        tokens.splice(index);
-      }
-
-      var prom = sendCommand('delete_token', token);
-      prom.catch(function (err) {
-        console.error('clearing token', token, 'failed:', err);
-      });
-
-      return prom;
     }
+    console.info('[info] closing due to tr.end()');
+    wstunneler.close(1000, 're-connect');
+    wstunneler.on('close', function () {
+      me.emit('end');
+    });
   };
 }
 
-module.exports.connect = _connect;
+TelebitRemote.prototype = EventEmitter.prototype;
-module.exports.createConnection = _connect;
+TelebitRemote._tokenFromState = function (state) {
+  return {
+    subject: state.config.email
+  , subject_scheme: 'mailto'
+    // TODO create domains list earlier
+  , scope: Object.keys(state.config.servernames || {}).join(',')
+  , otp: state.otp
+  , hostname: os.hostname()
+    // Used for User-Agent
+  , os_type: os.type()
+  , os_platform: os.platform()
+  , os_release: os.release()
+  , os_arch: os.arch()
+  };
+};
+
+TelebitRemote.create = function (opts) {
+  return new TelebitRemote(opts);
+};
+TelebitRemote.createConnection = function (opts, cb) {
+  var tunnel = TelebitRemote.create(opts);
+  tunnel.connect(opts);
+  tunnel.once('connect', cb);
+  return tunnel;
+};
+TelebitRemote.connect = TelebitRemote.createConnection;
+
+module.exports.TelebitRemote = TelebitRemote;
 
 }());

lib/sorting-hat.js

@@ -59,7 +59,7 @@ module.exports.print = function (config) {
 };
 
 module.exports.assign = function (state, tun, cb) {
-  console.log('first message from', tun);
+  //console.log('first message from', tun);
   var net = state.net || require('net');
 
   function trySsh(tun, cb) {
@@ -328,15 +328,15 @@ module.exports.assign = function (state, tun, cb) {
 
     try {
       handler = require(handlerpath);
-      console.info("Handling '" + handle + ":" + id + "' with '" + handlerpath + "'");
+      console.info("Trying to handle '" + handle + ":" + id + "' with '" + handlerpath + "'");
     } catch(e1) {
       try {
         handler = require(path.join(localshare, handlerpath));
-        console.info("Handling '" + handle + ":" + id + "' with '" + handlerpath + "'");
+        console.info("Skip. (couldn't require('" + handlerpath + "'):", e1.message + ")");
+        console.info("Trying to handle '" + handle + ":" + id + "' with '" + handlerpath + "'");
       } catch(e2) {
-        console.error("Failed to require('" + handlerpath + "'):", e1.message);
+        console.info("Skip. (couldn't require('" + path.join(localshare, handlerpath) + "'):", e2.message + ")");
-        console.error("Failed to require('" + path.join(localshare, handlerpath) + "'):", e2.message);
+        console.info("Last chance! (using static and index handlers for '" + handle + ":" + id + "')");
-        console.warn("Trying static and index handlers for '" + handle + ":" + id + "'");
         handler = null;
         // fallthru
       }
@@ -362,21 +362,47 @@ module.exports.assign = function (state, tun, cb) {
         state._serveIndex = require('serve-index');
         var serveIndex;
         var serveStatic;
+        var dlStatic;
         if (isFile) {
           serveStatic = state._serveStatic(path.dirname(conf.handler), { dotfiles: 'allow', index: [ 'index.html' ] });
+          dlStatic = state._serveStatic(path.dirname(conf.handler), { acceptRanges: false, dotfiles: 'allow', index: [ 'index.html' ] });
           serveIndex = function (req, res, next) { next(); };
           isFile = path.basename(conf.handler);
         } else {
           serveStatic = state._serveStatic(conf.handler, { dotfiles: 'allow', index: [ 'index.html' ] });
-          serveIndex = state._serveIndex(conf.handler, { hidden: true, icons: true, view: 'tiles' });
+          dlStatic = state._serveStatic(conf.handler, { acceptRanges: false, dotfiles: 'allow', index: [ 'index.html' ] });
+          serveIndex = state._serveIndex(conf.handler, {
+            hidden: true, icons: true
+          , template: require('serve-tpl-attachment')({ privatefiles: 'ignore' })
+          });
         }
         handler = function (req, res) {
+          var qIndex = req.url.indexOf('?');
+          var fIndex;
+          var fname;
+          if (-1 === qIndex) {
+            qIndex = req.url.length;
+          }
+          req.querystring = req.url.substr(qIndex);
+          req.url = req.url.substr(0, qIndex);
+          req.query = require('querystring').parse(req.querystring.substr(1));
           if (isFile) {
             req.url = '/' + isFile;
           }
-          serveStatic(req, res, function () {
+          //console.log('[req.query]', req.url, req.query);
-            serveIndex(req, res, state._finalHandler(req, res));
+          if (req.query.download) {
-          });
+            fIndex = req.url.lastIndexOf('/');
+            fname = req.url.substr(fIndex + 1);
+            res.setHeader('Content-Disposition', 'attachment; filename="'+decodeURIComponent(fname)+'"');
+            res.setHeader('Content-Type', 'application/octet-stream');
+            dlStatic(req, res, function () {
+              serveIndex(req, res, state._finalHandler(req, res));
+            });
+          } else {
+            serveStatic(req, res, function () {
+              serveIndex(req, res, state._finalHandler(req, res));
+            });
+          }
         };
         handlerservers[conf.handler] = http.createServer(handler);
         handlerservers[conf.handler].emit('connection', tlsSocket);
@@ -426,7 +452,7 @@ module.exports.assign = function (state, tun, cb) {
                 return;
               }
 
-              console.log('https invokeHandler');
+              //console.log('https invokeHandler');
               invokeHandler(conf, tlsSocket, tun, id);
             });
           });

lib/updater.js

@@ -8,6 +8,7 @@ module.exports = function (pkg) {
       https.get(url, function (resp) {
         var str = '';
         resp.on('data', function (chunk) {
+          //var chunk = conn.read();
           str += chunk.toString('utf8');
         });
         resp.on('end', function () {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "telebit",
-  "version": "0.19.28",
+  "version": "0.20.0-wip",
   "description": "Break out of localhost. Connect to any device from anywhere over any tcp port or securely in a browser. A secure tunnel. A poor man's reverse VPN.",
   "main": "lib/remote.js",
   "files": [
@@ -59,15 +59,18 @@
     "js-yaml": "^3.11.0",
     "jsonwebtoken": "^7.1.9",
     "mkdirp": "^0.5.1",
-    "proxy-packer": "^1.4.3",
+    "proxy-packer": "^2.0.2",
     "ps-list": "^5.0.0",
     "recase": "^1.0.4",
     "redirect-https": "^1.1.5",
+    "sclient": "^1.4.1",
     "serve-index": "^1.9.1",
     "serve-static": "^1.13.2",
+    "serve-tpl-attachment": "^1.0.4",
     "sni": "^1.0.0",
     "socket-pair": "^1.0.3",
-    "ws": "^2.2.3"
+    "toml": "^0.4.1",
+    "ws": "^6.0.0"
   },
   "trulyOptionalDependencies": {
     "bluebird": "^3.5.1"