v1.7.0 add stream support

README.md

@@ -14,11 +14,13 @@ Written from scratch, with zero-dependencies.
 
 ```bash
 npm install --save @root/request
+
+# or npm install git+ssh://git@git.therootcompany.com/request.js
 ```
 
 ```js
 var request = require('@root/request');
-request('http://www.google.com', function(error, response, body) {
+request('http://www.google.com', function (error, response, body) {
     console.log('error:', error); // Print the error if one occurred
     console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
     console.log('body:', body); // Print the HTML for the Google homepage.
@@ -31,15 +33,63 @@ request('http://www.google.com', function(error, response, body) {
 var request = require('@root/request');
 
 request('http://www.google.com')
-    .then(function(response) {
+    .then(function (response) {
         console.log('statusCode:', response.statusCode); // Print the response status code if a response was received
         console.log('body:', response.body); // Print the HTML for the Google homepage.
     })
-    .catch(function(error) {
+    .catch(function (error) {
         console.log('error:', error); // Print the error if one occurred
     });
 ```
 
+**Streaming**
+
+In order to keep this library lightweight, performant, and keep the code easy to
+read, the streaming behavior is **_slightly different_** from that of
+`request.js`.
+
+```js
+var request = require('@root/request');
+
+var resp = await request({
+    url: 'http://www.google.com',
+    stream: true
+});
+
+resp.on('data', function () {
+    // got some data
+});
+
+resp.on('end', function () {
+    // the data has ended
+});
+
+// resp.stream is a Promise that is resolved when the read stream is destroyed
+await resp.stream;
+console.log('Done');
+```
+
+The difference is that we don't add an extra layer of stream abstraction.
+You must use the response from await, a Promise, or the callback.
+
+You can also give a file path:
+
+```js
+request({
+    url: 'http://www.google.com',
+    stream: '/tmp/google-index.html'
+});
+```
+
+Which is equivalent to passing a write stream for the file:
+
+```js
+request({
+    url: 'http://www.google.com',
+    stream: fs.createWriteStream('/tmp/google-index.html')
+});
+```
+
 ## Table of contents
 
 -   [Forms](#forms)
@@ -63,7 +113,7 @@ request.post('http://service.com/upload', { form: { key: 'value' } });
 // or
 request.post(
     { url: 'http://service.com/upload', form: { key: 'value' } },
-    function(err, httpResponse, body) {
+    function (err, httpResponse, body) {
         /* ... */
     }
 );
@@ -190,7 +240,7 @@ var username = 'username',
     password = 'password',
     url = 'http://' + username + ':' + password + '@some.server.com';
 
-request({ url: url }, function(error, response, body) {
+request({ url: url }, function (error, response, body) {
     // Do more stuff with 'body' here
 });
 ```

examples/follow-redirect.js

@@ -5,16 +5,15 @@ var request = require('../');
 
 // will redirect to https://www.github.com and then https://github.com
 //request('http://www.github.com', function (error, response, body) {
-request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function(
-    error,
-    response,
-    body
-) {
-    if (error) {
-        console.log('error:', error); // Print the error if one occurred
-        return;
+request(
+    { uri: { protocol: 'http:', hostname: 'www.github.com' } },
+    function (error, response, body) {
+        if (error) {
+            console.log('error:', error); // Print the error if one occurred
+            return;
+        }
+        console.log('statusCode:', response.statusCode); // The final statusCode
+        console.log('Final href:', response.request.uri.href); // The final URI
+        console.log('Body Length:', body.length); // body length
     }
-    console.log('statusCode:', response.statusCode); // The final statusCode
-    console.log('Final href:', response.request.uri.href); // The final URI
-    console.log('Body Length:', body.length); // body length
-});
+);

examples/form-data.js

@@ -20,7 +20,7 @@ request(
             )
         }
     },
-    function(error, response, body) {
+    function (error, response, body) {
         if (error) {
             console.log('error:', error); // Print the error if one occurred
             return;

examples/get-to-json.js

@@ -2,7 +2,7 @@
 
 //var request = require('urequest');
 var request = require('../');
-request('https://www.google.com', function(error, response, body) {
+request('https://www.google.com', function (error, response, body) {
     if (error) {
         console.log('error:', error); // Print the error if one occurred
         return;

examples/http-string.js

@@ -2,7 +2,7 @@
 
 //var request = require('urequest');
 var request = require('../');
-request('http://www.google.com', function(error, response, body) {
+request('http://www.google.com', function (error, response, body) {
     console.log('error:', error); // Print the error if one occurred
     console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
     console.log('body:', body); // Print the HTML for the Google homepage.

examples/https-string.js

@@ -2,7 +2,7 @@
 
 //var request = require('urequest');
 var request = require('../');
-request('https://www.google.com', function(error, response, body) {
+request('https://www.google.com', function (error, response, body) {
     console.log('error:', error); // Print the error if one occurred
     console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
     console.log('body:', body); // Print the HTML for the Google homepage.

examples/no-follow-redirect.js

@@ -4,17 +4,16 @@
 var request = require('../');
 
 // would normally redirect to https://www.github.com and then https://github.com
-request({ uri: 'https://www.github.com', followRedirect: false }, function(
-    error,
-    response,
-    body
-) {
-    if (error) {
-        console.log('error:', error); // Print the error if one occurred
-        return;
+request(
+    { uri: 'https://www.github.com', followRedirect: false },
+    function (error, response, body) {
+        if (error) {
+            console.log('error:', error); // Print the error if one occurred
+            return;
+        }
+        console.log('href:', response.request.uri.href); // The final URI
+        console.log('statusCode:', response.statusCode); // Should be 301 or 302
+        console.log('Location:', response.headers.location); // The redirect
+        console.log('Body:', body || JSON.stringify(body));
     }
-    console.log('href:', response.request.uri.href); // The final URI
-    console.log('statusCode:', response.statusCode); // Should be 301 or 302
-    console.log('Location:', response.headers.location); // The redirect
-    console.log('Body:', body || JSON.stringify(body));
-});
+);

examples/postbin.js

@@ -7,9 +7,9 @@ request({
     //headers: { 'user-agent': 'test/1.0' } // overwrite
     //userAgent: 'test/1.1' // add to the default
 })
-    .then(function(resp) {
+    .then(function (resp) {
         console.log(resp.body);
     })
-    .catch(function(err) {
+    .catch(function (err) {
         console.error(err);
     });

examples/stream-to-file.js

@@ -0,0 +1,27 @@
+'use strict';
+
+var request = require('../');
+
+async function main() {
+    var tpath = '/tmp/google-index.html';
+    var resp = await request({
+        url: 'https://google.com',
+        encoding: null,
+        stream: tpath
+    });
+    console.log('[Response Headers]');
+    console.log(resp.toJSON().headers);
+
+    //console.error(resp.headers, resp.body.byteLength);
+    await resp.stream;
+    console.log('[Response Body] written to', tpath);
+}
+
+main()
+    .then(function () {
+        console.log('Pass');
+    })
+    .catch(function (e) {
+        console.error('Fail');
+        console.error(e.stack);
+    });

examples/stream.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var request = require('../');
+
+async function main() {
+    var tpath = '/tmp/google-index.html';
+    var resp = await request({
+        url: 'https://google.com',
+        encoding: null,
+        stream: true
+    });
+    console.log('[Response Headers]');
+    console.log(resp.toJSON().headers);
+
+    resp.on('data', function (chunk) {
+	    console.log('[Data]', chunk.byteLength);
+    });
+    resp.on('end', function (chunk) {
+	    console.log('[End]');
+    });
+
+    //console.error(resp.headers, resp.body.byteLength);
+    await resp.stream;
+    console.log('[Close]');
+}
+
+main()
+    .then(function () {
+        console.log('Pass');
+    })
+    .catch(function (e) {
+        console.error('Fail');
+        console.error(e.stack);
+    });

examples/www-form.js

@@ -14,7 +14,7 @@ request(
         headers: { 'X-Foo': 'Bar' },
         form: { foo: 'bar', baz: 'qux' }
     },
-    function(error, response, body) {
+    function (error, response, body) {
         if (error) {
             console.log('error:', error); // Print the error if one occurred
             return;

index.js

@@ -5,6 +5,7 @@ var https = require('https');
 var url = require('url');
 var os = require('os');
 var pkg = require('./package.json');
+var fs = require('fs'); // only for streams
 
 function debug() {
     if (module.exports.debug) {
@@ -13,7 +14,7 @@ function debug() {
 }
 
 function mergeOrDelete(defaults, updates) {
-    Object.keys(defaults).forEach(function(key) {
+    Object.keys(defaults).forEach(function (key) {
         if (!(key in updates)) {
             updates[key] = defaults[key];
             return;
@@ -37,7 +38,7 @@ function mergeOrDelete(defaults, updates) {
 // retrieves an existing header, case-sensitive
 function getHeaderName(reqOpts, header) {
     var headerNames = {};
-    Object.keys(reqOpts.headers).forEach(function(casedName) {
+    Object.keys(reqOpts.headers).forEach(function (casedName) {
         headerNames[casedName.toLowerCase()] = casedName;
     });
     // returns the key, which in erroneous cases could be an empty string
@@ -49,11 +50,11 @@ function hasHeader(reqOpts, header) {
 }
 
 function toJSONifier(keys) {
-    return function() {
+    return function () {
         var obj = {};
         var me = this;
 
-        keys.forEach(function(key) {
+        keys.forEach(function (key) {
             if (me[key] && 'function' === typeof me[key].toJSON) {
                 obj[key] = me[key].toJSON();
             } else {
@@ -78,7 +79,7 @@ function setDefaults(defs) {
         function onResponse(resp) {
             var followRedirect;
 
-            Object.keys(defs).forEach(function(key) {
+            Object.keys(defs).forEach(function (key) {
                 if (key in opts && 'undefined' !== typeof opts[key]) {
                     return;
                 }
@@ -140,13 +141,75 @@ function setDefaults(defs) {
                     return urequestHelper(opts, cb);
                 }
             }
+
+            if (opts.stream) {
+                // make the response await-able
+                var resolve;
+                var reject;
+                resp.stream = new Promise(function (_resolve, _reject) {
+                    resolve = _resolve;
+                    reject = _reject;
+                });
+
+                // allow specifying a file
+                if ('string' === typeof opts.stream) {
+                    try {
+                        if (opts.debug) {
+                            console.debug(
+                                '[@root/request] file write stream created'
+                            );
+                        }
+                        opts.stream = fs.createWriteStream(opts.stream);
+                    } catch (e) {
+                        cb(e);
+                    }
+                }
+                // or an existing write stream
+                if ('function' === typeof opts.stream.pipe) {
+                    if (opts.debug) {
+                        console.debug('[@root/request] stream piped');
+                    }
+                    resp.pipe(opts.stream);
+                }
+                resp.on('error', function (e) {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'error'");
+                        console.error(e.stack);
+                    }
+                    resp.destroy();
+                    if ('function' === opts.stream.destroy) {
+                        opts.stream.destroy(e);
+                    }
+                    reject(e);
+                });
+                resp.on('end', function () {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'end'");
+                    }
+                    if ('function' === opts.stream.destroy) {
+                        opts.stream.end();
+                        // this will close the stream (i.e. sync to disk)
+                        opts.stream.destroy();
+                    }
+                });
+                resp.on('close', function () {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'close'");
+                    }
+                    resolve();
+                });
+                // and in all cases, return the stream
+                cb(null, resp);
+                return;
+            }
+
             if (null === opts.encoding) {
                 resp._body = [];
             } else {
                 resp.body = '';
             }
             resp._bodyLength = 0;
-            resp.on('data', function(chunk) {
+            resp.on('data', function (chunk) {
                 if ('string' === typeof resp.body) {
                     resp.body += chunk.toString(opts.encoding);
                 } else {
@@ -154,7 +217,7 @@ function setDefaults(defs) {
                     resp._bodyLength += chunk.length;
                 }
             });
-            resp.on('end', function() {
+            resp.on('end', function () {
                 if ('string' !== typeof resp.body) {
                     if (1 === resp._body.length) {
                         resp.body = resp._body[0];
@@ -174,7 +237,9 @@ function setDefaults(defs) {
                 }
 
                 debug('\n[urequest] resp.toJSON():');
-                debug(resp.toJSON());
+                if (module.exports.debug) {
+                    debug(resp.toJSON());
+                }
                 if (opts.debug) {
                     console.debug('[@root/request] Response Body:');
                     console.debug(resp.body);
@@ -199,12 +264,12 @@ function setDefaults(defs) {
             _body = JSON.stringify(opts.json);
         } else if (opts.form) {
             _body = Object.keys(opts.form)
-                .filter(function(key) {
+                .filter(function (key) {
                     if ('undefined' !== typeof opts.form[key]) {
                         return true;
                     }
                 })
-                .map(function(key) {
+                .map(function (key) {
                     return (
                         encodeURIComponent(key) +
                         '=' +
@@ -218,7 +283,7 @@ function setDefaults(defs) {
             _body = Buffer.from(_body);
         }
 
-        Object.keys(opts.uri).forEach(function(key) {
+        Object.keys(opts.uri).forEach(function (key) {
             finalOpts[key] = opts.uri[key];
         });
 
@@ -233,7 +298,7 @@ function setDefaults(defs) {
             'createConnection',
             'timeout',
             'setHost'
-        ].forEach(function(key) {
+        ].forEach(function (key) {
             finalOpts[key] = opts.uri[key];
         });
 
@@ -299,7 +364,7 @@ function setDefaults(defs) {
             }
             try {
                 form = new MyFormData();
-                Object.keys(opts.formData).forEach(function(key) {
+                Object.keys(opts.formData).forEach(function (key) {
                     function add(key, data, opts) {
                         if (data.value) {
                             opts = data.options;
@@ -308,7 +373,7 @@ function setDefaults(defs) {
                         form.append(key, data, opts);
                     }
                     if (Array.isArray(opts.formData[key])) {
-                        opts.formData[key].forEach(function(data) {
+                        opts.formData[key].forEach(function (data) {
                             add(key, data);
                         });
                     } else {
@@ -320,7 +385,7 @@ function setDefaults(defs) {
                 return;
             }
             formHeaders = form.getHeaders();
-            Object.keys(formHeaders).forEach(function(header) {
+            Object.keys(formHeaders).forEach(function (header) {
                 finalOpts.headers[header] = formHeaders[header];
             });
         }
@@ -346,7 +411,7 @@ function setDefaults(defs) {
             debug(formHeaders);
             // generally uploads don't use Chunked Encoding (some systems have issues with it)
             // and I don't want to do the work to calculate the content-lengths. This seems to work.
-            req = form.submit(finalOpts, function(err, resp) {
+            req = form.submit(finalOpts, function (err, resp) {
                 if (err) {
                     cb(err);
                     return;
@@ -380,7 +445,7 @@ function setDefaults(defs) {
             if ('function' === typeof _body.pipe) {
                 // used for chunked encoding
                 _body.pipe(req);
-                _body.on('error', function(err) {
+                _body.on('error', function (err) {
                     // https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
                     // if the Readable stream emits an error during processing,
                     // the Writable destination is not closed automatically
@@ -427,7 +492,7 @@ function setDefaults(defs) {
             opts = { url: opts };
         }
 
-        module.exports._keys.forEach(function(key) {
+        module.exports._keys.forEach(function (key) {
             if (key in opts && 'undefined' !== typeof opts[key]) {
                 reqOpts[key] = opts[key];
             } else if (key in defs) {
@@ -489,14 +554,14 @@ function setDefaults(defs) {
     }
 
     function smartPromisify(fn) {
-        return function(opts) {
+        return function (opts) {
             var cb;
             if ('function' === typeof arguments[1]) {
                 cb = Array.prototype.slice.call(arguments)[1];
                 return fn(opts, cb);
             }
-            return new Promise(function(resolve, reject) {
-                fn(opts, function(err, resp) {
+            return new Promise(function (resolve, reject) {
+                fn(opts, function (err, resp) {
                     if (err) {
                         err._response = resp;
                         reject(err);
@@ -510,13 +575,13 @@ function setDefaults(defs) {
 
     var smartUrequest = smartPromisify(urequest);
 
-    smartUrequest.defaults = function(_defs) {
+    smartUrequest.defaults = function (_defs) {
         _defs = mergeOrDelete(defs, _defs);
         return setDefaults(_defs);
     };
     ['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
-        function(method) {
-            smartUrequest[method] = smartPromisify(function(obj, cb) {
+        function (method) {
+            smartUrequest[method] = smartPromisify(function (obj, cb) {
                 if ('string' === typeof obj) {
                     obj = { url: obj };
                 }
@@ -568,7 +633,8 @@ var _defaults = {
     followOriginalHttpMethod: false,
     maxRedirects: 10,
     removeRefererHeader: false,
-    //, encoding: undefined
+    // encoding: undefined,
+    // stream: false, // TODO allow a stream?
     gzip: false
     //, body: undefined
     //, json: undefined
@@ -577,6 +643,7 @@ module.exports = setDefaults(_defaults);
 
 module.exports._keys = Object.keys(_defaults).concat([
     'encoding',
+    'stream',
     'body',
     'json',
     'form',

package-lock.json

@@ -1,5 +1,5 @@
 {
     "name": "@root/request",
-    "version": "1.5.0",
+    "version": "1.7.0",
     "lockfileVersion": 1
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@root/request",
-    "version": "1.6.1",
+    "version": "1.7.0",
     "description": "A lightweight, zero-dependency drop-in replacement for request",
     "main": "index.js",
     "files": [

tests/tcp-listener.js

@@ -1,11 +1,11 @@
 'use strict';
 
 var net = require('net');
-var server = net.createServer(function(socket) {
-    socket.on('data', function(chunk) {
+var server = net.createServer(function (socket) {
+    socket.on('data', function (chunk) {
         console.info(chunk.toString('utf8'));
     });
 });
-server.listen(3007, function() {
+server.listen(3007, function () {
     console.info('Listening on', this.address());
 });