v1.7.0 add stream support

README.md

@@ -14,6 +14,8 @@ Written from scratch, with zero-dependencies.
 
 ```bash
 npm install --save @root/request
+
+# or npm install git+ssh://git@git.therootcompany.com/request.js
 ```
 
 ```js
@@ -40,6 +42,54 @@ request('http://www.google.com')
     });
 ```
 
+**Streaming**
+
+In order to keep this library lightweight, performant, and keep the code easy to
+read, the streaming behavior is **_slightly different_** from that of
+`request.js`.
+
+```js
+var request = require('@root/request');
+
+var resp = await request({
+    url: 'http://www.google.com',
+    stream: true
+});
+
+resp.on('data', function () {
+    // got some data
+});
+
+resp.on('end', function () {
+    // the data has ended
+});
+
+// resp.stream is a Promise that is resolved when the read stream is destroyed
+await resp.stream;
+console.log('Done');
+```
+
+The difference is that we don't add an extra layer of stream abstraction.
+You must use the response from await, a Promise, or the callback.
+
+You can also give a file path:
+
+```js
+request({
+    url: 'http://www.google.com',
+    stream: '/tmp/google-index.html'
+});
+```
+
+Which is equivalent to passing a write stream for the file:
+
+```js
+request({
+    url: 'http://www.google.com',
+    stream: fs.createWriteStream('/tmp/google-index.html')
+});
+```
+
 ## Table of contents
 
 -   [Forms](#forms)

examples/follow-redirect.js

@@ -5,11 +5,9 @@ var request = require('../');
 
 // will redirect to https://www.github.com and then https://github.com
 //request('http://www.github.com', function (error, response, body) {
-request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function(
+request(
-    error,
+    { uri: { protocol: 'http:', hostname: 'www.github.com' } },
-    response,
+    function (error, response, body) {
-    body
-) {
         if (error) {
             console.log('error:', error); // Print the error if one occurred
             return;
@@ -17,4 +15,5 @@ request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function(
         console.log('statusCode:', response.statusCode); // The final statusCode
         console.log('Final href:', response.request.uri.href); // The final URI
         console.log('Body Length:', body.length); // body length
-});
+    }
+);

examples/no-follow-redirect.js

@@ -4,11 +4,9 @@
 var request = require('../');
 
 // would normally redirect to https://www.github.com and then https://github.com
-request({ uri: 'https://www.github.com', followRedirect: false }, function(
+request(
-    error,
+    { uri: 'https://www.github.com', followRedirect: false },
-    response,
+    function (error, response, body) {
-    body
-) {
         if (error) {
             console.log('error:', error); // Print the error if one occurred
             return;
@@ -17,4 +15,5 @@ request({ uri: 'https://www.github.com', followRedirect: false }, function(
         console.log('statusCode:', response.statusCode); // Should be 301 or 302
         console.log('Location:', response.headers.location); // The redirect
         console.log('Body:', body || JSON.stringify(body));
-});
+    }
+);

examples/stream-to-file.js

@@ -0,0 +1,27 @@
+'use strict';
+
+var request = require('../');
+
+async function main() {
+    var tpath = '/tmp/google-index.html';
+    var resp = await request({
+        url: 'https://google.com',
+        encoding: null,
+        stream: tpath
+    });
+    console.log('[Response Headers]');
+    console.log(resp.toJSON().headers);
+
+    //console.error(resp.headers, resp.body.byteLength);
+    await resp.stream;
+    console.log('[Response Body] written to', tpath);
+}
+
+main()
+    .then(function () {
+        console.log('Pass');
+    })
+    .catch(function (e) {
+        console.error('Fail');
+        console.error(e.stack);
+    });

examples/stream.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var request = require('../');
+
+async function main() {
+    var tpath = '/tmp/google-index.html';
+    var resp = await request({
+        url: 'https://google.com',
+        encoding: null,
+        stream: true
+    });
+    console.log('[Response Headers]');
+    console.log(resp.toJSON().headers);
+
+    resp.on('data', function (chunk) {
+	    console.log('[Data]', chunk.byteLength);
+    });
+    resp.on('end', function (chunk) {
+	    console.log('[End]');
+    });
+
+    //console.error(resp.headers, resp.body.byteLength);
+    await resp.stream;
+    console.log('[Close]');
+}
+
+main()
+    .then(function () {
+        console.log('Pass');
+    })
+    .catch(function (e) {
+        console.error('Fail');
+        console.error(e.stack);
+    });

index.js

@@ -5,6 +5,7 @@ var https = require('https');
 var url = require('url');
 var os = require('os');
 var pkg = require('./package.json');
+var fs = require('fs'); // only for streams
 
 function debug() {
     if (module.exports.debug) {
@@ -140,6 +141,68 @@ function setDefaults(defs) {
                     return urequestHelper(opts, cb);
                 }
             }
+
+            if (opts.stream) {
+                // make the response await-able
+                var resolve;
+                var reject;
+                resp.stream = new Promise(function (_resolve, _reject) {
+                    resolve = _resolve;
+                    reject = _reject;
+                });
+
+                // allow specifying a file
+                if ('string' === typeof opts.stream) {
+                    try {
+                        if (opts.debug) {
+                            console.debug(
+                                '[@root/request] file write stream created'
+                            );
+                        }
+                        opts.stream = fs.createWriteStream(opts.stream);
+                    } catch (e) {
+                        cb(e);
+                    }
+                }
+                // or an existing write stream
+                if ('function' === typeof opts.stream.pipe) {
+                    if (opts.debug) {
+                        console.debug('[@root/request] stream piped');
+                    }
+                    resp.pipe(opts.stream);
+                }
+                resp.on('error', function (e) {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'error'");
+                        console.error(e.stack);
+                    }
+                    resp.destroy();
+                    if ('function' === opts.stream.destroy) {
+                        opts.stream.destroy(e);
+                    }
+                    reject(e);
+                });
+                resp.on('end', function () {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'end'");
+                    }
+                    if ('function' === opts.stream.destroy) {
+                        opts.stream.end();
+                        // this will close the stream (i.e. sync to disk)
+                        opts.stream.destroy();
+                    }
+                });
+                resp.on('close', function () {
+                    if (opts.debug) {
+                        console.debug("[@root/request] stream 'close'");
+                    }
+                    resolve();
+                });
+                // and in all cases, return the stream
+                cb(null, resp);
+                return;
+            }
+
             if (null === opts.encoding) {
                 resp._body = [];
             } else {
@@ -174,7 +237,9 @@ function setDefaults(defs) {
                 }
 
                 debug('\n[urequest] resp.toJSON():');
+                if (module.exports.debug) {
                     debug(resp.toJSON());
+                }
                 if (opts.debug) {
                     console.debug('[@root/request] Response Body:');
                     console.debug(resp.body);
@@ -568,7 +633,8 @@ var _defaults = {
     followOriginalHttpMethod: false,
     maxRedirects: 10,
     removeRefererHeader: false,
-    //, encoding: undefined
+    // encoding: undefined,
+    // stream: false, // TODO allow a stream?
     gzip: false
     //, body: undefined
     //, json: undefined
@@ -577,6 +643,7 @@ module.exports = setDefaults(_defaults);
 
 module.exports._keys = Object.keys(_defaults).concat([
     'encoding',
+    'stream',
     'body',
     'json',
     'form',

package-lock.json

@@ -1,5 +1,5 @@
 {
     "name": "@root/request",
-    "version": "1.5.0",
+    "version": "1.7.0",
     "lockfileVersion": 1
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@root/request",
-    "version": "1.6.1",
+    "version": "1.7.0",
     "description": "A lightweight, zero-dependency drop-in replacement for request",
     "main": "index.js",
     "files": [