From 95a12a82850a16c50ea239c7056c8f1807bc7564 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 14 Jan 2021 16:35:07 -0700 Subject: [PATCH] v1.7.0 add stream support --- README.md | 50 +++++++++++++++++++++++++++ examples/stream-to-file.js | 27 +++++++++++++++ examples/stream.js | 34 ++++++++++++++++++ index.js | 71 ++++++++++++++++++++++++++++++++++++-- package-lock.json | 2 +- package.json | 2 +- 6 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 examples/stream-to-file.js create mode 100644 examples/stream.js diff --git a/README.md b/README.md index 830b4ab..5f00163 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Written from scratch, with zero-dependencies. ```bash npm install --save @root/request + +# or npm install git+ssh://git@git.therootcompany.com/request.js ``` ```js @@ -40,6 +42,54 @@ request('http://www.google.com') }); ``` +**Streaming** + +In order to keep this library lightweight, performant, and keep the code easy to +read, the streaming behavior is **_slightly different_** from that of +`request.js`. + +```js +var request = require('@root/request'); + +var resp = await request({ + url: 'http://www.google.com', + stream: true +}); + +resp.on('data', function () { + // got some data +}); + +resp.on('end', function () { + // the data has ended +}); + +// resp.stream is a Promise that is resolved when the read stream is destroyed +await resp.stream; +console.log('Done'); +``` + +The difference is that we don't add an extra layer of stream abstraction. +You must use the response from await, a Promise, or the callback. + +You can also give a file path: + +```js +request({ + url: 'http://www.google.com', + stream: '/tmp/google-index.html' +}); +``` + +Which is equivalent to passing a write stream for the file: + +```js +request({ + url: 'http://www.google.com', + stream: fs.createWriteStream('/tmp/google-index.html') +}); +``` + ## Table of contents - [Forms](#forms) diff --git a/examples/stream-to-file.js b/examples/stream-to-file.js new file mode 100644 index 0000000..7258c16 --- /dev/null +++ b/examples/stream-to-file.js @@ -0,0 +1,27 @@ +'use strict'; + +var request = require('../'); + +async function main() { + var tpath = '/tmp/google-index.html'; + var resp = await request({ + url: 'https://google.com', + encoding: null, + stream: tpath + }); + console.log('[Response Headers]'); + console.log(resp.toJSON().headers); + + //console.error(resp.headers, resp.body.byteLength); + await resp.stream; + console.log('[Response Body] written to', tpath); +} + +main() + .then(function () { + console.log('Pass'); + }) + .catch(function (e) { + console.error('Fail'); + console.error(e.stack); + }); diff --git a/examples/stream.js b/examples/stream.js new file mode 100644 index 0000000..16908ab --- /dev/null +++ b/examples/stream.js @@ -0,0 +1,34 @@ +'use strict'; + +var request = require('../'); + +async function main() { + var tpath = '/tmp/google-index.html'; + var resp = await request({ + url: 'https://google.com', + encoding: null, + stream: true + }); + console.log('[Response Headers]'); + console.log(resp.toJSON().headers); + + resp.on('data', function (chunk) { + console.log('[Data]', chunk.byteLength); + }); + resp.on('end', function (chunk) { + console.log('[End]'); + }); + + //console.error(resp.headers, resp.body.byteLength); + await resp.stream; + console.log('[Close]'); +} + +main() + .then(function () { + console.log('Pass'); + }) + .catch(function (e) { + console.error('Fail'); + console.error(e.stack); + }); diff --git a/index.js b/index.js index d2ec7be..85b5a89 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,7 @@ var https = require('https'); var url = require('url'); var os = require('os'); var pkg = require('./package.json'); +var fs = require('fs'); // only for streams function debug() { if (module.exports.debug) { @@ -140,6 +141,68 @@ function setDefaults(defs) { return urequestHelper(opts, cb); } } + + if (opts.stream) { + // make the response await-able + var resolve; + var reject; + resp.stream = new Promise(function (_resolve, _reject) { + resolve = _resolve; + reject = _reject; + }); + + // allow specifying a file + if ('string' === typeof opts.stream) { + try { + if (opts.debug) { + console.debug( + '[@root/request] file write stream created' + ); + } + opts.stream = fs.createWriteStream(opts.stream); + } catch (e) { + cb(e); + } + } + // or an existing write stream + if ('function' === typeof opts.stream.pipe) { + if (opts.debug) { + console.debug('[@root/request] stream piped'); + } + resp.pipe(opts.stream); + } + resp.on('error', function (e) { + if (opts.debug) { + console.debug("[@root/request] stream 'error'"); + console.error(e.stack); + } + resp.destroy(); + if ('function' === opts.stream.destroy) { + opts.stream.destroy(e); + } + reject(e); + }); + resp.on('end', function () { + if (opts.debug) { + console.debug("[@root/request] stream 'end'"); + } + if ('function' === opts.stream.destroy) { + opts.stream.end(); + // this will close the stream (i.e. sync to disk) + opts.stream.destroy(); + } + }); + resp.on('close', function () { + if (opts.debug) { + console.debug("[@root/request] stream 'close'"); + } + resolve(); + }); + // and in all cases, return the stream + cb(null, resp); + return; + } + if (null === opts.encoding) { resp._body = []; } else { @@ -174,7 +237,9 @@ function setDefaults(defs) { } debug('\n[urequest] resp.toJSON():'); - debug(resp.toJSON()); + if (module.exports.debug) { + debug(resp.toJSON()); + } if (opts.debug) { console.debug('[@root/request] Response Body:'); console.debug(resp.body); @@ -568,7 +633,8 @@ var _defaults = { followOriginalHttpMethod: false, maxRedirects: 10, removeRefererHeader: false, - //, encoding: undefined + // encoding: undefined, + // stream: false, // TODO allow a stream? gzip: false //, body: undefined //, json: undefined @@ -577,6 +643,7 @@ module.exports = setDefaults(_defaults); module.exports._keys = Object.keys(_defaults).concat([ 'encoding', + 'stream', 'body', 'json', 'form', diff --git a/package-lock.json b/package-lock.json index ad57535..e363303 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { "name": "@root/request", - "version": "1.5.0", + "version": "1.7.0", "lockfileVersion": 1 } diff --git a/package.json b/package.json index 7b6f85a..0d8bfa4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@root/request", - "version": "1.6.1", + "version": "1.7.0", "description": "A lightweight, zero-dependency drop-in replacement for request", "main": "index.js", "files": [