v1.7.0 add stream support

This commit is contained in:
AJ ONeal 2021-01-14 16:35:07 -07:00
parent 9395ec96e3
commit 95a12a8285
6 changed files with 182 additions and 4 deletions

View File

@ -14,6 +14,8 @@ Written from scratch, with zero-dependencies.
```bash ```bash
npm install --save @root/request npm install --save @root/request
# or npm install git+ssh://git@git.therootcompany.com/request.js
``` ```
```js ```js
@ -40,6 +42,54 @@ request('http://www.google.com')
}); });
``` ```
**Streaming**
In order to keep this library lightweight, performant, and keep the code easy to
read, the streaming behavior is **_slightly different_** from that of
`request.js`.
```js
var request = require('@root/request');
var resp = await request({
url: 'http://www.google.com',
stream: true
});
resp.on('data', function () {
// got some data
});
resp.on('end', function () {
// the data has ended
});
// resp.stream is a Promise that is resolved when the read stream is destroyed
await resp.stream;
console.log('Done');
```
The difference is that we don't add an extra layer of stream abstraction.
You must use the response from await, a Promise, or the callback.
You can also give a file path:
```js
request({
url: 'http://www.google.com',
stream: '/tmp/google-index.html'
});
```
Which is equivalent to passing a write stream for the file:
```js
request({
url: 'http://www.google.com',
stream: fs.createWriteStream('/tmp/google-index.html')
});
```
## Table of contents ## Table of contents
- [Forms](#forms) - [Forms](#forms)

View File

@ -0,0 +1,27 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: tpath
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Response Body] written to', tpath);
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

34
examples/stream.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: true
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
resp.on('data', function (chunk) {
console.log('[Data]', chunk.byteLength);
});
resp.on('end', function (chunk) {
console.log('[End]');
});
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Close]');
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

View File

@ -5,6 +5,7 @@ var https = require('https');
var url = require('url'); var url = require('url');
var os = require('os'); var os = require('os');
var pkg = require('./package.json'); var pkg = require('./package.json');
var fs = require('fs'); // only for streams
function debug() { function debug() {
if (module.exports.debug) { if (module.exports.debug) {
@ -140,6 +141,68 @@ function setDefaults(defs) {
return urequestHelper(opts, cb); return urequestHelper(opts, cb);
} }
} }
if (opts.stream) {
// make the response await-able
var resolve;
var reject;
resp.stream = new Promise(function (_resolve, _reject) {
resolve = _resolve;
reject = _reject;
});
// allow specifying a file
if ('string' === typeof opts.stream) {
try {
if (opts.debug) {
console.debug(
'[@root/request] file write stream created'
);
}
opts.stream = fs.createWriteStream(opts.stream);
} catch (e) {
cb(e);
}
}
// or an existing write stream
if ('function' === typeof opts.stream.pipe) {
if (opts.debug) {
console.debug('[@root/request] stream piped');
}
resp.pipe(opts.stream);
}
resp.on('error', function (e) {
if (opts.debug) {
console.debug("[@root/request] stream 'error'");
console.error(e.stack);
}
resp.destroy();
if ('function' === opts.stream.destroy) {
opts.stream.destroy(e);
}
reject(e);
});
resp.on('end', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'end'");
}
if ('function' === opts.stream.destroy) {
opts.stream.end();
// this will close the stream (i.e. sync to disk)
opts.stream.destroy();
}
});
resp.on('close', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'close'");
}
resolve();
});
// and in all cases, return the stream
cb(null, resp);
return;
}
if (null === opts.encoding) { if (null === opts.encoding) {
resp._body = []; resp._body = [];
} else { } else {
@ -174,7 +237,9 @@ function setDefaults(defs) {
} }
debug('\n[urequest] resp.toJSON():'); debug('\n[urequest] resp.toJSON():');
if (module.exports.debug) {
debug(resp.toJSON()); debug(resp.toJSON());
}
if (opts.debug) { if (opts.debug) {
console.debug('[@root/request] Response Body:'); console.debug('[@root/request] Response Body:');
console.debug(resp.body); console.debug(resp.body);
@ -568,7 +633,8 @@ var _defaults = {
followOriginalHttpMethod: false, followOriginalHttpMethod: false,
maxRedirects: 10, maxRedirects: 10,
removeRefererHeader: false, removeRefererHeader: false,
//, encoding: undefined // encoding: undefined,
// stream: false, // TODO allow a stream?
gzip: false gzip: false
//, body: undefined //, body: undefined
//, json: undefined //, json: undefined
@ -577,6 +643,7 @@ module.exports = setDefaults(_defaults);
module.exports._keys = Object.keys(_defaults).concat([ module.exports._keys = Object.keys(_defaults).concat([
'encoding', 'encoding',
'stream',
'body', 'body',
'json', 'json',
'form', 'form',

2
package-lock.json generated
View File

@ -1,5 +1,5 @@
{ {
"name": "@root/request", "name": "@root/request",
"version": "1.5.0", "version": "1.7.0",
"lockfileVersion": 1 "lockfileVersion": 1
} }

View File

@ -1,6 +1,6 @@
{ {
"name": "@root/request", "name": "@root/request",
"version": "1.6.1", "version": "1.7.0",
"description": "A lightweight, zero-dependency drop-in replacement for request", "description": "A lightweight, zero-dependency drop-in replacement for request",
"main": "index.js", "main": "index.js",
"files": [ "files": [