Compare commits

...

2 Commits

Author SHA1 Message Date
AJ ONeal 95a12a8285 v1.7.0 add stream support 2021-01-14 16:35:07 -07:00
AJ ONeal 9395ec96e3 make Prettier (v2) 2021-01-14 16:07:48 -07:00
15 changed files with 243 additions and 67 deletions

View File

@ -14,11 +14,13 @@ Written from scratch, with zero-dependencies.
```bash
npm install --save @root/request
# or npm install git+ssh://git@git.therootcompany.com/request.js
```
```js
var request = require('@root/request');
request('http://www.google.com', function(error, response, body) {
request('http://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
@ -31,15 +33,63 @@ request('http://www.google.com', function(error, response, body) {
var request = require('@root/request');
request('http://www.google.com')
.then(function(response) {
.then(function (response) {
console.log('statusCode:', response.statusCode); // Print the response status code if a response was received
console.log('body:', response.body); // Print the HTML for the Google homepage.
})
.catch(function(error) {
.catch(function (error) {
console.log('error:', error); // Print the error if one occurred
});
```
**Streaming**
In order to keep this library lightweight, performant, and keep the code easy to
read, the streaming behavior is **_slightly different_** from that of
`request.js`.
```js
var request = require('@root/request');
var resp = await request({
url: 'http://www.google.com',
stream: true
});
resp.on('data', function () {
// got some data
});
resp.on('end', function () {
// the data has ended
});
// resp.stream is a Promise that is resolved when the read stream is destroyed
await resp.stream;
console.log('Done');
```
The difference is that we don't add an extra layer of stream abstraction.
You must use the response from await, a Promise, or the callback.
You can also give a file path:
```js
request({
url: 'http://www.google.com',
stream: '/tmp/google-index.html'
});
```
Which is equivalent to passing a write stream for the file:
```js
request({
url: 'http://www.google.com',
stream: fs.createWriteStream('/tmp/google-index.html')
});
```
## Table of contents
- [Forms](#forms)
@ -63,7 +113,7 @@ request.post('http://service.com/upload', { form: { key: 'value' } });
// or
request.post(
{ url: 'http://service.com/upload', form: { key: 'value' } },
function(err, httpResponse, body) {
function (err, httpResponse, body) {
/* ... */
}
);
@ -190,7 +240,7 @@ var username = 'username',
password = 'password',
url = 'http://' + username + ':' + password + '@some.server.com';
request({ url: url }, function(error, response, body) {
request({ url: url }, function (error, response, body) {
// Do more stuff with 'body' here
});
```

View File

@ -5,16 +5,15 @@ var request = require('../');
// will redirect to https://www.github.com and then https://github.com
//request('http://www.github.com', function (error, response, body) {
request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function(
error,
response,
body
) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
request(
{ uri: { protocol: 'http:', hostname: 'www.github.com' } },
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
}
console.log('statusCode:', response.statusCode); // The final statusCode
console.log('Final href:', response.request.uri.href); // The final URI
console.log('Body Length:', body.length); // body length
}
console.log('statusCode:', response.statusCode); // The final statusCode
console.log('Final href:', response.request.uri.href); // The final URI
console.log('Body Length:', body.length); // body length
});
);

View File

@ -20,7 +20,7 @@ request(
)
}
},
function(error, response, body) {
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;

View File

@ -2,7 +2,7 @@
//var request = require('urequest');
var request = require('../');
request('https://www.google.com', function(error, response, body) {
request('https://www.google.com', function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;

View File

@ -2,7 +2,7 @@
//var request = require('urequest');
var request = require('../');
request('http://www.google.com', function(error, response, body) {
request('http://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.

View File

@ -2,7 +2,7 @@
//var request = require('urequest');
var request = require('../');
request('https://www.google.com', function(error, response, body) {
request('https://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.

View File

@ -4,17 +4,16 @@
var request = require('../');
// would normally redirect to https://www.github.com and then https://github.com
request({ uri: 'https://www.github.com', followRedirect: false }, function(
error,
response,
body
) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
request(
{ uri: 'https://www.github.com', followRedirect: false },
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
}
console.log('href:', response.request.uri.href); // The final URI
console.log('statusCode:', response.statusCode); // Should be 301 or 302
console.log('Location:', response.headers.location); // The redirect
console.log('Body:', body || JSON.stringify(body));
}
console.log('href:', response.request.uri.href); // The final URI
console.log('statusCode:', response.statusCode); // Should be 301 or 302
console.log('Location:', response.headers.location); // The redirect
console.log('Body:', body || JSON.stringify(body));
});
);

View File

@ -7,9 +7,9 @@ request({
//headers: { 'user-agent': 'test/1.0' } // overwrite
//userAgent: 'test/1.1' // add to the default
})
.then(function(resp) {
.then(function (resp) {
console.log(resp.body);
})
.catch(function(err) {
.catch(function (err) {
console.error(err);
});

View File

@ -0,0 +1,27 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: tpath
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Response Body] written to', tpath);
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

34
examples/stream.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: true
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
resp.on('data', function (chunk) {
console.log('[Data]', chunk.byteLength);
});
resp.on('end', function (chunk) {
console.log('[End]');
});
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Close]');
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

View File

@ -14,7 +14,7 @@ request(
headers: { 'X-Foo': 'Bar' },
form: { foo: 'bar', baz: 'qux' }
},
function(error, response, body) {
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;

117
index.js
View File

@ -5,6 +5,7 @@ var https = require('https');
var url = require('url');
var os = require('os');
var pkg = require('./package.json');
var fs = require('fs'); // only for streams
function debug() {
if (module.exports.debug) {
@ -13,7 +14,7 @@ function debug() {
}
function mergeOrDelete(defaults, updates) {
Object.keys(defaults).forEach(function(key) {
Object.keys(defaults).forEach(function (key) {
if (!(key in updates)) {
updates[key] = defaults[key];
return;
@ -37,7 +38,7 @@ function mergeOrDelete(defaults, updates) {
// retrieves an existing header, case-sensitive
function getHeaderName(reqOpts, header) {
var headerNames = {};
Object.keys(reqOpts.headers).forEach(function(casedName) {
Object.keys(reqOpts.headers).forEach(function (casedName) {
headerNames[casedName.toLowerCase()] = casedName;
});
// returns the key, which in erroneous cases could be an empty string
@ -49,11 +50,11 @@ function hasHeader(reqOpts, header) {
}
function toJSONifier(keys) {
return function() {
return function () {
var obj = {};
var me = this;
keys.forEach(function(key) {
keys.forEach(function (key) {
if (me[key] && 'function' === typeof me[key].toJSON) {
obj[key] = me[key].toJSON();
} else {
@ -78,7 +79,7 @@ function setDefaults(defs) {
function onResponse(resp) {
var followRedirect;
Object.keys(defs).forEach(function(key) {
Object.keys(defs).forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
return;
}
@ -140,13 +141,75 @@ function setDefaults(defs) {
return urequestHelper(opts, cb);
}
}
if (opts.stream) {
// make the response await-able
var resolve;
var reject;
resp.stream = new Promise(function (_resolve, _reject) {
resolve = _resolve;
reject = _reject;
});
// allow specifying a file
if ('string' === typeof opts.stream) {
try {
if (opts.debug) {
console.debug(
'[@root/request] file write stream created'
);
}
opts.stream = fs.createWriteStream(opts.stream);
} catch (e) {
cb(e);
}
}
// or an existing write stream
if ('function' === typeof opts.stream.pipe) {
if (opts.debug) {
console.debug('[@root/request] stream piped');
}
resp.pipe(opts.stream);
}
resp.on('error', function (e) {
if (opts.debug) {
console.debug("[@root/request] stream 'error'");
console.error(e.stack);
}
resp.destroy();
if ('function' === opts.stream.destroy) {
opts.stream.destroy(e);
}
reject(e);
});
resp.on('end', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'end'");
}
if ('function' === opts.stream.destroy) {
opts.stream.end();
// this will close the stream (i.e. sync to disk)
opts.stream.destroy();
}
});
resp.on('close', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'close'");
}
resolve();
});
// and in all cases, return the stream
cb(null, resp);
return;
}
if (null === opts.encoding) {
resp._body = [];
} else {
resp.body = '';
}
resp._bodyLength = 0;
resp.on('data', function(chunk) {
resp.on('data', function (chunk) {
if ('string' === typeof resp.body) {
resp.body += chunk.toString(opts.encoding);
} else {
@ -154,7 +217,7 @@ function setDefaults(defs) {
resp._bodyLength += chunk.length;
}
});
resp.on('end', function() {
resp.on('end', function () {
if ('string' !== typeof resp.body) {
if (1 === resp._body.length) {
resp.body = resp._body[0];
@ -174,7 +237,9 @@ function setDefaults(defs) {
}
debug('\n[urequest] resp.toJSON():');
debug(resp.toJSON());
if (module.exports.debug) {
debug(resp.toJSON());
}
if (opts.debug) {
console.debug('[@root/request] Response Body:');
console.debug(resp.body);
@ -199,12 +264,12 @@ function setDefaults(defs) {
_body = JSON.stringify(opts.json);
} else if (opts.form) {
_body = Object.keys(opts.form)
.filter(function(key) {
.filter(function (key) {
if ('undefined' !== typeof opts.form[key]) {
return true;
}
})
.map(function(key) {
.map(function (key) {
return (
encodeURIComponent(key) +
'=' +
@ -218,7 +283,7 @@ function setDefaults(defs) {
_body = Buffer.from(_body);
}
Object.keys(opts.uri).forEach(function(key) {
Object.keys(opts.uri).forEach(function (key) {
finalOpts[key] = opts.uri[key];
});
@ -233,7 +298,7 @@ function setDefaults(defs) {
'createConnection',
'timeout',
'setHost'
].forEach(function(key) {
].forEach(function (key) {
finalOpts[key] = opts.uri[key];
});
@ -299,7 +364,7 @@ function setDefaults(defs) {
}
try {
form = new MyFormData();
Object.keys(opts.formData).forEach(function(key) {
Object.keys(opts.formData).forEach(function (key) {
function add(key, data, opts) {
if (data.value) {
opts = data.options;
@ -308,7 +373,7 @@ function setDefaults(defs) {
form.append(key, data, opts);
}
if (Array.isArray(opts.formData[key])) {
opts.formData[key].forEach(function(data) {
opts.formData[key].forEach(function (data) {
add(key, data);
});
} else {
@ -320,7 +385,7 @@ function setDefaults(defs) {
return;
}
formHeaders = form.getHeaders();
Object.keys(formHeaders).forEach(function(header) {
Object.keys(formHeaders).forEach(function (header) {
finalOpts.headers[header] = formHeaders[header];
});
}
@ -346,7 +411,7 @@ function setDefaults(defs) {
debug(formHeaders);
// generally uploads don't use Chunked Encoding (some systems have issues with it)
// and I don't want to do the work to calculate the content-lengths. This seems to work.
req = form.submit(finalOpts, function(err, resp) {
req = form.submit(finalOpts, function (err, resp) {
if (err) {
cb(err);
return;
@ -380,7 +445,7 @@ function setDefaults(defs) {
if ('function' === typeof _body.pipe) {
// used for chunked encoding
_body.pipe(req);
_body.on('error', function(err) {
_body.on('error', function (err) {
// https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
// if the Readable stream emits an error during processing,
// the Writable destination is not closed automatically
@ -427,7 +492,7 @@ function setDefaults(defs) {
opts = { url: opts };
}
module.exports._keys.forEach(function(key) {
module.exports._keys.forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
reqOpts[key] = opts[key];
} else if (key in defs) {
@ -489,14 +554,14 @@ function setDefaults(defs) {
}
function smartPromisify(fn) {
return function(opts) {
return function (opts) {
var cb;
if ('function' === typeof arguments[1]) {
cb = Array.prototype.slice.call(arguments)[1];
return fn(opts, cb);
}
return new Promise(function(resolve, reject) {
fn(opts, function(err, resp) {
return new Promise(function (resolve, reject) {
fn(opts, function (err, resp) {
if (err) {
err._response = resp;
reject(err);
@ -510,13 +575,13 @@ function setDefaults(defs) {
var smartUrequest = smartPromisify(urequest);
smartUrequest.defaults = function(_defs) {
smartUrequest.defaults = function (_defs) {
_defs = mergeOrDelete(defs, _defs);
return setDefaults(_defs);
};
['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
function(method) {
smartUrequest[method] = smartPromisify(function(obj, cb) {
function (method) {
smartUrequest[method] = smartPromisify(function (obj, cb) {
if ('string' === typeof obj) {
obj = { url: obj };
}
@ -568,7 +633,8 @@ var _defaults = {
followOriginalHttpMethod: false,
maxRedirects: 10,
removeRefererHeader: false,
//, encoding: undefined
// encoding: undefined,
// stream: false, // TODO allow a stream?
gzip: false
//, body: undefined
//, json: undefined
@ -577,6 +643,7 @@ module.exports = setDefaults(_defaults);
module.exports._keys = Object.keys(_defaults).concat([
'encoding',
'stream',
'body',
'json',
'form',

2
package-lock.json generated
View File

@ -1,5 +1,5 @@
{
"name": "@root/request",
"version": "1.5.0",
"version": "1.7.0",
"lockfileVersion": 1
}

View File

@ -1,6 +1,6 @@
{
"name": "@root/request",
"version": "1.6.1",
"version": "1.7.0",
"description": "A lightweight, zero-dependency drop-in replacement for request",
"main": "index.js",
"files": [

View File

@ -1,11 +1,11 @@
'use strict';
var net = require('net');
var server = net.createServer(function(socket) {
socket.on('data', function(chunk) {
var server = net.createServer(function (socket) {
socket.on('data', function (chunk) {
console.info(chunk.toString('utf8'));
});
});
server.listen(3007, function() {
server.listen(3007, function () {
console.info('Listening on', this.address());
});