Compare commits

...

21 Commits

Author SHA1 Message Date
AJ ONeal 3425dc7a7b
1.9.0-8 2022-09-01 01:09:32 -06:00
AJ ONeal d37a7fa392
f 2022-09-01 01:09:28 -06:00
AJ ONeal 39384ea727
f 2022-09-01 00:59:05 -06:00
AJ ONeal 927576a006
refactor: mv types.js index.js (and rexport request.js) 2022-09-01 00:58:51 -06:00
AJ ONeal 5948928cb8
f 2022-09-01 00:58:18 -06:00
AJ ONeal 2a17ce09eb
refactor: mv index.js request.js 2022-09-01 00:57:57 -06:00
AJ ONeal b06ceb71d6
1.9.0-7 2022-09-01 00:51:03 -06:00
AJ ONeal 8076c667bb
1.9.0-6 2022-09-01 00:49:07 -06:00
AJ ONeal f6a6ddbdcc
fix: add jsconfig.json (again) 2022-09-01 00:49:05 -06:00
AJ ONeal 9aff0c8ff8
1.9.0-5 2022-09-01 00:45:29 -06:00
AJ ONeal c5dfb20a7d
fix: add jsconfig.json 2022-09-01 00:45:26 -06:00
AJ ONeal b08ed4b162
1.9.0-4 2022-09-01 00:33:04 -06:00
AJ ONeal 968926cdc5
fix(types): add RequestOptions 2022-09-01 00:33:01 -06:00
AJ ONeal 88ae1a2086
1.9.0-3 2022-09-01 00:30:19 -06:00
AJ ONeal 2ffc422406
fix(types): body and json can be any 2022-09-01 00:30:14 -06:00
AJ ONeal be8b94c187
1.9.0-2 2022-09-01 00:22:41 -06:00
AJ ONeal a98d4b7341
fix(browser): allow single string url as only argument 2022-09-01 00:22:38 -06:00
AJ ONeal 420eb4f197
1.9.0-1 2022-09-01 00:09:12 -06:00
AJ ONeal 750751966e
lint+feat: add types + minor ws-ish refactor 2022-09-01 00:06:19 -06:00
AJ ONeal e397dcc9cb
1.9.0-0 2022-08-31 23:11:03 -06:00
AJ ONeal 2e233391b9
feat: add fetch-based browser.js 2022-08-31 17:19:02 -06:00
9 changed files with 1042 additions and 686 deletions

14
bin/localize-types Executable file
View File

@ -0,0 +1,14 @@
#!/bin/sh
my_typedefs="$(
grep typedef ./index.js | cut -d ' ' -f5
)"
rm -f ./types.js
{
echo '/**'
for my_type in $my_typedefs; do
echo " * @typedef {import('./').${my_type}} ${my_type}"
done
echo ' */'
} >> ./types.js

201
browser.js Normal file
View File

@ -0,0 +1,201 @@
'use strict';
// `fetch` will be available for node and browsers as a global
//var fetch = window.fetch;
// https://developer.mozilla.org/en-US/docs/Web/API/fetch
// https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch
let _fetchDefaults = {
method: 'GET', // *GET, POST, PATCH, PUT, DELETE, etc
headers: {},
body: undefined, // String, ArrayBuffer, FormData, etc
mode: 'cors', // no-cors, *cors, same-origin
credentials: 'same-origin', // omit, *same-origin, include
cache: 'default', // *default, no-store, reload, no-cache, force-cache, only-if-cached
redirect: 'follow', // *follow, error, manual,
referrer: undefined,
referrerPolicy: 'no-referrer-when-downgrade', // no-referrer, *no-referrer-when-downgrade, same-origin, origin, strict-origin, origin-when-cross-origin, strict-origin-when-cross-origin, unsafe-url
integrity: '',
keepalive: false,
signal: null //
};
let _optionKeys = Object.keys(_fetchDefaults).concat([
//'encoding', // N/A
//'stream', // TODO via getReader
//'json' // handled manually
//'form', // TODO
//'auth' // handled manually
//'formData', // TODO
//'FormData', // TODO
//'userAgent' // not allowed, non-standard for request.js
]);
function setDefaults(_defs) {
return async function request(opts) {
if ('string' === typeof opts) {
opts = { url: opts };
}
let reqOpts = { headers: {} };
if (
opts.body ||
(opts.json && true !== opts.json) ||
opts.form ||
opts.formData
) {
// TODO this is probably a deviation from request's API
// need to check and probably eliminate it
reqOpts.method = (reqOpts.method || 'POST').toUpperCase();
} else {
reqOpts.method = (reqOpts.method || 'GET').toUpperCase();
}
_optionKeys.forEach(function (key) {
if (key in opts) {
if ('undefined' !== typeof opts[key]) {
reqOpts[key] = opts[key];
}
} else if (key in _defs) {
reqOpts[key] = _defs[key];
}
});
if (opts.auth) {
// if opts.uri specifies auth it will be parsed by url.parse and passed directly to the http module
if ('string' !== typeof opts.auth) {
let u = opts.auth.user || opts.auth.username || '';
let p = opts.auth.pass || opts.auth.password || '';
reqOpts.headers.Authorization = encodeBasicAuth(`${u}:${p}`);
} else if ('string' === typeof opts.auth) {
reqOpts.headers.Authorization = encodeBasicAuth(`${opts.auth}`);
}
// [request-compat]
if (opts.auth.bearer) {
// having a shortcut for base64 encoding makes sense,
// but this? Eh, whatevs...
reqOpts.header.Authorization = `Bearer ${opts.auth.bearer}`;
}
}
let body;
if (opts.json && true !== opts.json) {
if (!opts.headers['content-type']) {
opts.headers['content-type'] = 'application/json';
}
body = JSON.stringify(opts.json);
if (!opts.method) {
opts.method = 'POST';
}
}
// The node version will send HTTP Auth by default, but not Cookies.
// We don't have an equivalent option for `fetch`. Furthermore,
// `fetch` caches HTTP Auth Basic across browser refreshes,
// which is not analogous to the node behavior.
//
// "In the face of ambiguity, refuse the temptation to guess"
//
//if (!('credentials' in opts)) {
// opts.credentials = 'include';
//}
if (!('mode' in opts)) {
reqOpts.mode = 'cors';
}
if (!('body' in opts)) {
if (body) {
reqOpts.body = body;
}
}
let resp = await fetch(opts.url, reqOpts);
let result = {
ok: resp.ok,
headers: headersToObj(resp.headers),
body: undefined,
// swapped to match request.js
statusCode: resp.status,
status: resp.statusText,
request: reqOpts,
response: resp
};
result.toJSON = function () {
return {
ok: result.ok,
headers: result.headers,
body: result.body,
statusCode: result.statusCode,
status: result.status
};
};
// return early if there's no body
if (!result.headers['content-type']) {
return result;
}
// TODO blob, formData ?
if (null === opts.encoding) {
return await resp.arrayBuffer();
}
if (!opts.json) {
result.body = await resp.text();
} else {
result.body = await resp.json().catch(async function () {
return await resp.text();
});
}
return result;
};
}
/**
* @param {Iterable.<*>} rheaders
* @returns {Object.<String, String>}
*/
function headersToObj(rheaders) {
/*
Array.from(resp.headers.entries()).forEach(function (h) {
headers[h[0]] = h[1];
});
*/
let headerNames = Array.from(rheaders.keys());
let resHeaders = {};
headerNames.forEach(function (k) {
resHeaders[k] = rheaders.get(k);
});
return resHeaders;
}
/**
* @param {String} utf8
* @returns {String}
*/
function encodeBasicAuth(utf8) {
let b64 = unicodeToBase64(utf8);
return `Basic ${b64}`;
}
/**
* @param {String} utf8
* @returns {String}
*/
function unicodeToBase64(utf8) {
let str = '';
let uint8 = new TextEncoder().encode(utf8);
uint8.forEach(function (b) {
str += String.fromCharCode(b);
});
let b64 = btoa(str);
return b64;
}
let defaultRequest = setDefaults({ mode: 'cors' });
exports.request = defaultRequest;
exports.defaults = setDefaults;
module.exports = defaultRequest;
module.exports.defaults = setDefaults;

710
index.js
View File

@ -1,688 +1,34 @@
'use strict'; 'use strict';
var http = require('http'); module.exports = require('./request.js');
var https = require('https');
var url = require('url');
var os = require('os');
var pkg = require('./package.json');
var fs = require('fs'); // only for streams
function debug() { /**
if (module.exports.debug) { * @callback Request
console.log.apply(console, arguments); * @param {RequestOptions} opts
} * @returns {Response}
} */
function mergeOrDelete(defaults, updates) { /**
Object.keys(defaults).forEach(function (key) { * @typedef {Object} RequestOptions
if (!(key in updates)) { * @prop {any} [opts.body]
updates[key] = defaults[key]; * @prop {Object.<String,any>} [opts.form]
return; * @prop {Headers} [opts.headers]
} * @prop {String} [opts.method]
* @prop {Boolean | any} [opts.json]
* @prop {String} opts.url
*/
// neither accept the prior default nor define an explicit value /**
// CRDT probs... * @typedef {Object} Response
if ('undefined' === typeof updates[key]) { * @prop {any} body
delete updates[key]; * @prop {Headers} headers
} else if ( * @prop {Boolean} ok
'object' === typeof defaults[key] && * @prop {any} [response] - TODO (browser only)
'object' === typeof updates[key] * @prop {any} [request] - TODO
) { * @prop {String} status
updates[key] = mergeOrDelete(defaults[key], updates[key]); * @prop {Number} statusCode
} */
});
return updates; /**
} * @typedef {Object.<String,String|Array<String>>} Headers
*/
// retrieves an existing header, case-sensitive
function getHeaderName(reqOpts, header) {
var headerNames = {};
Object.keys(reqOpts.headers).forEach(function (casedName) {
headerNames[casedName.toLowerCase()] = casedName;
});
// returns the key, which in erroneous cases could be an empty string
return headerNames[header.toLowerCase()];
}
// returns whether or not a header exists, case-insensitive
function hasHeader(reqOpts, header) {
return 'undefined' !== typeof getHeaderName(reqOpts, header);
}
function toJSONifier(keys) {
return function () {
var obj = {};
var me = this;
keys.forEach(function (key) {
if (me[key] && 'function' === typeof me[key].toJSON) {
obj[key] = me[key].toJSON();
} else {
obj[key] = me[key];
}
});
return obj;
};
}
function setupPipe(resp, opts) {
// make the response await-able
var resolve;
var reject;
var p = new Promise(function (_resolve, _reject) {
resolve = _resolve;
reject = _reject;
});
// or an existing write stream
if ('function' === typeof opts.stream.pipe) {
if (opts.debug) {
console.debug('[@root/request] stream piped');
}
resp.pipe(opts.stream);
}
resp.once('error', function (e) {
if (opts.debug) {
console.debug("[@root/request] stream 'error'");
console.error(e.stack);
}
resp.destroy();
if ('function' === opts.stream.destroy) {
opts.stream.destroy(e);
}
reject(e);
});
resp.once('end', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'end'");
}
if ('function' === opts.stream.destroy) {
opts.stream.end();
// this will close the stream (i.e. sync to disk)
opts.stream.destroy();
}
});
resp.once('close', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'close'");
}
resolve();
});
return p;
}
function handleResponse(resp, opts, cb) {
// body can be buffer, string, or json
if (null === opts.encoding) {
resp._body = [];
} else {
resp.body = '';
}
resp._bodyLength = 0;
resp.once('error', function (err) {
cb(err, resp, resp.body);
});
resp.on('readable', function () {
var chunk;
while ((chunk = resp.read())) {
if ('string' === typeof resp.body) {
resp.body += chunk.toString(opts.encoding);
} else {
resp._body.push(chunk);
resp._bodyLength += chunk.length;
}
}
});
resp.once('end', function () {
if ('string' !== typeof resp.body) {
if (1 === resp._body.length) {
resp.body = resp._body[0];
} else {
resp.body = Buffer.concat(resp._body, resp._bodyLength);
}
resp._body = null;
}
if (opts.json && 'string' === typeof resp.body) {
// TODO I would parse based on Content-Type
// but request.js doesn't do that.
try {
resp.body = JSON.parse(resp.body);
} catch (e) {
// ignore
}
}
debug('\n[urequest] resp.toJSON():');
if (module.exports.debug) {
debug(resp.toJSON());
}
if (opts.debug) {
console.debug('[@root/request] Response Body:');
console.debug(resp.body);
}
cb(null, resp, resp.body);
});
}
function setDefaults(defs) {
defs = defs || {};
function urequestHelper(opts, cb) {
debug('\n[urequest] processed options:');
debug(opts);
var req;
var finalOpts = {};
// allow specifying a file
if ('string' === typeof opts.stream) {
if (opts.debug) {
console.debug('[@root/request] creating file write stream');
}
try {
opts.stream = fs.createWriteStream(opts.stream);
} catch (e) {
cb(e);
return;
}
}
function onResponse(resp) {
var followRedirect;
Object.keys(defs).forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
return;
}
opts[key] = defs[key];
});
followRedirect = opts.followRedirect;
// copied from WHATWG fetch
resp.ok = false;
if (resp.statusCode >= 200 && resp.statusCode < 300) {
resp.ok = true;
}
resp.toJSON = toJSONifier([
'statusCode',
'body',
'ok',
'headers',
'request'
]);
resp.request = req;
resp.request.uri = url.parse(opts.url);
//resp.request.method = opts.method;
resp.request.headers = finalOpts.headers;
resp.request.toJSON = toJSONifier(['uri', 'method', 'headers']);
if (opts.debug) {
console.debug('[@root/request] Response Headers:');
console.debug(resp.toJSON());
}
if (
followRedirect &&
resp.headers.location &&
-1 !== [301, 302, 307, 308].indexOf(resp.statusCode)
) {
debug('Following redirect: ' + resp.headers.location);
if ('GET' !== opts.method && !opts.followAllRedirects) {
followRedirect = false;
}
if (opts._redirectCount >= opts.maxRedirects) {
followRedirect = false;
}
if ('function' === opts.followRedirect) {
if (!opts.followRedirect(resp)) {
followRedirect = false;
}
}
if (followRedirect) {
if (!opts.followOriginalHttpMethod) {
opts.method = 'GET';
opts.body = null;
delete opts.headers[
getHeaderName(opts, 'Content-Length')
];
delete opts.headers[
getHeaderName(opts, 'Transfer-Encoding')
];
}
if (opts.removeRefererHeader && opts.headers) {
delete opts.headers.referer;
}
// TODO needs baseUrl, maybe test for host / socketPath?
opts.url = resp.headers.location;
opts.uri = url.parse(opts.url);
return urequestHelper(opts, cb);
}
}
if (opts.stream) {
resp.stream = setupPipe(resp, opts);
// can be string, buffer, or json... why not an async function too?
resp.stream.body = async function () {
handleResponse(resp, opts, cb);
await resp.stream;
return resp.body;
};
cb(null, resp);
return;
}
handleResponse(resp, opts, cb);
}
var _body;
var MyFormData;
var form;
var formHeaders;
var requester;
if (opts.body) {
if (true === opts.json && 'string' !== typeof opts.body) {
_body = JSON.stringify(opts.body);
} else {
_body = opts.body;
}
} else if (opts.json && true !== opts.json) {
_body = JSON.stringify(opts.json);
} else if (opts.form) {
_body = Object.keys(opts.form)
.filter(function (key) {
if ('undefined' !== typeof opts.form[key]) {
return true;
}
})
.map(function (key) {
return (
encodeURIComponent(key) +
'=' +
encodeURIComponent(String(opts.form[key]))
);
})
.join('&');
opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
}
if ('string' === typeof _body) {
_body = Buffer.from(_body);
}
Object.keys(opts.uri).forEach(function (key) {
finalOpts[key] = opts.uri[key];
});
// A bug should be raised if request does it differently,
// but I think we're supposed to pass all acceptable options
// on to the raw http request
[
'family',
'host',
'localAddress',
'agent',
'createConnection',
'timeout',
'setHost'
].forEach(function (key) {
if (key in opts) {
finalOpts[key] = opts[key];
}
});
finalOpts.method = opts.method;
finalOpts.headers = JSON.parse(JSON.stringify(opts.headers));
var uaHeader = getHeaderName(finalOpts, 'User-Agent') || 'User-Agent';
// set a default user-agent
if (!finalOpts.headers[uaHeader]) {
if (false === finalOpts.headers[uaHeader]) {
delete finalOpts.headers[uaHeader];
} else {
finalOpts.headers[uaHeader] = getUserAgent(opts.userAgent);
}
}
if (_body) {
// Most APIs expect (or require) Content-Length except in the case of multipart uploads
// Transfer-Encoding: Chunked (the default) is generally only well-supported downstream
if (
'undefined' !== typeof _body.byteLength ||
'undefined' !== typeof _body.length
) {
finalOpts.headers['Content-Length'] =
_body.byteLength || _body.length;
}
}
if (opts.auth) {
// if opts.uri specifies auth it will be parsed by url.parse and passed directly to the http module
if ('string' !== typeof opts.auth) {
opts.auth =
(opts.auth.user || opts.auth.username || '') +
':' +
(opts.auth.pass || opts.auth.password || '');
}
if ('string' === typeof opts.auth) {
finalOpts.auth = opts.auth;
}
if (false === opts.sendImmediately) {
console.warn(
'[Warn] setting `sendImmediately: false` is not yet supported. Please open an issue if this is an important feature that you need.'
);
}
// [request-compat]
if (opts.auth.bearer) {
// having a shortcut for base64 encoding makes sense, but this? Eh, whatevs...
finalOpts.header.Authorization = 'Bearer ' + opts.auth.bearer;
}
}
if (opts.formData) {
try {
MyFormData = opts.FormData || require('form-data');
// potential options https://github.com/felixge/node-combined-stream/blob/master/lib/combined_stream.js#L7-L15
} catch (e) {
console.error(
'urequest does not include extra dependencies by default'
);
console.error(
"if you need to use 'form-data' you may install it, like so:"
);
console.error(' npm install --save form-data');
cb(e);
return;
}
try {
form = new MyFormData();
Object.keys(opts.formData).forEach(function (key) {
function add(key, data, opts) {
if (data.value) {
opts = data.options;
data = data.value;
}
form.append(key, data, opts);
}
if (Array.isArray(opts.formData[key])) {
opts.formData[key].forEach(function (data) {
add(key, data);
});
} else {
add(key, opts.formData[key]);
}
});
} catch (e) {
cb(e);
return;
}
formHeaders = form.getHeaders();
Object.keys(formHeaders).forEach(function (header) {
finalOpts.headers[header] = formHeaders[header];
});
}
// TODO support unix sockets
if ('https:' === finalOpts.protocol) {
// https://nodejs.org/api/https.html#https_https_request_options_callback
debug('\n[urequest] https.request(opts):');
debug(finalOpts);
requester = https;
} else if ('http:' === finalOpts.protocol) {
// https://nodejs.org/api/http.html#http_http_request_options_callback
debug('\n[urequest] http.request(opts):');
debug(finalOpts);
requester = http;
} else {
cb(new Error("unknown protocol: '" + opts.uri.protocol + "'"));
return;
}
if (form) {
debug("\n[urequest] '" + finalOpts.method + "' (request) form");
debug(formHeaders);
// generally uploads don't use Chunked Encoding (some systems have issues with it)
// and I don't want to do the work to calculate the content-lengths. This seems to work.
req = form.submit(finalOpts, function (err, resp) {
if (err) {
cb(err);
return;
}
onResponse(resp);
resp.resume();
});
//req = requester.request(finalOpts, onResponse);
//req.on('error', cb);
//form.pipe(req);
return;
}
if (opts.debug) {
console.debug('');
console.debug('[@root/request] Request Options:');
console.debug(finalOpts);
if (_body) {
console.debug('[@root/request] Request Body:');
console.debug(
opts.body || opts.form || opts.formData || opts.json
);
}
}
req = requester.request(finalOpts, onResponse);
req.once('error', cb);
if (_body) {
debug("\n[urequest] '" + finalOpts.method + "' (request) body");
debug(_body);
if ('function' === typeof _body.pipe) {
// used for chunked encoding
_body.pipe(req);
_body.once('error', function (err) {
// https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
// if the Readable stream emits an error during processing,
// the Writable destination is not closed automatically
_body.destroy();
req.destroy(err);
});
} else {
// used for known content-length
req.end(_body);
}
} else {
req.end();
}
}
function parseUrl(str) {
var obj = url.parse(str);
var paths;
if ('unix' !== (obj.hostname || obj.host || '').toLowerCase()) {
return obj;
}
obj.href = null;
obj.hostname = obj.host = null;
paths = (obj.pathname || obj.path || '').split(':');
obj.socketPath = paths.shift();
obj.pathname = obj.path = paths.join(':');
return obj;
}
function urequest(opts, cb) {
debug('\n[urequest] received options:');
debug(opts);
var reqOpts = {};
// request.js behavior:
// encoding: null + json ? unknown
// json => attempt to parse, fail silently
// encoding => buffer.toString(encoding)
// null === encoding => Buffer.concat(buffers)
if ('string' === typeof opts) {
opts = { url: opts };
}
module.exports._keys.forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
reqOpts[key] = opts[key];
} else if (key in defs) {
reqOpts[key] = defs[key];
}
});
// TODO url.resolve(defs.baseUrl, opts.url);
if ('string' === typeof opts.url || 'string' === typeof opts.uri) {
if ('string' === typeof opts.url) {
reqOpts.url = opts.url;
reqOpts.uri = parseUrl(opts.url);
} else if ('string' === typeof opts.uri) {
reqOpts.url = opts.uri;
reqOpts.uri = parseUrl(opts.uri);
}
} else {
if ('object' === typeof opts.uri) {
reqOpts.url = url.format(opts.uri);
reqOpts.uri = opts.uri;
//reqOpts.uri = url.parse(reqOpts.uri);
} else if ('object' === typeof opts.url) {
reqOpts.url = url.format(opts.url);
reqOpts.uri = opts.url;
//reqOpts.uri = url.parse(reqOpts.url);
}
}
if (
opts.body ||
(opts.json && true !== opts.json) ||
opts.form ||
opts.formData
) {
// TODO this is probably a deviation from request's API
// need to check and probably eliminate it
reqOpts.method = (reqOpts.method || 'POST').toUpperCase();
} else {
reqOpts.method = (reqOpts.method || 'GET').toUpperCase();
}
if (!reqOpts.headers) {
reqOpts.headers = {};
}
// crazy case for easier testing
if (!hasHeader(reqOpts, 'CoNTeNT-TyPe')) {
if (
(true === reqOpts.json && reqOpts.body) ||
(true !== reqOpts.json && reqOpts.json)
) {
reqOpts.headers['Content-Type'] = 'application/json';
}
}
if (opts.debug) {
reqOpts.debug = opts.debug;
}
return urequestHelper(reqOpts, cb);
}
function smartPromisify(fn) {
return function (opts) {
var cb;
if ('function' === typeof arguments[1]) {
cb = Array.prototype.slice.call(arguments)[1];
return fn(opts, cb);
}
return new Promise(function (resolve, reject) {
fn(opts, function (err, resp) {
if (err) {
err._response = resp;
reject(err);
return;
}
resolve(resp);
});
});
};
}
var smartUrequest = smartPromisify(urequest);
smartUrequest.defaults = function (_defs) {
_defs = mergeOrDelete(defs, _defs);
return setDefaults(_defs);
};
['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
function (method) {
smartUrequest[method] = smartPromisify(function (obj, cb) {
if ('string' === typeof obj) {
obj = { url: obj };
}
obj.method = method.toUpperCase();
urequest(obj, cb);
});
}
);
smartUrequest.del = urequest.delete;
return smartUrequest;
}
var nodeUa =
'@root+request/' +
pkg.version +
' ' +
process.release.name +
'/' +
process.version +
' ' +
os.platform() +
'/' +
os.release() +
' ' +
os.type() +
'/' +
process.arch;
function getUserAgent(additional) {
// See https://tools.ietf.org/html/rfc8555#section-6.1
// And https://tools.ietf.org/html/rfc7231#section-5.5.3
// And https://community.letsencrypt.org/t/user-agent-flag-explained/3843/2
var ua = nodeUa;
if (additional) {
ua = additional + ' ' + ua;
}
return ua;
}
var _defaults = {
sendImmediately: true,
method: '',
headers: {},
useQuerystring: false,
followRedirect: true,
followAllRedirects: false,
followOriginalHttpMethod: false,
maxRedirects: 10,
removeRefererHeader: false,
// encoding: undefined,
// stream: false, // TODO allow a stream?
gzip: false
//, body: undefined
//, json: undefined
};
module.exports = setDefaults(_defaults);
module.exports._keys = Object.keys(_defaults).concat([
'encoding',
'stream',
'body',
'json',
'form',
'auth',
'formData',
'FormData',
'userAgent' // non-standard for request.js
]);
module.exports.debug =
-1 !== (process.env.NODE_DEBUG || '').split(/\s+/g).indexOf('urequest');
debug('DEBUG ON for urequest');

76
jsconfig.json Normal file
View File

@ -0,0 +1,76 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
"target": "es2021" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */,
"module": "commonjs",
// "lib": [], /* Specify library files to be included in the compilation. */
"allowJs": true /* Allow javascript files to be compiled. */,
"checkJs": true /* Report errors in .js files. */,
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
// "outDir": "./", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
"noEmit": true /* Do not emit outputs. */,
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true /* Enable all strict type-checking options. */,
"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
"alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
/* Additional Checks */
"noUnusedLocals": true /* Report errors on unused locals. */,
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
/* Module Resolution Options */
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
"typeRoots": [
"./typings",
"./node_modules/@types"
] /* List of folders to include type definitions from. */,
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
"preserveSymlinks": false /* Do not resolve the real path of symlinks. */,
"maxNodeModuleJsDepth": 20,
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
/* Advanced Options */
"skipLibCheck": true /* Skip type checking of declaration files. */,
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
},
"include": ["index.js", "browser.js", "./types.js", "./local-types.js", "lib/**/*.js", "bin/**/*.js"],
"exclude": ["node_modules"]
}

5
local-types.js Normal file
View File

@ -0,0 +1,5 @@
/**
* @typedef {import('./types.js').RequestOptions} RequestOptions
* @typedef {import('./types.js').Response} Response
* @typedef {import('./types.js').Headers} Headers
*/

2
package-lock.json generated
View File

@ -1,5 +1,5 @@
{ {
"name": "@root/request", "name": "@root/request",
"version": "1.8.3", "version": "1.9.0-8",
"lockfileVersion": 1 "lockfileVersion": 1
} }

View File

@ -1,16 +1,25 @@
{ {
"name": "@root/request", "name": "@root/request",
"version": "1.8.3", "version": "1.9.0-8",
"description": "A lightweight, zero-dependency drop-in replacement for request", "description": "A lightweight, zero-dependency drop-in replacement for request",
"main": "index.js", "main": "index.js",
"browser": {
"request.js": "browser.js"
},
"files": [ "files": [
"lib" "browser.js",
"jsconfig.json",
"lib",
"local-types.js",
"request.js",
"types.js"
], ],
"directories": { "directories": {
"example": "examples" "example": "examples"
}, },
"scripts": { "scripts": {
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1",
"prepublish": "./bin/localize-types"
}, },
"repository": { "repository": {
"type": "git", "type": "git",

700
request.js Normal file
View File

@ -0,0 +1,700 @@
'use strict';
var http = require('http');
var https = require('https');
var url = require('url');
var os = require('os');
var pkg = require('./package.json');
var fs = require('fs'); // only for streams
var _defaults = {
sendImmediately: true,
method: '',
headers: {},
useQuerystring: false,
followRedirect: true,
followAllRedirects: false,
followOriginalHttpMethod: false,
maxRedirects: 10,
removeRefererHeader: false,
// encoding: undefined,
// stream: false, // TODO allow a stream?
gzip: false
//, body: undefined
//, json: undefined
};
var _keys = Object.keys(_defaults).concat([
'encoding',
'stream',
'body',
'json',
'form',
'auth',
'formData',
'FormData',
'userAgent' // non-standard for request.js
]);
function debug() {
if (module.exports.debug) {
console.log.apply(console, arguments);
}
}
function mergeOrDelete(defaults, updates) {
Object.keys(defaults).forEach(function (key) {
if (!(key in updates)) {
updates[key] = defaults[key];
return;
}
// neither accept the prior default nor define an explicit value
// CRDT probs...
if ('undefined' === typeof updates[key]) {
delete updates[key];
} else if (
'object' === typeof defaults[key] &&
'object' === typeof updates[key]
) {
updates[key] = mergeOrDelete(defaults[key], updates[key]);
}
});
return updates;
}
// retrieves an existing header, case-sensitive
function getHeaderName(reqOpts, header) {
var headerNames = {};
Object.keys(reqOpts.headers).forEach(function (casedName) {
headerNames[casedName.toLowerCase()] = casedName;
});
// returns the key, which in erroneous cases could be an empty string
return headerNames[header.toLowerCase()];
}
// returns whether or not a header exists, case-insensitive
function hasHeader(reqOpts, header) {
return 'undefined' !== typeof getHeaderName(reqOpts, header);
}
function toJSONifier(keys) {
return function () {
var obj = {};
var me = this;
keys.forEach(function (key) {
if (me[key] && 'function' === typeof me[key].toJSON) {
obj[key] = me[key].toJSON();
} else {
obj[key] = me[key];
}
});
return obj;
};
}
function setupPipe(resp, opts) {
// make the response await-able
var resolve;
var reject;
var p = new Promise(function (_resolve, _reject) {
resolve = _resolve;
reject = _reject;
});
// or an existing write stream
if ('function' === typeof opts.stream.pipe) {
if (opts.debug) {
console.debug('[@root/request] stream piped');
}
resp.pipe(opts.stream);
}
resp.once('error', function (e) {
if (opts.debug) {
console.debug("[@root/request] stream 'error'");
console.error(e.stack);
}
resp.destroy();
if ('function' === opts.stream.destroy) {
opts.stream.destroy(e);
}
reject(e);
});
resp.once('end', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'end'");
}
if ('function' === opts.stream.destroy) {
opts.stream.end();
// this will close the stream (i.e. sync to disk)
opts.stream.destroy();
}
});
resp.once('close', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'close'");
}
resolve();
});
return p;
}
function handleResponse(resp, opts, cb) {
// body can be buffer, string, or json
if (null === opts.encoding) {
resp._body = [];
} else {
resp.body = '';
}
resp._bodyLength = 0;
resp.once('error', function (err) {
cb(err, resp, resp.body);
});
resp.on('readable', function () {
var chunk;
while ((chunk = resp.read())) {
if ('string' === typeof resp.body) {
resp.body += chunk.toString(opts.encoding);
} else {
resp._body.push(chunk);
resp._bodyLength += chunk.length;
}
}
});
resp.once('end', function () {
if ('string' !== typeof resp.body) {
if (1 === resp._body.length) {
resp.body = resp._body[0];
} else {
resp.body = Buffer.concat(resp._body, resp._bodyLength);
}
resp._body = null;
}
if (opts.json && 'string' === typeof resp.body) {
// TODO I would parse based on Content-Type
// but request.js doesn't do that.
try {
resp.body = JSON.parse(resp.body);
} catch (e) {
// ignore
}
}
debug('\n[urequest] resp.toJSON():');
if (module.exports.debug) {
debug(resp.toJSON());
}
if (opts.debug) {
console.debug('[@root/request] Response Body:');
console.debug(resp.body);
}
cb(null, resp, resp.body);
});
}
/**
* @param {any} defs - TODO enumerate defaults
* @returns {Request}
*/
function setDefaults(defs) {
defs = defs || {};
/** @type {Request} */
function urequestHelper(opts, cb) {
//jshint maxcomplexity:42
debug('\n[urequest] processed options:');
debug(opts);
var req;
var finalOpts = {};
// allow specifying a file
if ('string' === typeof opts.stream) {
if (opts.debug) {
console.debug('[@root/request] creating file write stream');
}
try {
opts.stream = fs.createWriteStream(opts.stream);
} catch (e) {
cb(e);
return;
}
}
function onResponse(resp) {
var followRedirect;
Object.keys(defs).forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
return;
}
opts[key] = defs[key];
});
followRedirect = opts.followRedirect;
// copied from WHATWG fetch
resp.ok = false;
if (resp.statusCode >= 200 && resp.statusCode < 300) {
resp.ok = true;
}
resp.toJSON = toJSONifier([
'statusCode',
'body',
'ok',
'headers',
'request'
]);
resp.request = req;
resp.request.uri = url.parse(opts.url);
//resp.request.method = opts.method;
resp.request.headers = finalOpts.headers;
resp.request.toJSON = toJSONifier(['uri', 'method', 'headers']);
if (opts.debug) {
console.debug('[@root/request] Response Headers:');
console.debug(resp.toJSON());
}
if (
followRedirect &&
resp.headers.location &&
-1 !== [301, 302, 307, 308].indexOf(resp.statusCode)
) {
debug('Following redirect: ' + resp.headers.location);
if ('GET' !== opts.method && !opts.followAllRedirects) {
followRedirect = false;
}
if (opts._redirectCount >= opts.maxRedirects) {
followRedirect = false;
}
if ('function' === opts.followRedirect) {
if (!opts.followRedirect(resp)) {
followRedirect = false;
}
}
if (followRedirect) {
if (!opts.followOriginalHttpMethod) {
opts.method = 'GET';
opts.body = null;
delete opts.headers[
getHeaderName(opts, 'Content-Length')
];
delete opts.headers[
getHeaderName(opts, 'Transfer-Encoding')
];
}
if (opts.removeRefererHeader && opts.headers) {
delete opts.headers.referer;
}
// TODO needs baseUrl, maybe test for host / socketPath?
opts.url = resp.headers.location;
opts.uri = url.parse(opts.url);
return urequestHelper(opts, cb);
}
}
if (opts.stream) {
resp.stream = setupPipe(resp, opts);
// can be string, buffer, or json... why not an async function too?
resp.stream.body = async function () {
handleResponse(resp, opts, cb);
await resp.stream;
return resp.body;
};
cb(null, resp);
return;
}
handleResponse(resp, opts, cb);
}
var _body;
var MyFormData;
var form;
var formHeaders;
var requester;
if (opts.body) {
if (true === opts.json && 'string' !== typeof opts.body) {
_body = JSON.stringify(opts.body);
} else {
_body = opts.body;
}
} else if (opts.json && true !== opts.json) {
_body = JSON.stringify(opts.json);
} else if (opts.form) {
_body = Object.keys(opts.form)
.filter(function (key) {
if ('undefined' !== typeof opts.form[key]) {
return true;
}
})
.map(function (key) {
return (
encodeURIComponent(key) +
'=' +
encodeURIComponent(String(opts.form[key]))
);
})
.join('&');
opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
}
if ('string' === typeof _body) {
_body = Buffer.from(_body);
}
Object.keys(opts.uri).forEach(function (key) {
finalOpts[key] = opts.uri[key];
});
// A bug should be raised if request does it differently,
// but I think we're supposed to pass all acceptable options
// on to the raw http request
[
'family',
'host',
'localAddress',
'agent',
'createConnection',
'timeout',
'setHost'
].forEach(function (key) {
if (key in opts) {
finalOpts[key] = opts[key];
}
});
finalOpts.method = opts.method;
finalOpts.headers = JSON.parse(JSON.stringify(opts.headers));
var uaHeader = getHeaderName(finalOpts, 'User-Agent') || 'User-Agent';
// set a default user-agent
if (!finalOpts.headers[uaHeader]) {
if (false === finalOpts.headers[uaHeader]) {
delete finalOpts.headers[uaHeader];
} else {
finalOpts.headers[uaHeader] = getUserAgent(opts.userAgent);
}
}
if (_body) {
// Most APIs expect (or require) Content-Length except in the case of multipart uploads
// Transfer-Encoding: Chunked (the default) is generally only well-supported downstream
if (
'undefined' !== typeof _body.byteLength ||
'undefined' !== typeof _body.length
) {
finalOpts.headers['Content-Length'] =
_body.byteLength || _body.length;
}
}
if (opts.auth) {
// if opts.uri specifies auth it will be parsed by url.parse and passed directly to the http module
if ('string' !== typeof opts.auth) {
opts.auth =
(opts.auth.user || opts.auth.username || '') +
':' +
(opts.auth.pass || opts.auth.password || '');
}
if ('string' === typeof opts.auth) {
finalOpts.auth = opts.auth;
}
if (false === opts.sendImmediately) {
console.warn(
'[Warn] setting `sendImmediately: false` is not yet supported. Please open an issue if this is an important feature that you need.'
);
}
// [request-compat]
if (opts.auth.bearer) {
// having a shortcut for base64 encoding makes sense, but this? Eh, whatevs...
finalOpts.header.Authorization = 'Bearer ' + opts.auth.bearer;
}
}
if (opts.formData) {
try {
MyFormData = opts.FormData || require('form-data');
// potential options https://github.com/felixge/node-combined-stream/blob/master/lib/combined_stream.js#L7-L15
} catch (e) {
console.error(
'urequest does not include extra dependencies by default'
);
console.error(
"if you need to use 'form-data' you may install it, like so:"
);
console.error(' npm install --save form-data');
cb(e);
return;
}
try {
form = new MyFormData();
Object.keys(opts.formData).forEach(function (key) {
function add(key, data, opts) {
if (data.value) {
opts = data.options;
data = data.value;
}
form.append(key, data, opts);
}
if (Array.isArray(opts.formData[key])) {
opts.formData[key].forEach(function (data) {
add(key, data);
});
} else {
add(key, opts.formData[key]);
}
});
} catch (e) {
cb(e);
return;
}
formHeaders = form.getHeaders();
Object.keys(formHeaders).forEach(function (header) {
finalOpts.headers[header] = formHeaders[header];
});
}
// TODO support unix sockets
if ('https:' === finalOpts.protocol) {
// https://nodejs.org/api/https.html#https_https_request_options_callback
debug('\n[urequest] https.request(opts):');
debug(finalOpts);
requester = https;
} else if ('http:' === finalOpts.protocol) {
// https://nodejs.org/api/http.html#http_http_request_options_callback
debug('\n[urequest] http.request(opts):');
debug(finalOpts);
requester = http;
} else {
cb(new Error("unknown protocol: '" + opts.uri.protocol + "'"));
return;
}
if (form) {
debug("\n[urequest] '" + finalOpts.method + "' (request) form");
debug(formHeaders);
// generally uploads don't use Chunked Encoding (some systems have issues with it)
// and I don't want to do the work to calculate the content-lengths. This seems to work.
req = form.submit(finalOpts, function (err, resp) {
if (err) {
cb(err);
return;
}
onResponse(resp);
resp.resume();
});
//req = requester.request(finalOpts, onResponse);
//req.on('error', cb);
//form.pipe(req);
return;
}
if (opts.debug) {
console.debug('');
console.debug('[@root/request] Request Options:');
console.debug(finalOpts);
if (_body) {
console.debug('[@root/request] Request Body:');
console.debug(
opts.body || opts.form || opts.formData || opts.json
);
}
}
req = requester.request(finalOpts, onResponse);
req.once('error', cb);
if (_body) {
debug("\n[urequest] '" + finalOpts.method + "' (request) body");
debug(_body);
if ('function' === typeof _body.pipe) {
// used for chunked encoding
_body.pipe(req);
_body.once('error', function (err) {
// https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
// if the Readable stream emits an error during processing,
// the Writable destination is not closed automatically
_body.destroy();
req.destroy(err);
});
} else {
// used for known content-length
req.end(_body);
}
} else {
req.end();
}
}
function parseUrl(str) {
var obj = url.parse(str);
var paths;
if ('unix' !== (obj.hostname || obj.host || '').toLowerCase()) {
return obj;
}
obj.href = null;
obj.hostname = obj.host = null;
paths = (obj.pathname || obj.path || '').split(':');
obj.socketPath = paths.shift();
obj.pathname = obj.path = paths.join(':');
return obj;
}
function urequest(opts, cb) {
debug('\n[urequest] received options:');
debug(opts);
var reqOpts = {};
// request.js behavior:
// encoding: null + json ? unknown
// json => attempt to parse, fail silently
// encoding => buffer.toString(encoding)
// null === encoding => Buffer.concat(buffers)
if ('string' === typeof opts) {
opts = { url: opts };
}
_keys.forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
reqOpts[key] = opts[key];
} else if (key in defs) {
reqOpts[key] = defs[key];
}
});
// TODO url.resolve(defs.baseUrl, opts.url);
if ('string' === typeof opts.url || 'string' === typeof opts.uri) {
if ('string' === typeof opts.url) {
reqOpts.url = opts.url;
reqOpts.uri = parseUrl(opts.url);
} else if ('string' === typeof opts.uri) {
reqOpts.url = opts.uri;
reqOpts.uri = parseUrl(opts.uri);
}
} else {
if ('object' === typeof opts.uri) {
reqOpts.url = url.format(opts.uri);
reqOpts.uri = opts.uri;
//reqOpts.uri = url.parse(reqOpts.uri);
} else if ('object' === typeof opts.url) {
reqOpts.url = url.format(opts.url);
reqOpts.uri = opts.url;
//reqOpts.uri = url.parse(reqOpts.url);
}
}
if (
opts.body ||
(opts.json && true !== opts.json) ||
opts.form ||
opts.formData
) {
// TODO this is probably a deviation from request's API
// need to check and probably eliminate it
reqOpts.method = (reqOpts.method || 'POST').toUpperCase();
} else {
reqOpts.method = (reqOpts.method || 'GET').toUpperCase();
}
if (!reqOpts.headers) {
reqOpts.headers = {};
}
// crazy case for easier testing
if (!hasHeader(reqOpts, 'CoNTeNT-TyPe')) {
if (
(true === reqOpts.json && reqOpts.body) ||
(true !== reqOpts.json && reqOpts.json)
) {
reqOpts.headers['Content-Type'] = 'application/json';
}
}
if (opts.debug) {
reqOpts.debug = opts.debug;
}
return urequestHelper(reqOpts, cb);
}
function smartPromisify(fn) {
return function (opts) {
var cb;
if ('function' === typeof arguments[1]) {
cb = Array.prototype.slice.call(arguments)[1];
return fn(opts, cb);
}
return new Promise(function (resolve, reject) {
fn(opts, function (err, resp) {
if (err) {
err._response = resp;
reject(err);
return;
}
resolve(resp);
});
});
};
}
var smartUrequest = smartPromisify(urequest);
smartUrequest.defaults = function (_defs) {
_defs = mergeOrDelete(defs, _defs);
return setDefaults(_defs);
};
['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
function (method) {
smartUrequest[method] = smartPromisify(function (obj, cb) {
if ('string' === typeof obj) {
obj = { url: obj };
}
obj.method = method.toUpperCase();
urequest(obj, cb);
});
}
);
smartUrequest.del = urequest.delete;
return smartUrequest;
}
var nodeUa =
'@root+request/' +
pkg.version +
' ' +
process.release.name +
'/' +
process.version +
' ' +
os.platform() +
'/' +
os.release() +
' ' +
os.type() +
'/' +
process.arch;
function getUserAgent(additional) {
// See https://tools.ietf.org/html/rfc8555#section-6.1
// And https://tools.ietf.org/html/rfc7231#section-5.5.3
// And https://community.letsencrypt.org/t/user-agent-flag-explained/3843/2
var ua = nodeUa;
if (additional) {
ua = additional + ' ' + ua;
}
return ua;
}
exports.request = setDefaults(_defaults);
exports._keys = _keys;
module.exports = exports.request;
module.exports._keys = _keys;
module.exports.debug =
-1 !== (process.env.NODE_DEBUG || '').split(/\s+/g).indexOf('urequest');
debug('DEBUG ON for urequest');

5
types.js Normal file
View File

@ -0,0 +1,5 @@
/**
* @typedef {import('./').RequestOptions} RequestOptions
* @typedef {import('./').Response} Response
* @typedef {import('./').Headers} Headers
*/