v1.2.0: bugfix for buckets with invalid domain names, and pass request options

- Buckets with names like 'example.bucket' and 'example_bucket' are valid as paths, but not as domain names. - allow passthrough support for latest @root/request, which supports pipes and streams
make Prettier
2021-01-26 16:57:20 -07:00 · 2021-01-26 16:48:15 -07:00
6 changed files with 183 additions and 76 deletions
--- a/README.md
+++ b/README.md
@ -13,8 +13,28 @@ A lightweight alternative to the S3 SDK that uses only @root/request and aws4.

 ### Download a file from S3

+This library supports the same streaming options as [@root/request.js](https://git.rootprojects.org/root/request.js).
+
+#### as a stream
+
 ```js
-s3.get({
+var resp = await s3.get({
+    accessKeyId,        // 'AKIAXXXXXXXXXXXXXXXX'
+    secretAccessKey,    // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+    region,             // 'us-east-2'
+    bucket,             // 'bucket-name'
+    prefix,             // 'my-prefix/' (optional)
+    key,                // 'data/stats.csv' (omits prefix, if any)
+    stream              // fs.createWriteStream('./path/to/file.bin')
+});
+
+await resp.stream;
+```
+
+#### in-memory
+
+```js
+var resp = await s3.get({
    accessKeyId,        // 'AKIAXXXXXXXXXXXXXXXX'
    secretAccessKey,    // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
    region,             // 'us-east-2'
@ -22,12 +42,14 @@ s3.get({
    prefix,             // 'my-prefix/' (optional)
    key                 // 'data/stats.csv' (omits prefix, if any)
 });
+
+fs.writeFile(resp.body, './path/to/file.bin');
 ```

 ### Upload a new file to S3

 ```js
-s3.set({
+await s3.set({
    accessKeyId,
    secretAccessKey,
    region,
@ -41,6 +63,36 @@ s3.set({
 });
 ```

+### Check that a file exists
+
+```js
+var resp = await s3.head({
+    accessKeyId,        // 'AKIAXXXXXXXXXXXXXXXX'
+    secretAccessKey,    // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+    region,             // 'us-east-2'
+    bucket,             // 'bucket-name'
+    prefix,             // 'my-prefix/' (optional)
+    key                 // 'data/stats.csv' (omits prefix, if any)
+});
+
+console.log(resp.headers);
+```
+
+### Delete file
+
+```js
+var resp = await s3.delete({
+    accessKeyId,        // 'AKIAXXXXXXXXXXXXXXXX'
+    secretAccessKey,    // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+    region,             // 'us-east-2'
+    bucket,             // 'bucket-name'
+    prefix,             // 'my-prefix/' (optional)
+    key                 // 'data/stats.csv' (omits prefix, if any)
+});
+
+console.log(resp.headers);
+```
+
 ### Return signed URL without fetching.

 ```js
@ -53,7 +105,7 @@ s3.sign({
    prefix,
    key
 });
-```
+````

 ### A note on S3 terminology

--- a/bin/s3-download.js
+++ b/bin/s3-download.js
@ -23,32 +23,34 @@ if (!key || !filepath) {

 async function run() {
    // GET STREAMED FILE
-    await s3
-        .get({
-            accessKeyId,
-            secretAccessKey,
-            region,
-            bucket,
-            prefix,
-            key
-        })
-        .then(function(resp) {
-            console.log(resp.url);
-            return fs.promises.writeFile(filepath, resp.body);
-        })
-        .catch(function(err) {
-            console.error('Error:');
-            if (err.response) {
-                console.error(err.url);
-                console.error('GET Response:');
-                console.error(err.response.statusCode);
-                console.error(err.response.headers);
-                console.error(err.response.body.toString('utf8'));
-            } else {
-                console.error(err);
-            }
-            process.exit(1);
-        });
+    var resp = await s3.get({
+        accessKeyId,
+        secretAccessKey,
+        region,
+        bucket,
+        prefix,
+        key,
+        stream: filepath
+    });
+
+    console.log('Downloading', resp.url);
+    await resp.stream;
+
+    console.log('');
+    console.log('Saved as', filepath);
+    console.log('');
 }

-run();
+run().catch(function (err) {
+    console.error('Error:');
+    if (err.response) {
+        console.error(err.url);
+        console.error('GET Response:');
+        console.error(err.response.statusCode);
+        console.error(err.response.headers);
+        console.error(err.response.body.toString('utf8'));
+    } else {
+        console.error(err);
+    }
+    process.exit(1);
+});
--- a/index.js
+++ b/index.js
@ -5,10 +5,40 @@ var request = require('@root/request');
 var env = process.env;

 var S3;
+
+function toAwsBucketHost(host, bucket, region) {
+    if (host) {
+        return [host];
+    }
+
+    // Handle simply if it contains only valid subdomain characters
+    // (most notably that it does not have a '.' or '_')
+    if (/^[a-z0-9-]+$/i.test(bucket)) {
+        return ['', bucket + '.s3.amazonaws.com'];
+    }
+
+    // Otherwise use region-specific handling rules
+    // (TODO: handle other regional exceptions)
+    // http://www.wryway.com/blog/aws-s3-url-styles/
+    if (!region || 'us-east-1' === region) {
+        return ['s3.amazonaws.com'];
+    }
+    return ['s3-' + region + '.amazonaws.com'];
+}
+
 module.exports = S3 = {
    // HEAD
    head: function (
-        { host, accessKeyId, secretAccessKey, region, bucket, prefix, key },
+        {
+            host,
+            accessKeyId,
+            secretAccessKey,
+            region,
+            bucket,
+            prefix,
+            key,
+            ...requestOpts
+        },
        _sign
    ) {
        // TODO support minio
@ -39,9 +69,10 @@ module.exports = S3 = {
            // whatever/ => whatever/
            prefix = prefix.replace(/\/?$/, '/');
        }
+        var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
        var signed = aws4.sign(
            {
-                host: host || (bucket + '.s3.amazonaws.com'),
+                host: host || defaultHost,
                service: 's3',
                region: region,
                path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -55,7 +86,9 @@ module.exports = S3 = {
            return url;
        }

-        return request({ method: 'HEAD', url }).then(function (resp) {
+        return request(
+            Object.assign(requestOpts, { method: 'HEAD', url })
+        ).then(function (resp) {
            if (200 === resp.statusCode) {
                resp.url = url;
                return resp;
@ -81,7 +114,8 @@ module.exports = S3 = {
            bucket,
            prefix,
            key,
-            json
+            json,
+            ...requestOpts
        },
        _sign
    ) {
@ -89,9 +123,10 @@ module.exports = S3 = {
        if (prefix) {
            prefix = prefix.replace(/\/?$/, '/');
        }
+        var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
        var signed = aws4.sign(
            {
-                host: host || (bucket + '.s3.amazonaws.com'),
+                host: host || defaultHost,
                service: 's3',
                region: region,
                path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -110,12 +145,14 @@ module.exports = S3 = {
        if (json) {
            encoding = undefined;
        }
-        return request({
-            method: 'GET',
-            url,
-            encoding: encoding,
-            json: json
-        }).then(function (resp) {
+        return request(
+            Object.assign(requestOpts, {
+                method: 'GET',
+                url,
+                encoding: encoding,
+                json: json
+            })
+        ).then(function (resp) {
            if (200 === resp.statusCode) {
                resp.url = url;
                return resp;
@ -142,7 +179,8 @@ module.exports = S3 = {
            prefix,
            key,
            body,
-            size
+            size,
+            ...requestOpts
        },
        _sign
    ) {
@ -150,9 +188,10 @@ module.exports = S3 = {
        if (prefix) {
            prefix = prefix.replace(/\/?$/, '/');
        }
+        var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
        var signed = aws4.sign(
            {
-                host: host || (bucket + '.s3.amazonaws.com'),
+                host: host || defaultHost,
                service: 's3',
                region: region,
                path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -167,9 +206,9 @@ module.exports = S3 = {
            headers['Content-Length'] = size;
        }

-        return request({ method: 'PUT', url, body, headers }).then(function (
-            resp
-        ) {
+        return request(
+            Object.assign(requestOpts, { method: 'PUT', url, body, headers })
+        ).then(function (resp) {
            if (200 === resp.statusCode) {
                resp.url = url;
                return resp;
@ -186,17 +225,27 @@ module.exports = S3 = {
    },

    // DELETE
-    del: function (
-        { host, accessKeyId, secretAccessKey, region, bucket, prefix, key },
+    delete: function (
+        {
+            host,
+            accessKeyId,
+            secretAccessKey,
+            region,
+            bucket,
+            prefix,
+            key,
+            ...requestOpts
+        },
        _sign
    ) {
        prefix = prefix || '';
        if (prefix) {
            prefix = prefix.replace(/\/?$/, '/');
        }
+        var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
        var signed = aws4.sign(
            {
-                host: host || (bucket + '.s3.amazonaws.com'),
+                host: host || defaultHost,
                service: 's3',
                region: region,
                path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -207,7 +256,9 @@ module.exports = S3 = {
        );
        var url = 'https://' + signed.host + signed.path;

-        return request({ method: 'DELETE', url }).then(function (resp) {
+        return request(
+            Object.assign(requestOpts, { method: 'DELETE', url })
+        ).then(function (resp) {
            if (204 === resp.statusCode) {
                resp.url = url;
                return resp;
@ -246,3 +297,4 @@ module.exports = S3 = {
        }
    }
 };
+S3.del = S3.delete;
--- a/package-lock.json
+++ b/package-lock.json
@ -1,13 +1,13 @@
 {
    "name": "@root/s3",
-    "version": "1.1.3",
+    "version": "1.2.0",
    "lockfileVersion": 1,
    "requires": true,
    "dependencies": {
        "@root/request": {
-            "version": "1.5.0",
-            "resolved": "https://registry.npmjs.org/@root/request/-/request-1.5.0.tgz",
-            "integrity": "sha512-J9RUIwVU99/cOVuDVYlNpr4G0A1/3ZxhCXIRiTZzu8RntOnb0lmDBMckhaus5ry9x/dBqJKDplFIgwHbLi6rLA=="
+            "version": "1.7.0",
+            "resolved": "https://registry.npmjs.org/@root/request/-/request-1.7.0.tgz",
+            "integrity": "sha512-lre7XVeEwszgyrayWWb/kRn5fuJfa+n0Nh+rflM9E+EpC28yIYA+FPm/OL1uhzp3TxhQM0HFN4FE2RDIPGlnmg=="
        },
        "aws4": {
            "version": "1.9.1",
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
    "name": "@root/s3",
-    "version": "1.1.3",
+    "version": "1.2.0",
    "description": "A simple, lightweight s3 client with only 2 dependencies",
    "main": "index.js",
    "bin": {
@ -13,6 +13,7 @@
        "example": "examples"
    },
    "scripts": {
+        "prettier": "npx prettier -w '**/*.js'",
        "test": "node test.js"
    },
    "repository": {
@ -27,7 +28,7 @@
    "author": "AJ ONeal <coolaj86@gmail.com> (https://coolaj86.com/)",
    "license": "(MIT OR Apache-2.0)",
    "dependencies": {
-        "@root/request": "^1.5.0",
+        "@root/request": "^1.7.0",
        "aws4": "^1.9.1"
    },
    "devDependencies": {
--- a/test.js
+++ b/test.js
@ -32,11 +32,11 @@ async function run() {
            body: stream,
            size
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: stream uploaded file');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('PUT Response:');
            if (err.response) {
@ -62,11 +62,11 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: streamed file exists');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('HEAD Response:');
            if (err.response) {
                console.error(err.response.statusCode);
@ -87,7 +87,7 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            if (file.toString('binary') === resp.body.toString('binary')) {
                console.info(
                    'PASS: streamed file downloaded with same contents'
@ -96,7 +96,7 @@ async function run() {
            }
            throw new Error("file contents don't match");
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('GET Response:');
            if (err.response) {
@ -118,11 +118,11 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: delete file');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('DELETE Response:');
            if (err.response) {
@ -144,12 +144,12 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            var err = new Error('file should not exist');
            err.response = resp;
            throw err;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            if (err.response && 404 === err.response.statusCode) {
                console.info('PASS: streamed file deleted');
                return null;
@ -176,11 +176,11 @@ async function run() {
            key,
            body: file
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: one-shot upload');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('PUT Response:');
            if (err.response) {
@ -206,11 +206,11 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: one-shot upload exists');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('HEAD Response:');
            if (err.response) {
@ -232,7 +232,7 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            if (file.toString('binary') === resp.body.toString('binary')) {
                console.info(
                    'PASS: one-shot file downloaded with same contents'
@ -241,7 +241,7 @@ async function run() {
            }
            throw new Error("file contents don't match");
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('GET Response:');
            if (err.response) {
@ -263,11 +263,11 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            console.info('PASS: DELETE');
            return null;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            console.error('Error:');
            console.error('DELETE Response:');
            if (err.response) {
@ -289,12 +289,12 @@ async function run() {
            prefix,
            key
        })
-        .then(function(resp) {
+        .then(function (resp) {
            var err = new Error('file should not exist');
            err.response = resp;
            throw err;
        })
-        .catch(function(err) {
+        .catch(function (err) {
            if (err.response && 404 === err.response.statusCode) {
                console.info('PASS: streamed file deleted');
                return null;
Author	SHA1	Message	Date
AJ ONeal	efdfabb9a4	v1.2.0: bugfix for buckets with invalid domain names, and pass request options - Buckets with names like 'example.bucket' and 'example_bucket' are valid as paths, but not as domain names. - allow passthrough support for latest @root/request, which supports pipes and streams	2021-01-26 16:57:20 -07:00
AJ ONeal	ad0fa1f83b	make Prettier	2021-01-26 16:48:15 -07:00