From 59c0a5cc8c00623307efaba0ca2f3e6732fb536a Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 3 Oct 2019 01:28:01 -0600 Subject: [PATCH] v3: node, web, and webpack support --- README.md | 151 +++++++++++++++++-------------- base32.js | 215 ++++++++++++++++++++++++++++++++++++++++++++ bin/build.js | 62 +++++++++++++ examples/index.html | 5 +- examples/test.js | 95 -------------------- hex.js | 43 +++++++++ index.js | 104 --------------------- node.js | 19 ---- package-lock.json | 30 +++++++ package.json | 79 ++++++++-------- tests/index.js | 93 +++++++++++++++++++ unibabel.base32.js | 142 ----------------------------- unibabel.hex.js | 46 ---------- unibabel.js | 115 ++++++++++++++++++++++++ unibabel.node.js | 63 +++++++++++++ 15 files changed, 753 insertions(+), 509 deletions(-) create mode 100644 base32.js create mode 100644 bin/build.js delete mode 100644 examples/test.js create mode 100644 hex.js delete mode 100644 index.js delete mode 100644 node.js create mode 100644 package-lock.json create mode 100644 tests/index.js delete mode 100644 unibabel.base32.js delete mode 100644 unibabel.hex.js create mode 100644 unibabel.js create mode 100644 unibabel.node.js diff --git a/README.md b/README.md index df55004..032fa19 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,60 @@ -Unibabel -======== +# Unibabel -Minimalistic **Base64**, **TypedArrays**, and **UTF-8** / **Unicode** conversions in Browser (and Node) JavaScript. Optional add-on support for **hex** and **base32**. +Minimalistic **Base64**, **TypedArrays**, and **UTF-8** / **Unicode** conversions in Browser (and Node) JavaScript. +Optional add-on support for **hex** and **base32**. See See also - * [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encode) / [TextDecoder](https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/decode) - * [DateView](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView) - * [text-encoding](https://github.com/inexorabletash/text-encoding) - * [TextEncoderLite (based on Buffer)](https://github.com/coolaj86/TextEncoderLite/tree/litest) - * [TextEncoderLite (based on text-encoding)](https://github.com/coolaj86/TextEncoderLite/tree/lite) - * [Beatgammit's base64-js](https://github.com/beatgammit/base64-js) +- [TextEncoder](https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encode) / [TextDecoder](https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/decode) +- [DateView](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView) +- [text-encoding](https://github.com/inexorabletash/text-encoding) +- [TextEncoderLite (based on Buffer)](https://github.com/coolaj86/TextEncoderLite/tree/litest) +- [TextEncoderLite (based on text-encoding)](https://github.com/coolaj86/TextEncoderLite/tree/lite) +- [Beatgammit's base64-js](https://github.com/beatgammit/base64-js) -Are you in the right place? ------------------------- +# Example (Unicode to ArrayBuffer): -Dear Node.js Users: +```js +var unicodeString = "I'm a ☢ ☃ that plays 𝄢 guitar and spea̧͈͖ks Ar̽̾̈́͒͑ ̶̧̨̱̹̭̯ͧ̾ͬC̷̙̲̝͖ͭ̏ͥͮ͟Oͮ͏̮̪̝͍M̲̖͊̒ͪͩͬ̚̚͜!"; +var buf = Unibabel.utf8ToBuffer(unicodeString); +``` -You SHOULD NOT use this module. You already have [`Buffer`](https://nodejs.org/api/buffer.html) and [`thirty-two`](https://github.com/chrisumbel/thirty-two): +## Supported Environments + +- VanillaJS +- Node.js +- CommonJS +- WebPack +- ... everything? + +# Browser Usage + +## CDN + +- +- + +## Bower + +```bash +bower install --save unibabel +``` + +```html + +``` + +# Node Usage + +You don't _need_ this module, as you already have +[`Buffer`](https://nodejs.org/api/buffer.html) and [`thirty-two`](https://github.com/chrisumbel/thirty-two): + +For example: ```javascript -var buf = new Buffer('I ½ ♥ 💩', 'utf8'); +var buf = Buffer.from('I ½ ♥ 💩', 'utf8'); buf.toString('hex'); buf.toString('base64'); buf.toString('ascii'); @@ -30,27 +62,9 @@ buf.toString('utf8'); buf.toString('binary'); // deprecated, do not use ``` -Install -------- +However, you _can_ use it (perhaps for isomorphic js?) -You just include the `index.js` in a lovely script tag. - -```bash -bower install --save unibabel -``` - -```html - -``` - -Dear WebPack / Broccoli, Gulp / Grunt Users: - -I don't know how your build systems work these days, -but Unibabel is exported as `Unibabel` -and you can access it as `window.Unibabel`. - -API -=== +# API ```javascript // TypedArray <--> UTF8 @@ -58,50 +72,53 @@ var uint8Array = Unibabel.strToUtf8Arr(str); var str = Unibabel.utf8ArrToStr(uint8Array); // TypedArray <--> Base64 -var base64 = Unibabel.arrToBase64(uint8Array) -var uint8Array = Unibabel.base64ToArr(base64) +var base64 = Unibabel.arrToBase64(uint8Array); +var uint8Array = Unibabel.base64ToArr(base64); ``` **Normal APIs** -`index.js` +`unibabel.js` -* utf8ToBuffer(utf8str) => array -* bufferToUtf8(array) => string -* utf8ToBase64(utf8str) => base64 -* base64ToUtf8(base64) => string -* bufferToBase64(array) => base64 -* base64ToBuffer(base64) => array +- utf8ToBuffer(utf8str) => array +- bufferToUtf8(array) => string +- utf8ToBase64(utf8str) => base64 +- base64ToUtf8(base64) => string +- bufferToBase64(array) => base64 +- base64ToBuffer(base64) => array **Hex APIs** `unibabel.hex.js` -* hexToBuffer(hexstr) => array -* bufferToHex(array) => hexstr +- hexToBuffer(hexstr) => array +- bufferToHex(array) => hexstr **Base32 APIs** `unibabel.base32.js` -* base32ToBuffer(b32str) => array -* bufferToBase32(array) => b32str +- base32ToBuffer(b32str) => array +- bufferToBase32(array) => b32str **Helper APIs** -* utf8ToBinaryString(utf8str) => binstr -* binaryStringToUtf8(binstr) => utf8str -* bufferToBinaryString(buffer) => binstr -* binaryStringToBuffer(binstr) => array +- utf8ToBinaryString(utf8str) => binstr +- binaryStringToUtf8(binstr) => utf8str +- bufferToBinaryString(buffer) => binstr +- binaryStringToBuffer(binstr) => array -Examples -======== +# Examples ```javascript // Base64 -var myArray = Unibabel.base64ToArr("QmFzZSA2NCDigJQgTW96aWxsYSBEZXZlbG9wZXIgTmV0d29yaw=="); // "Base 64 \u2014 Mozilla Developer Network" -var myBuffer = Unibabel.base64ToArr("QmFzZSA2NCDigJQgTW96aWxsYSBEZXZlbG9wZXIgTmV0d29yaw==").buffer; // "Base 64 \u2014 Mozilla Developer Network" +var myArray = Unibabel.base64ToArr( + 'QmFzZSA2NCDigJQgTW96aWxsYSBEZXZlbG9wZXIgTmV0d29yaw==' +); // "Base 64 \u2014 Mozilla Developer Network" +var myBuffer = Unibabel.base64ToArr( + 'QmFzZSA2NCDigJQgTW96aWxsYSBEZXZlbG9wZXIgTmV0d29yaw==' +).buffer; // "Base 64 \u2014 Mozilla Developer Network" console.log(myBuffer.byteLength); @@ -119,27 +136,28 @@ var sMyOutput = Unibabel.utf8ArrToStr(aMyUTF8Output); alert(sMyOutput); ``` -License -======= +# License -* `index.js` and `unibabel.hex.js` are dual-licensed as Apache 2.0 and MIT. -* `unibabel.base32.js` is a modified version of [thirty-two](https://github.com/chrisumbel/thirty-two) and is therefore licensed MIT. +- `index.js` and `unibabel.hex.js` are dual-licensed as Apache 2.0 and MIT. +- `unibabel.base32.js` is a modified version of [thirty-two](https://github.com/chrisumbel/thirty-two) and is therefore licensed MIT. Some parts of the code were taken from MDN, which Mozilla has licensed in the Public Domain, which means that I am at liberty to re-license my copy under the Apache 2 and MIT licenses. See -ChangeLog -==== +# ChangeLog -v2.1.0 ------- +## v3.0.0 + +- Proper node support (`unibabel.node.js`) +- Various minor updates + +## v2.1.0 Added `unibabel.base32.js` -v2.0.0 ------- +## v2.0.0 The new implementation is binary compatible with node.js, TextEncoder, and other more-common UTF-8 encodings. @@ -150,8 +168,7 @@ it happens to work). See -v1.0.0 ------- +## v1.0.0 This version was based on the work by good folks at the MDN, however, the UTF-8 conversion was not byte-compatible with other UTF-8 conversions diff --git a/base32.js b/base32.js new file mode 100644 index 0000000..95b2b96 --- /dev/null +++ b/base32.js @@ -0,0 +1,215 @@ +/* +Copyright (c) 2011, Chris Umbel +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +'use strict'; + +var charTable = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; +var byteTable = [ + 0xff, + 0xff, + 0x1a, + 0x1b, + 0x1c, + 0x1d, + 0x1e, + 0x1f, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff +]; + +function quintetCount(buff) { + var quintets = Math.floor(buff.length / 5); + return buff.length % 5 === 0 ? quintets : quintets + 1; +} + +exports.bufferToBase32 = function(plain) { + // plain MUST come in either as Array or Uint8Array + if ('undefined' !== typeof Uint8Array) { + if (!(plain instanceof Uint8Array)) { + plain = new Uint8Array(plain); + } + } + var i = 0; + var j = 0; + var shiftIndex = 0; + var digit = 0; + var encoded = new Array(quintetCount(plain) * 8); + + /* byte by byte isn't as pretty as quintet by quintet but tests a bit + faster. will have to revisit. */ + while (i < plain.length) { + var current = plain[i]; + + if (shiftIndex > 3) { + digit = current & (0xff >> shiftIndex); + shiftIndex = (shiftIndex + 5) % 8; + digit = + (digit << shiftIndex) | + ((i + 1 < plain.length ? plain[i + 1] : 0) >> (8 - shiftIndex)); + i++; + } else { + digit = (current >> (8 - (shiftIndex + 5))) & 0x1f; + shiftIndex = (shiftIndex + 5) % 8; + if (shiftIndex === 0) { + i++; + } + } + + encoded[j] = charTable[digit]; + j++; + } + + for (i = j; i < encoded.length; i++) { + encoded[i] = '='; + } + + return encoded.join(''); +}; + +exports.base32ToBuffer = function(encoded) { + var shiftIndex = 0; + var plainDigit = 0; + var plainChar; + var plainPos = 0; + var len = Math.ceil((encoded.length * 5) / 8); + var decoded; + encoded = encoded.split('').map(function(ch) { + return ch.charCodeAt(0); + }); + if ('undefined' !== typeof Uint8Array) { + encoded = new Uint8Array(encoded); + decoded = new Uint8Array(len); + } else { + decoded = new Array(len); + } + + /* byte by byte isn't as pretty as octet by octet but tests a bit + faster. will have to revisit. */ + for (var i = 0; i < encoded.length; i++) { + if (encoded[i] === 0x3d) { + //'=' + break; + } + + var encodedByte = encoded[i] - 0x30; + + if (encodedByte < byteTable.length) { + plainDigit = byteTable[encodedByte]; + + if (shiftIndex <= 3) { + shiftIndex = (shiftIndex + 5) % 8; + + if (shiftIndex === 0) { + plainChar |= plainDigit; + decoded[plainPos] = plainChar; + plainPos++; + plainChar = 0; + } else { + plainChar |= 0xff & (plainDigit << (8 - shiftIndex)); + } + } else { + shiftIndex = (shiftIndex + 5) % 8; + plainChar |= 0xff & (plainDigit >>> shiftIndex); + decoded[plainPos] = plainChar; + plainPos++; + + plainChar = 0xff & (plainDigit << (8 - shiftIndex)); + } + } else { + throw new Error('Invalid input - it is not base32 encoded string'); + } + } + + if (decoded.slice) { + // Array or TypedArray + return decoded.slice(0, plainPos); + } else { + // Mobile Safari TypedArray + return new Uint8Array(Array.prototype.slice.call(decoded, 0, plainPos)); + } +}; diff --git a/bin/build.js b/bin/build.js new file mode 100644 index 0000000..01848f9 --- /dev/null +++ b/bin/build.js @@ -0,0 +1,62 @@ +#!/usr/bin/env node +(async function() { + 'use strict'; + + var UglifyJS = require('uglify-js'); + var path = require('path'); + var fs = require('fs'); + var promisify = require('util').promisify; + var readFile = promisify(fs.readFile); + var writeFile = promisify(fs.writeFile); + var gzip = promisify(require('zlib').gzip); + var name = require('../package.json').name; + + // The order is specific, and it matters + var files = await Promise.all( + ['../unibabel.js', '../hex.js', '../base32.js'].map( + async function(file) { + return (await readFile( + path.join(__dirname, file), + 'utf8' + )).trim(); + } + ) + ); + + var license = + [ + '// Unibabel Copyright 2015-2019 AJ ONeal. All rights reserved.', + '/* License at http://mozilla.org/MPL/2.0/ */', + '// Thirty-Two Copyright (c) 2011, Chris Umbel. All rights reserved.', + '/* License at https://github.com/chrisumbel/thirty-two */' + ].join('\n') + '\n'; + var header = ['var Unibabel = {};', ';(function () {'].join('\n') + '\n'; + var footer = ['}());'].join('\n') + '\n'; + + var file = ( + (license + header + files.join('\n') + '\n' + footer).trim() + '\n' + ) + .replace(/\bmodule.exports\b/g, 'Unibabel') + .replace(/\bexports\b/g, 'Unibabel'); + await writeFile(path.join(__dirname, '../dist', name + '.all.js'), file); + await writeFile( + path.join(__dirname, '../dist', name + '.all.js.gz'), + await gzip(file) + ); + + // TODO source maps? + var result = UglifyJS.minify(file, { + compress: true, + // mangling doesn't save significant + mangle: false + }); + if (result.error) { + throw result.error; + } + file = license + result.code; + await writeFile(path.join(__dirname, '../dist', name + '.all.min.js'), file); + await writeFile( + path.join(__dirname, '../dist', name + '.all.min.js.gz'), + await gzip(file) + ); +})(); diff --git a/examples/index.html b/examples/index.html index d56f98a..4013b59 100644 --- a/examples/index.html +++ b/examples/index.html @@ -4,9 +4,12 @@ This is for testing. Look in the console. + + + diff --git a/examples/test.js b/examples/test.js deleted file mode 100644 index 52c1a2f..0000000 --- a/examples/test.js +++ /dev/null @@ -1,95 +0,0 @@ -(function () { -'use strict'; - -var Unibabel = window.Unibabel; - -//UTF-8 -var pass = true; -var references = { - string: "I ½ ♥ 𩶘" -, array: [ 73, 32, 194, 189, 32, 226, 153, 165, 32, 240, 169, 182, 152 ] -, hex: "4920c2bd20e299a520f0a9b698" -, base64: "SSDCvSDimaUg8Km2mA==" -, base32: 'JEQMFPJA4KM2KIHQVG3JQ===' -}; -references.buffer = new Uint8Array(references.array); -var binrefs = { - // Note that the binary string "ÿâó<86>Î<93>k" can't be serialized to text - array: [ 255, 226, 26, 243, 134, 206, 147, 107 ] -, hex: "ffe21af386ce936b" -, base64: "/+Ia84bOk2s=" -}; -binrefs.buffer = new Uint8Array(binrefs.array); - -var str = references.string; -var buf = Unibabel.utf8ToBuffer(references.string); -var base64 = Unibabel.bufferToBase64(references.buffer); -var hex = Unibabel.bufferToHex(references.buffer); -var b32 = Unibabel.bufferToBase32(references.buffer); - -function buffersAreEqual(buf1, buf2) { - if (buf1.length !== buf2.length) { - return false; - } - return Array.prototype.every.call(buf1, function (byte, i) { - if (byte === buf2[i]) { - return true; - } - }); -} - -// TODO compare buffer -if (!buffersAreEqual(buf, references.array)) { - pass = false; - console.warn('[FAIL] utf8 -> buffer', buf); -} -if (base64 !== references.base64) { - pass = false; - console.warn('[FAIL] utf8 -> base64', base64); -} -if (hex !== references.hex) { - pass = false; - console.warn('[FAIL] utf8 -> hex', hex); -} - - -// binary -var bytes = binrefs.array; -buf = new Uint8Array(bytes); -str = Unibabel.bufferToBinaryString(buf); -base64 = Unibabel.bufferToBase64(buf); -hex = Unibabel.bufferToHex(buf); - -// This can't be properly tested because binary strings can't be parsed -// if (str !== "ÿâó†Î“k") { -// pass = false; -// console.log('[FAIL] binary -> str', str); -// } -if (binrefs.base64 !== base64) { - pass = false; - console.warn('[FAIL] binary -> base64', base64); -} -if (binrefs.hex !== hex) { - pass = false; - console.warn('[FAIL] binary -> hex', hex); -} - -// -// Base32 -// -b32 = Unibabel.bufferToBase32(references.buffer); -if (references.base32 !== b32) { - pass = false; - console.warn('[FAIL] binary -> base32', references.base32, '!==', b32); -} -buf = Unibabel.base32ToBuffer(references.base32); -if (!buffersAreEqual(buf, references.buffer)) { - pass = false; - console.warn('[FAIL] base32 -> binary', references.buffer, '!==', buf); -} - -if (pass) { - console.info('[PASS] :-D'); -} - -}()); diff --git a/hex.js b/hex.js new file mode 100644 index 0000000..ef5ea4c --- /dev/null +++ b/hex.js @@ -0,0 +1,43 @@ +'use strict'; + +function bufferToHex(arr) { + var i; + var len; + var hex = ''; + var c; + + for (i = 0, len = arr.length; i < len; i += 1) { + c = arr[i].toString(16); + if (c.length < 2) { + c = '0' + c; + } + hex += c; + } + + return hex; +} + +function hexToBuffer(hex) { + // TODO use Uint8Array or ArrayBuffer or DataView + var i; + var byteLen = hex.length / 2; + var arr; + var j = 0; + + if (byteLen !== parseInt(byteLen, 10)) { + throw new Error("Invalid hex length '" + hex.length + "'"); + } + + arr = new Uint8Array(byteLen); + + for (i = 0; i < byteLen; i += 1) { + arr[i] = parseInt(hex[j] + hex[j + 1], 16); + j += 2; + } + + return arr; +} + +// Hex Convenience Functions +exports.hexToBuffer = hexToBuffer; +exports.bufferToHex = bufferToHex; diff --git a/index.js b/index.js deleted file mode 100644 index a8b3fb5..0000000 --- a/index.js +++ /dev/null @@ -1,104 +0,0 @@ -(function (exports) { -'use strict'; - -function utf8ToBinaryString(str) { - var escstr = encodeURIComponent(str); - // replaces any uri escape sequence, such as %0A, - // with binary escape, such as 0x0A - var binstr = escstr.replace(/%([0-9A-F]{2})/g, function(match, p1) { - return String.fromCharCode(parseInt(p1, 16)); - }); - - return binstr; -} - -function utf8ToBuffer(str) { - var binstr = utf8ToBinaryString(str); - var buf = binaryStringToBuffer(binstr); - return buf; -} - -function utf8ToBase64(str) { - var binstr = utf8ToBinaryString(str); - return btoa(binstr); -} - -function binaryStringToUtf8(binstr) { - var escstr = binstr.replace(/(.)/g, function (m, p) { - var code = p.charCodeAt(0).toString(16).toUpperCase(); - if (code.length < 2) { - code = '0' + code; - } - return '%' + code; - }); - - return decodeURIComponent(escstr); -} - -function bufferToUtf8(buf) { - var binstr = bufferToBinaryString(buf); - - return binaryStringToUtf8(binstr); -} - -function base64ToUtf8(b64) { - var binstr = atob(b64); - - return binaryStringToUtf8(binstr); -} - -function bufferToBinaryString(buf) { - var binstr = Array.prototype.map.call(buf, function (ch) { - return String.fromCharCode(ch); - }).join(''); - - return binstr; -} - -function bufferToBase64(arr) { - var binstr = bufferToBinaryString(arr); - return btoa(binstr); -} - -function binaryStringToBuffer(binstr) { - var buf; - - if ('undefined' !== typeof Uint8Array) { - buf = new Uint8Array(binstr.length); - } else { - buf = []; - } - - Array.prototype.forEach.call(binstr, function (ch, i) { - buf[i] = ch.charCodeAt(0); - }); - - return buf; -} - -function base64ToBuffer(base64) { - var binstr = atob(base64); - var buf = binaryStringToBuffer(binstr); - return buf; -} - -exports.Unibabel = { - utf8ToBinaryString: utf8ToBinaryString -, utf8ToBuffer: utf8ToBuffer -, utf8ToBase64: utf8ToBase64 -, binaryStringToUtf8: binaryStringToUtf8 -, bufferToUtf8: bufferToUtf8 -, base64ToUtf8: base64ToUtf8 -, bufferToBinaryString: bufferToBinaryString -, bufferToBase64: bufferToBase64 -, binaryStringToBuffer: binaryStringToBuffer -, base64ToBuffer: base64ToBuffer - -// compat -, strToUtf8Arr: utf8ToBuffer -, utf8ArrToStr: bufferToUtf8 -, arrToBase64: bufferToBase64 -, base64ToArr: base64ToBuffer -}; - -}('undefined' !== typeof exports && exports || 'undefined' !== typeof window && window || global)); diff --git a/node.js b/node.js deleted file mode 100644 index 5cc1a2e..0000000 --- a/node.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict'; - -console.warn("Please don't use Unibabel in node.js. If you think you really think you have a valid use case please report it at https://git.coolaj86.com/coolaj86/unibabel.js/issues/new"); -throw new Error("[unibabel] you're doing it wrong"); - -/* -var data = 'I ½ ♥ 💩'; -var encoding = 'utf8'; -var buf = new Buffer(data, encoding); -buf.toString('hex'); -buf.toString('base64'); -buf.toString('ascii'); -buf.toString('utf8'); -buf.toString('binary'); // deprecated, do not use - -var Base32 = require('thirty-two'); -var b32 = Base32.encode(buf); -Base32.decode(buf); -*/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..9350f4b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,30 @@ +{ + "name": "unibabel", + "version": "3.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "commander": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.1.tgz", + "integrity": "sha512-cCuLsMhJeWQ/ZpsFTbE765kvVfoeSddc4nU3up4fV+fDBcfUXnbITJ+JzhkdjzOqhURjZgujxaioam4RM9yGUg==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "uglify-js": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", + "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "dev": true, + "requires": { + "commander": "~2.20.0", + "source-map": "~0.6.1" + } + } + } +} diff --git a/package.json b/package.json index f7a8dec..020af03 100644 --- a/package.json +++ b/package.json @@ -1,37 +1,46 @@ { - "name": "unibabel", - "version": "2.1.8", - "description": "Base64, TypedArrays, and UTF-8 / Unicode conversions in Browser (and Node) JavaScript", - "main": "node.js", - "browser": { - "./node.js": "./index.js" - }, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "repository": { - "type": "git", - "url": "https://git.coolaj86.com/coolaj86/unibabel.js.git" - }, - "keywords": [ - "ascii", - "binary", - "utf8", - "utf-8", - "ArrayBuffer", - "TypedArrays", - "TypedArray", - "Uint", - "Uint8Array", - "Base64", - "b64", - "Base32", - "b32" - ], - "author": "AJ ONeal (https://coolaj86.com/)", - "license": "(MIT OR Apache-2.0)", - "bugs": { - "url": "https://git.coolaj86.com/coolaj86/unibabel.js/issues" - }, - "homepage": "https://git.coolaj86.com/coolaj86/unibabel.js#readme" + "name": "unibabel", + "version": "3.0.0", + "description": "Base64, TypedArrays, and UTF-8 / Unicode conversions in Browser (and Node) JavaScript", + "main": "unibabel.node.js", + "browser": { + "./unibabel.node.js": "./unibabel.js" + }, + "files": [ + "*.js", + "dist/*.js" + ], + "scripts": { + "test": "node tests", + "build": "node bin/build.js", + "prepublish": "npm run-script build" + }, + "repository": { + "type": "git", + "url": "https://git.coolaj86.com/coolaj86/unibabel.js.git" + }, + "keywords": [ + "ascii", + "binary", + "utf8", + "utf-8", + "ArrayBuffer", + "TypedArrays", + "TypedArray", + "Uint", + "Uint8Array", + "Base64", + "b64", + "Base32", + "b32" + ], + "author": "AJ ONeal (https://coolaj86.com/)", + "license": "(MIT OR Apache-2.0)", + "bugs": { + "url": "https://git.coolaj86.com/coolaj86/unibabel.js/issues" + }, + "homepage": "https://git.coolaj86.com/coolaj86/unibabel.js#readme", + "devDependencies": { + "uglify-js": "^3.6.0" + } } diff --git a/tests/index.js b/tests/index.js new file mode 100644 index 0000000..564d9d3 --- /dev/null +++ b/tests/index.js @@ -0,0 +1,93 @@ +(function(exports) { + 'use strict'; + + var Unibabel = exports.Unibabel || require('../'); + + //UTF-8 + var pass = true; + var references = { + string: 'I ½ ♥ 𩶘', + array: [73, 32, 194, 189, 32, 226, 153, 165, 32, 240, 169, 182, 152], + hex: '4920c2bd20e299a520f0a9b698', + base64: 'SSDCvSDimaUg8Km2mA==', + base32: 'JEQMFPJA4KM2KIHQVG3JQ===' + }; + references.buffer = Uint8Array.from(references.array); + var binrefs = { + // Note that the binary string "ÿâó<86>Î<93>k" can't be serialized to text + array: [255, 226, 26, 243, 134, 206, 147, 107], + hex: 'ffe21af386ce936b', + base64: '/+Ia84bOk2s=' + }; + binrefs.buffer = new Uint8Array(binrefs.array); + + var str = references.string; + var buf = Unibabel.utf8ToBuffer(references.string); + var base64 = Unibabel.bufferToBase64(references.buffer); + var hex = Unibabel.bufferToHex(references.buffer); + var b32 = Unibabel.bufferToBase32(references.buffer); + + function buffersAreEqual(buf1, buf2) { + if (buf1.length !== buf2.length) { + return false; + } + return Array.prototype.every.call(buf1, function(byte, i) { + if (byte === buf2[i]) { + return true; + } + }); + } + + // TODO compare buffer + if (!buffersAreEqual(buf, references.array)) { + pass = false; + console.warn('[FAIL] utf8 -> buffer', buf); + } + if (base64 !== references.base64) { + pass = false; + console.warn('[FAIL] utf8 -> base64', base64); + } + if (hex !== references.hex) { + pass = false; + console.warn('[FAIL] utf8 -> hex', hex); + } + + // binary + var bytes = binrefs.array; + buf = new Uint8Array(bytes); + str = Unibabel.bufferToBinaryString(buf); + base64 = Unibabel.bufferToBase64(buf); + hex = Unibabel.bufferToHex(buf); + + // This can't be properly tested because binary strings can't be parsed + // if (str !== "ÿâó†Î“k") { + // pass = false; + // console.log('[FAIL] binary -> str', str); + // } + if (binrefs.base64 !== base64) { + pass = false; + console.warn('[FAIL] binary -> base64', base64); + } + if (binrefs.hex !== hex) { + pass = false; + console.warn('[FAIL] binary -> hex', hex); + } + + // + // Base32 + // + b32 = Unibabel.bufferToBase32(references.buffer); + if (references.base32 !== b32) { + pass = false; + console.warn('[FAIL] binary -> base32', references.base32, '!==', b32); + } + buf = Unibabel.base32ToBuffer(references.base32); + if (!buffersAreEqual(buf, references.buffer)) { + pass = false; + console.warn('[FAIL] base32 -> binary', references.buffer, '!==', buf); + } + + if (pass) { + console.info('[PASS] :-D'); + } +})('undefined' === typeof window ? module.exports : window); diff --git a/unibabel.base32.js b/unibabel.base32.js deleted file mode 100644 index 8a0baac..0000000 --- a/unibabel.base32.js +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright (c) 2011, Chris Umbel -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -(function (exports) { -'use strict'; - -var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; -var byteTable = [ - 0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, - 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, - 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff -]; - -function quintetCount(buff) { - var quintets = Math.floor(buff.length / 5); - return buff.length % 5 === 0 ? quintets: quintets + 1; -} - -exports.bufferToBase32 = function(plain) { - // plain MUST come in either as Array or Uint8Array - if('undefined' !== typeof Uint8Array) { - if (!(plain instanceof Uint8Array)){ - plain = new Uint8Array(plain); - } - } - var i = 0; - var j = 0; - var shiftIndex = 0; - var digit = 0; - var encoded = new Array(quintetCount(plain) * 8); - - /* byte by byte isn't as pretty as quintet by quintet but tests a bit - faster. will have to revisit. */ - while(i < plain.length) { - var current = plain[i]; - - if(shiftIndex > 3) { - digit = current & (0xff >> shiftIndex); - shiftIndex = (shiftIndex + 5) % 8; - digit = (digit << shiftIndex) | ((i + 1 < plain.length) ? - plain[i + 1] : 0) >> (8 - shiftIndex); - i++; - } else { - digit = (current >> (8 - (shiftIndex + 5))) & 0x1f; - shiftIndex = (shiftIndex + 5) % 8; - if(shiftIndex === 0) { i++; } - } - - encoded[j] = charTable[digit]; - j++; - } - - for(i = j; i < encoded.length; i++) { - encoded[i] = '='; - } - - return encoded.join(''); -}; - -exports.base32ToBuffer = function(encoded) { - var shiftIndex = 0; - var plainDigit = 0; - var plainChar; - var plainPos = 0; - var len = Math.ceil(encoded.length * 5 / 8); - var decoded; - encoded = encoded.split('').map(function (ch) { - return ch.charCodeAt(0); - }); - if('undefined' !== typeof Uint8Array) { - encoded = new Uint8Array(encoded); - decoded = new Uint8Array(len); - } else { - decoded = new Array(len); - } - - /* byte by byte isn't as pretty as octet by octet but tests a bit - faster. will have to revisit. */ - for(var i = 0; i < encoded.length; i++) { - if(encoded[i] === 0x3d){ //'=' - break; - } - - var encodedByte = encoded[i] - 0x30; - - if(encodedByte < byteTable.length) { - plainDigit = byteTable[encodedByte]; - - if(shiftIndex <= 3) { - shiftIndex = (shiftIndex + 5) % 8; - - if(shiftIndex === 0) { - plainChar |= plainDigit; - decoded[plainPos] = plainChar; - plainPos++; - plainChar = 0; - } else { - plainChar |= 0xff & (plainDigit << (8 - shiftIndex)); - } - } else { - shiftIndex = (shiftIndex + 5) % 8; - plainChar |= 0xff & (plainDigit >>> shiftIndex); - decoded[plainPos] = plainChar; - plainPos++; - - plainChar = 0xff & (plainDigit << (8 - shiftIndex)); - } - } else { - throw new Error('Invalid input - it is not base32 encoded string'); - } - } - - if (decoded.slice) { // Array or TypedArray - return decoded.slice(0, plainPos); - } else { // Mobile Safari TypedArray - return new Uint8Array(Array.prototype.slice.call(decoded, 0, plainPos)); - } -}; - -}(window.Unibabel || window)); diff --git a/unibabel.hex.js b/unibabel.hex.js deleted file mode 100644 index e1bf50d..0000000 --- a/unibabel.hex.js +++ /dev/null @@ -1,46 +0,0 @@ -(function () { -'use strict'; - -function bufferToHex(arr) { - var i; - var len; - var hex = ''; - var c; - - for (i = 0, len = arr.length; i < len; i += 1) { - c = arr[i].toString(16); - if (c.length < 2) { - c = '0' + c; - } - hex += c; - } - - return hex; -} - -function hexToBuffer(hex) { - // TODO use Uint8Array or ArrayBuffer or DataView - var i; - var byteLen = hex.length / 2; - var arr; - var j = 0; - - if (byteLen !== parseInt(byteLen, 10)) { - throw new Error("Invalid hex length '" + hex.length + "'"); - } - - arr = new Uint8Array(byteLen); - - for (i = 0; i < byteLen; i += 1) { - arr[i] = parseInt(hex[j] + hex[j + 1], 16); - j += 2; - } - - return arr; -} - -// Hex Convenience Functions -window.Unibabel.hexToBuffer = hexToBuffer; -window.Unibabel.bufferToHex = bufferToHex; - -}()); diff --git a/unibabel.js b/unibabel.js new file mode 100644 index 0000000..d83ba54 --- /dev/null +++ b/unibabel.js @@ -0,0 +1,115 @@ +'use strict'; + +function utf8ToBinaryString(str) { + var escstr = encodeURIComponent(str); + // replaces any uri escape sequence, such as %0A, + // with binary escape, such as 0x0A + var binstr = escstr.replace(/%([0-9A-F]{2})/g, function(match, p1) { + return String.fromCharCode(parseInt(p1, 16)); + }); + + return binstr; +} + +function utf8ToBuffer(str) { + var binstr = utf8ToBinaryString(str); + var buf = binaryStringToBuffer(binstr); + return buf; +} + +function utf8ToBase64(str) { + var binstr = utf8ToBinaryString(str); + return btoa(binstr); +} + +function binaryStringToUtf8(binstr) { + var escstr = binstr.replace(/(.)/g, function(m, p) { + var code = p + .charCodeAt(0) + .toString(16) + .toUpperCase(); + if (code.length < 2) { + code = '0' + code; + } + return '%' + code; + }); + + return decodeURIComponent(escstr); +} + +function bufferToUtf8(buf) { + var binstr = bufferToBinaryString(buf); + + return binaryStringToUtf8(binstr); +} + +function base64ToUtf8(b64) { + var binstr = atob(b64); + + return binaryStringToUtf8(binstr); +} + +function bufferToBinaryString(buf) { + var binstr = Array.prototype.map + .call(buf, function(ch) { + return String.fromCharCode(ch); + }) + .join(''); + + return binstr; +} + +function bufferToBase64(arr) { + var binstr = bufferToBinaryString(arr); + return btoa(binstr); +} + +function binaryStringToBuffer(binstr) { + var buf; + + if ('undefined' !== typeof Uint8Array) { + buf = new Uint8Array(binstr.length); + } else { + buf = []; + } + + Array.prototype.forEach.call(binstr, function(ch, i) { + buf[i] = ch.charCodeAt(0); + }); + + return buf; +} + +function base64ToBuffer(base64) { + var binstr = atob(base64); + var buf = binaryStringToBuffer(binstr); + return buf; +} + +module.exports = { + utf8ToBinaryString: utf8ToBinaryString, + utf8ToBuffer: utf8ToBuffer, + utf8ToBase64: utf8ToBase64, + binaryStringToUtf8: binaryStringToUtf8, + bufferToUtf8: bufferToUtf8, + base64ToUtf8: base64ToUtf8, + bufferToBinaryString: bufferToBinaryString, + bufferToBase64: bufferToBase64, + binaryStringToBuffer: binaryStringToBuffer, + base64ToBuffer: base64ToBuffer +}; + +/** WEBPACK **/ +module.exports.bufferToHex = function(buf) { + return require('./hex.js').bufferToHex(buf); +}; +module.exports.hexToBuffer = function(hex) { + return require('./hex.js').hexToBuffer(hex); +}; +module.exports.bufferToBase32 = function(buf) { + return require('./base32.js').bufferToBase32(buf); +}; +module.exports.base32ToBuffer = function(b32) { + return require('./base32.js').base32ToBuffer(b32); +}; +/** END WEBPACK **/ diff --git a/unibabel.node.js b/unibabel.node.js new file mode 100644 index 0000000..7dbe1b8 --- /dev/null +++ b/unibabel.node.js @@ -0,0 +1,63 @@ +'use strict'; + +var Unibabel = (module.exports = { + utf8ToBinaryString: function(utf8) { + return Buffer.from(utf8, 'utf8').toString('binary'); + }, + utf8ToBuffer: function(utf8) { + return Buffer.from(utf8, 'utf8'); + }, + utf8ToBase64: function(utf8) { + return Buffer.from(utf8, 'utf8').toString('base64'); + }, + binaryStringToUtf8: function(bin) { + return Buffer.from(bin, 'binary').toString('utf8'); + }, + bufferToUtf8: function(buf) { + return Buffer.from(buf).toString('utf8'); + }, + base64ToUtf8: function(b64) { + return Buffer.from(b64, 'base64').toString('utf8'); + }, + bufferToBinaryString: function(buf) { + return Buffer.from(buf).toString('binary'); + }, + bufferToBase64: function(buf) { + return Buffer.from(buf).toString('base64'); + }, + binaryStringToBuffer: function(bin) { + return Buffer.from(bin, 'binary'); + }, + base64ToBuffer: function(b64) { + return Buffer.from(b64, 'base64'); + }, + hexToBuffer: function(hex) { + return Buffer.from(hex, 'hex'); + }, + bufferToHex: function(buf) { + return Buffer.from(buf).toString('hex'); + }, + bufferToBase32: function(buf) { + Unibabel.bufferToBase32 = require('./base32.js').bufferToBase32; + return Unibabel.bufferToBase32(buf); + }, + base32ToBuffer: function(b32) { + Unibabel.base32ToBuffer = require('./base32.js').base32ToBuffer; + return Unibabel.base32ToBuffer(b32); + } +}); + +/* +var data = 'I ½ ♥ 💩'; +var encoding = 'utf8'; +var buf = new Buffer(data, encoding); +buf.toString('hex'); +buf.toString('base64'); +buf.toString('ascii'); +buf.toString('utf8'); +buf.toString('binary'); // deprecated, do not use + +var Base32 = require('thirty-two'); +var b32 = Base32.encode(buf); +Base32.decode(buf); +*/