From f31f6ea1a48dac11062f3e484951404a161e3480 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 5 Jan 2015 19:23:26 +0000 Subject: [PATCH] initial commit --- .gitignore | 3 + bower.json | 40 +++++++ deardesi.js | 216 ++++++++++++++++++++++++++++++++++++ index.html | 19 ++++ lib/convert-ruhoh-config.js | 7 ++ lib/deardesi-browser.js | 128 +++++++++++++++++++++ lib/deardesi-node.js | 9 ++ lib/deardesi-utils.js | 23 ++++ lib/frontmatter.js | 103 +++++++++++++++++ lib/verify-config.js | 56 ++++++++++ lib/walk.js | 28 +++++ package.json | 47 ++++++++ server.js | 41 +++++++ 13 files changed, 720 insertions(+) create mode 100644 bower.json create mode 100644 deardesi.js create mode 100644 index.html create mode 100644 lib/convert-ruhoh-config.js create mode 100644 lib/deardesi-browser.js create mode 100644 lib/deardesi-node.js create mode 100644 lib/deardesi-utils.js create mode 100644 lib/frontmatter.js create mode 100644 lib/verify-config.js create mode 100644 lib/walk.js create mode 100644 package.json create mode 100644 server.js diff --git a/.gitignore b/.gitignore index 59d842b..0e80248 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +node_modules +bower_components + # Logs logs *.log diff --git a/bower.json b/bower.json new file mode 100644 index 0000000..b7bc489 --- /dev/null +++ b/bower.json @@ -0,0 +1,40 @@ +{ + "name": "deardesi", + "version": "1.0.0", + "authors": [ + "AJ ONeal " + ], + "description": "A blogging platform in the browser. Wow!", + "main": "deardesi.js", + "moduleType": [ + "globals", + "node" + ], + "keywords": [ + "dear", + "desi", + "deardesi", + "blog", + "blogging", + "platform", + "browser" + ], + "license": "Apache2", + "homepage": "http://github.com/coolaj86/deardesi", + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ], + "dependencies": { + "mustache": "~0.8.2", + "bluebird": "~2.5.2", + "rsvp": "~3.0.16", + "escape-string-regexp": "~1.0.2", + "marked": "~0.3.2", + "js-yaml": "~3.2.5", + "path": "~3.46.1" + } +} diff --git a/deardesi.js b/deardesi.js new file mode 100644 index 0000000..6801b0f --- /dev/null +++ b/deardesi.js @@ -0,0 +1,216 @@ +;(function (exports) { + 'use strict'; + + //require('require-yaml'); + + var PromiseA = exports.Promise || require('bluebird').Promise + , path = exports.path || require('path') + , Mustache = exports.Mustache || require('mustache') + , marked = exports.marked || require('marked') + , forEachAsync = exports.forEachAsync || require('foreachasync').forEachAsync + , sha1sum = exports.sha1sum || require('./lib/deardesi-node').sha1sum + , frontmatter = exports.frontmatter || require('./lib/frontmatter').Frontmatter + , safeResolve = exports.safeResolve || require('./lib/deardesi-utils').safeResolve + , getStats = exports.getStats || require('./lib/deardesi-node').getStats + , getContents = exports.getContents || require('./lib/deardesi-node').getContents + ; + + function getCollections(blogbase, ignorable, collectionnames) { + var collectiondir + , collectiondirs = [] + , lost = [] + , found = [] + , errors = [] + ; + + + collectionnames.forEach(function (collectionname) { + collectiondir = safeResolve(_dirname, collectionname); + + if (!collectiondir) { + return PromiseA.reject(new Error("Please update your config.yml: " + collectionname + " is outside of '" + _dirname + "'")); + } + + collectiondirs.push({ name: collectionname, path: collectiondir }); + }); + + + return getFolders(collectiondirs, { recursive: true, limit: 5, stats: true }).then(function (stats) { + collectiondirs.forEach(function (collection) { + if (!stats[collection.path]) { + errors.push({ + collection: collection + , message: "server did not return success or error for " + collection.path + ':\n' + JSON.stringify(stats) + }); + } + else if (!stats[collection.path].type) { + lost.push(collection); + } + else if ('directory' !== stats[collection.path].type) { + errors.push({ + collection: collection + , message: collection.path + " is not a directory (might be a symbolic link)" + }); + } else { + found.push(collection); + } + }); + + return { + lost: lost + , found: found + , errors: errors + }; + }); + } + + function showCollectionNotes(notes) { + if (notes.lost.length) { + console.warn("WARNING: these collections you specified couldn't be found"); + notes.lost.forEach(function (node) { + console.warn('? ' + node.name); + }); + console.log(''); + } + + if (notes.found.length) { + console.log("Will compile these collections"); + notes.found.forEach(function (node) { + console.log('+ ' + node.name); + }); + console.log(''); + } + } + + function getLayouts() { + // TODO + } + + function updatePage(pagedir, node, lstat, data) { + var parts = frontmatter.parse(data) + , meta + , html + , view + ; + + if (!parts.yml) { + console.error("Could not parse frontmatter for " + node); + console.error(parts.frontmatter); + return; + } + + if (/\.(html|htm)$/.test(node)) { + html = parts.body.trim(); + } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(node)) { + console.log('parsing markdown...'); + html = marked(parts.body.trim()); + } else { + console.error('unknown parser for ' + node); + } + + meta = { + mtime: lstat.mtime + , ymlsum: sha1sum(parts.frontmatter.trim()) + , textsum: sha1sum(parts.body.trim()) + , htmlsum: sha1sum(html) + , filesum: sha1sum(data) + , filename: node + , filepath: pagedir + }; + + /* + // TODO + db.getCached(meta).error(function () { + // TODO rebuild and save + }); + */ + + // TODO meta.layout + view = { + page: parts.yml + , content: html + }; + + console.log(node); + console.log(parts.frontmatter); + console.log(parts.yml); //Mustache.render(pagetpl, view)); + //console.log(meta.mtime.valueOf(), meta.ymlsum, meta.textsum, node); + + return meta; + } + + function templatePosts() { + var pagetpl + , defaulttpl + ; + + // TODO declare path to theme + pagetpl = frontmatter.parse(fs.readFileSync(path.join(config.theme, 'layouts', 'page.html'), 'utf8')); + defaulttpl = frontmatter.parse(fs.readFileSync(path.join(config.theme, 'layouts', 'default.html'), 'utf8')); + + + } + + function getCollection() { + } + + console.log(''); + console.log(''); + console.log('loading caches...'); + + getMetaCache().then(function (db) { + console.log('last update: ' + (db.lastUpdate && new Date(db.lastUpdate) || 'never')); + + console.log('checking for local updates...'); + + + // TODO get layouts here + return getCollections('.', Object.keys(config.collections)).then(function (notes) { + showCollectionNotes(notes); + + return notes.found; + }).then(function (found) { + var metas = [] + ; + + return forEachAsync(found, function (collection) { + begintime = Date.now(); + console.log('begin', ((begintime - starttime) / 1000).toFixed(4)); + + return fs.readdirAsync(collection.path).then(function (nodes) { + + // TODO look for companion yml file aside html|md|jade + nodes = nodes.filter(function (node) { + // TODO have handlers accept or reject extensions in the order they are registered + if (!/\.(htm|html|md|markdown|mdown|mkdn|mkd|jade)$/.test(node)) { + console.warn("ignoring " + collection.name + '/' + node + " (unknown filetype processor)"); + return false; + } + + return true; + }); + + return forEachAsync(nodes, function (pagename) { + var pagepath = path.join(collection.path, pagename) + ; + + // TODO: support walking deep + // TODO: test .html, .md, etc + return fs.lstatAsync(pagepath).then(function (lstat) { + // no funny business allowed + if (!lstat.isFile()) { + return; + } + + return fs.readFileAsync(nodepath, 'utf8').then(function (data) { + updatePage(pagedir, node, lstat, data); + }); + }); + }); + }).then(function () { + console.log('doneish', ((Date.now() - begintime) / 1000).toFixed(4)); + }); + }); + }); + }); +}('undefined' !== typeof exports && exports || window)); diff --git a/index.html b/index.html new file mode 100644 index 0000000..245654c --- /dev/null +++ b/index.html @@ -0,0 +1,19 @@ + + + + Dear Desi + + + + + + + + + + + + +

Open up the console, fool!

+ + diff --git a/lib/convert-ruhoh-config.js b/lib/convert-ruhoh-config.js new file mode 100644 index 0000000..acc1df2 --- /dev/null +++ b/lib/convert-ruhoh-config.js @@ -0,0 +1,7 @@ +'use strict'; + +module.exports.convert = function () { + console.error("I haven't implemented a ruhoh -> nuhoh converter yet, but it's not very hard to do."); + console.error("see https://github.com/coolaj86/nuhoh/tree/master/MIGRATE.md"); + throw new Error('Not Implemented.'); +}; diff --git a/lib/deardesi-browser.js b/lib/deardesi-browser.js new file mode 100644 index 0000000..511bf36 --- /dev/null +++ b/lib/deardesi-browser.js @@ -0,0 +1,128 @@ +/*jshint -W054 */ +var tmpglobal + ; + +try { + tmpglobal = new Function('return this')(); +} catch(e) { + tmpglobal = window; +} + +;(function (exports) { + 'use strict'; + + // Chrome, Firefox, and even MSIE11+ all support crypto + var crypto = window.crypto || window.msCrypto + , algos + ; + + // convenience mappings for common digest algorithms + algos = { + 'sha1': 'SHA-1' + , 'sha256': 'SHA-256' + , 'sha512': 'SHA-512' + }; + + // The function to generate a sha1sum is the same as generating any digest + // but here's a shortcut function anyway + function sha1sum(str) { + return hashsum('sha1', str); + } + + // a more general convenience function + function hashsum(hash, str) { + // you have to convert from string to array buffer + var ab + // you have to represent the algorithm as an object + , algo = { name: algos[hash] } + ; + + if ('string' === typeof str) { + ab = str2ab(str); + } else { + ab = str; + } + + // All crypto digest methods return a promise + return crypto.subtle.digest(algo, ab).then(function (digest) { + // you have to convert the ArrayBuffer to a DataView and then to a hex String + return ab2hex(digest); + }).catch(function (e) { + // if you specify an unsupported digest algorithm or non-ArrayBuffer, you'll get an error + console.error('sha1sum ERROR'); + console.error(e); + throw e; + }); + } + + // convert from arraybuffer to hex + function ab2hex(ab) { + var dv = new DataView(ab) + , i + , len + , hex = '' + , c + ; + + for (i = 0, len = dv.byteLength; i < len; i += 1) { + c = dv.getUint8(i).toString(16); + + if (c.length < 2) { + c = '0' + c; + } + + hex += c; + } + + return hex; + } + + // convert from string to arraybuffer + function str2ab(stringToEncode, insertBom) { + stringToEncode = stringToEncode.replace(/\r\n/g,"\n"); + + var utftext = [] + , n + , c + ; + + if (true === insertBom) { + utftext[0] = 0xef; + utftext[1] = 0xbb; + utftext[2] = 0xbf; + } + + for (n = 0; n < stringToEncode.length; n += 1) { + + c = stringToEncode.charCodeAt(n); + + if (c < 128) { + utftext[utftext.length]= c; + } + else if((c > 127) && (c < 2048)) { + utftext[utftext.length] = (c >> 6) | 192; + utftext[utftext.length] = (c & 63) | 128; + } + else { + utftext[utftext.length] = (c >> 12) | 224; + utftext[utftext.length] = ((c >> 6) & 63) | 128; + utftext[utftext.length] = (c & 63) | 128; + } + + } + return new Uint8Array(utftext).buffer; + } + + exports.hashsum = hashsum; + exports.sha1sum = sha1sum; +}('undefined' !== typeof exports && exports || tmpglobal)); + +;(function () { + 'use strict'; + + exports.getStats + exports.getContents + exports.getMetaCache + exports.getContentCache + //require('./db').create(path.join(_dirname, 'db.json')) +}()); diff --git a/lib/deardesi-node.js b/lib/deardesi-node.js new file mode 100644 index 0000000..ff11556 --- /dev/null +++ b/lib/deardesi-node.js @@ -0,0 +1,9 @@ +'use strict'; + +var PromiseA = require('bluebird').Promise + , secretutils = require('secret-utils') + ; + +module.exports.sha1sum = function (str) { + return PromiseA.resolve( secretutils.hashsum('sha1', str) ); +}; diff --git a/lib/deardesi-utils.js b/lib/deardesi-utils.js new file mode 100644 index 0000000..fa8c392 --- /dev/null +++ b/lib/deardesi-utils.js @@ -0,0 +1,23 @@ +;(function (exports) { + 'use strict'; + + var path = exports.path || require('path') + ; + + function escapeRegExp(str) { + return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); + } + + function safeResolve(basename, target) { + basename = path.resolve(basename); + + var targetname = path.resolve(basename, target) + , re = new RegExp('^' + escapeRegExp(basename) + '(/|$)') + ; + + return re.test(targetname) && targetname; + } + + exports.safeResolve = safeResolve; + exports.escapeRegExp = escapeRegExp; +}('undefined' !== typeof exports && exports || window)); diff --git a/lib/frontmatter.js b/lib/frontmatter.js new file mode 100644 index 0000000..17be626 --- /dev/null +++ b/lib/frontmatter.js @@ -0,0 +1,103 @@ +/*jshint -W054 */ +;(function (exports) { + 'use strict'; + + var YAML = {} + ; + + YAML.parse = exports.jsyaml.load || require('jsyaml').load; + //YAML.parse = require('yaml').eval; + //YAML.parse2 = require('yamljs').parse; + + function readFrontMatter(text) { + var lines + , line + , padIndent = '' + , ymllines = [] + ; + + lines = text.split(/\n/); + line = lines.shift(); + + if (!line.match(/^---\s*$/)) { + return; + } + + // our yaml parser can't handle objects + // that start without indentation, so + // we can add it if this is the case + if (lines[0] && lines[0].match(/^\S/)) { + padIndent = ''; + } + + while (true) { + line = lines.shift(); + + // premature end-of-file (unsupported yaml) + if (!line && '' !== line) { + ymllines = []; + break; + } + + // end-of-yaml front-matter + if (line.match(/^---\s*$/)) { + break; + } + + if (line) { + // supported yaml + ymllines.push(padIndent + line); + } + } + + + // XXX can't be sorted because arrays get messed up + //ymllines.sort(); + if (ymllines) { + return '---\n' + ymllines.join('\n'); + } + + return; + } + + function separateText(text, fm) { + var len + , yml + ; + + yml = readFrontMatter(fm); + // strip frontmatter from text, if any + // including trailing '---' (which is accounted for by the added '\n') + if (yml) { + len = fm.split(/\n/).length; + } else { + len = 0; + } + + return text.split(/\n/).slice(len).join('\n'); + } + + function parseText(text) { + var fm = readFrontMatter(text) + , body = separateText(text, fm) + , yml + ; + + try { + yml = YAML.parse(fm); + } catch(e) { + // + } + + return { + yml: yml + , frontmatter: fm + , body: body + }; + } + + exports.Frontmatter.Frontmatter = exports.Frontmatter = {}; + exports.Frontmatter.readText = readFrontMatter; + exports.Frontmatter.separateText = separateText; + exports.Frontmatter.parse = parseText; +}('undefined' !== typeof exports && exports || window)); diff --git a/lib/verify-config.js b/lib/verify-config.js new file mode 100644 index 0000000..01f1314 --- /dev/null +++ b/lib/verify-config.js @@ -0,0 +1,56 @@ +'use strict'; + +module.export.verify = function (conf) { + if (!conf.NuhohSpec) { + throw new Error("missing key NuhohSpec"); + } + + if (!conf.production) { + throw new Error("missing key production"); + } + + if (!conf.production.canonical_url) { + throw new Error("missing key production.canonical_url"); + } + + if (!conf.production.base_path) { + throw new Error("missing key production.base_path"); + } + + if (!conf.development) { + throw new Error("missing key development"); + } + + if (!conf.development.compiled_path) { + throw new Error("missing key development.compiled_path"); + } + + if (!Array.isArray(conf.collections)) { + if (conf.posts) { + console.error("Please indent and nest 'posts' under the key 'collection' to continue"); + } + throw new Error("missing key 'collections'."); + } + + if (!conf.themes) { + if (conf.twitter) { + console.error("Please indent and nest 'twitter' under the key 'themes' to continue"); + } + throw new Error("missing key 'themes'"); + } + + if (!conf.themes.default) { + if (conf.twitter) { + console.error("Please set themes.default to 'twitter'"); + } + throw new Error("missing key 'themes.default'"); + } + + if (!conf.root) { + throw new Error("missing key root"); + } + + if (!conf.widgets) { + throw new Error("missing key root"); + } +}; diff --git a/lib/walk.js b/lib/walk.js new file mode 100644 index 0000000..be6943b --- /dev/null +++ b/lib/walk.js @@ -0,0 +1,28 @@ +'use strict'; + +var PromiseA = require('bluebird').Promise + , path = require('path') + , walk = require('walk') + , walker + ; + +function getFs(parent, sub) { + // TODO safe + var trueRoot = path.resolve(parent, sub) + ; + + return new PromiseA(function (resolve) { + walker = walk.walk('posts'); + walker.on('directories', function (root, stat, next) { + console.log(root, stat); + next(); + }); + walker.on('files', function (root, stat, next) { + //console.log(root, stat); + next(); + }); + walker.on('end', function () { + console.log('done'); + }); + }); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..783acb9 --- /dev/null +++ b/package.json @@ -0,0 +1,47 @@ +{ + "name": "deardesi", + "version": "0.1.0", + "description": "An in-browser knockoff of the Ruhoh static blog generator. (similar to Jekyll, Octopress, Nanoc, etc)", + "main": "server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git@github.com:coolaj86/deardesi.git" + }, + "keywords": [ + "dear", + "deardesi", + "desi", + "ruhoh", + "ruhoh", + "blog", + "static", + "jekyll", + "octopress", + "nanoc" + ], + "author": "AJ ONeal", + "license": "Apache2", + "bugs": { + "url": "https://github.com/coolaj86/deardesi/issues" + }, + "homepage": "https://github.com/coolaj86/deardesi", + "dependencies": { + "bluebird": "^2.5.3", + "circular-json": "^0.1.6", + "escape-string-regexp": "^1.0.2", + "foreachasync": "^5.0.2", + "json2yaml": "^1.0.3", + "markdown": "^0.5.0", + "marked": "^0.3.2", + "mustache": "^1.0.0", + "require-yaml": "0.0.1", + "require-yamljs": "^1.0.1", + "secret-utils": "^1.0.2", + "walk": "^2.3.5", + "yaml": "^0.2.3", + "yamljs": "^0.2.1" + } +} diff --git a/server.js b/server.js new file mode 100644 index 0000000..e33f3b8 --- /dev/null +++ b/server.js @@ -0,0 +1,41 @@ +'use strict'; + +var connect = require('connect') + , query = require('connect-query') + , bodyParser = require('body-parser') + , serveStatic = require('serve-static') + , app = connect() + ; + +app + .use('/api/fs', query()) + .use('/api/fs', bodyParser.json()) + .use('/api/fs', function (req, res, next) { + if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) { + next(); + return; + } + + /* + return forEachAsync(collectiondirs, function (collection) { + return fs.lstatAsync(collection.path).then(function (stat) { + if (!stat.isDirectory()) { + //return; + } + + }).error(function () { + }); + }).then(function () { + }); + */ + + res.end('not implemented'); + }) + .use('/api/fs', function (req, res, next) { + next(); + return; + }) + .use(serveStatic('.')) + ; + +module.exports = app;