diff --git a/bower.json b/bower.json index 83c745c..69a836a 100644 --- a/bower.json +++ b/bower.json @@ -38,5 +38,8 @@ "forEachAsync": "~5.0.5", "node-uuid": "~1.4.2", "markdown-it": "~3.0.2" + }, + "resolutions": { + "bluebird": "~2.6.2" } } diff --git a/deardesi.js b/deardesi.js index 122d7be..c8031b1 100644 --- a/deardesi.js +++ b/deardesi.js @@ -31,6 +31,10 @@ , 12: 'December' }; + function firstCap(str) { + return str.replace(/^./, function ($1) { return $1.toUpperCase(); }); + } + function pad(str) { str = str.toString(); if (str.length < 2) { @@ -170,7 +174,8 @@ themename = desi.config.themes.default; } if (!layout) { - layout = 'post.html'; + // TODO make configurable + layout = 'posts.html'; } @@ -210,7 +215,7 @@ ; desi.urls = desi.config.urls = {}; - if (development) { + if (true || development) { desi.urls.base_path = desi.config.development.base_path; desi.urls.url = desi.config.development.url; desi.urls.development_url = desi.config.development.url; @@ -229,11 +234,26 @@ dthemes = getDirty(cacheByPath, cacheBySha1, desi.meta.themes); droot = getDirty(cacheByPath, cacheBySha1, [desi.meta.root], dthemes); dfiles = getDirty(cacheByPath, cacheBySha1, desi.meta.collections, dthemes); + + /* + if (!droot.length) { + console.error("no root files to get"); + } + if (!dfiles.length) { + console.error("no content files to get"); + } + if (!dthemes.length) { + console.error("no theme files to get"); + } + if (!droot || !dfiles || !droot) { + throw new Error("didn't read files"); + } + */ return PromiseA.all([ - fsapi.getContents(Object.keys(droot)) - , fsapi.getContents(Object.keys(dfiles)) - , fsapi.getContents(Object.keys(dthemes)) + Object.keys(droot).length ? fsapi.getContents(Object.keys(droot)) : PromiseA.resolve([]) + , Object.keys(dfiles).length ? fsapi.getContents(Object.keys(dfiles)) : PromiseA.resolve([]) + , Object.keys(dthemes).length ? fsapi.getContents(Object.keys(dthemes)) : PromiseA.resolve([]) ]).then(function (arr) { // TODO XXX display errors in html function noErrors(o) { @@ -260,6 +280,8 @@ console.info('getting config, data, caches...'); return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (arr) { + console.info('config'); + console.log(arr[0]); var config = arr[0] , data = arr[1] , cache = arr[2] @@ -268,10 +290,16 @@ , themenames = Object.keys(config.themes) .filter(function (k) { return 'default' !== k; }) //.map(function (n) { return path.join(n, 'layouts'); }) + , assetnames = Object.keys(config.assets) ; - console.info('loaded config, data, caches.'); - console.log(arr); + console.info('loaded config, data, caches, partials'); + console.log({ + config: arr[0] + , data: arr[1] + , cache: arr[2] + , partials: arr[3] + }); console.info('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never')); // TODO make document configurability @@ -295,7 +323,21 @@ , extensions: ['md', 'markdown', 'htm', 'html', 'jade'] } ) + , fsapi.getMeta( + assetnames + , { dotfiles: false + //, extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml'] + } + ) ]).then(function (things) { + console.info('loaded theme meta, root meta, collection meta'); + console.log({ + theme: things[0] + , root: things[1] + , collection: things[2] + , asset: things[3] + }); + function noErrors(map) { Object.keys(map).forEach(function (path) { map[path] = map[path].filter(function (m) { @@ -316,11 +358,25 @@ return map; } - var themes = noErrors(things[0]) - , root = noErrors(things[1])[config.rootdir] + var themes = noErrors(things[0]) + , root = noErrors(things[1])[config.rootdir] , collections = noErrors(things[2]) + , assets = noErrors(things[3]) ; + if (!themes[Object.keys(themes)[0]].length) { + console.error('Missing THEMES!'); + throw new Error('It seems that your themes directory is missing'); + } + + if (!root.length) { + console.error('Missing ROOT!'); + } + + if (!collections[Object.keys(collections)[0]].length) { + console.error('Missing Collections!'); + } + return { config: config , data: data @@ -329,10 +385,43 @@ themes: themes , collections: collections , root: root + , assets: assets } , partials: partials }; }); + }).then(runDesi).then(function (desi) { + var files = {} + ; + + // copy assets -> easy! + // TODO check cache + Object.keys(desi.meta.assets).forEach(function (key) { + var assets = desi.meta.assets[key] + ; + + // TODO fix compiled_path + base_path + assets.forEach(function (asset) { + console.log(asset); + files[path.join(asset.relativePath, asset.name)] = path.join(desi.config.compiled_path, 'assets', asset.relativePath, asset.name); + }); + }); + + return Object.keys(files).length && fsapi.copy(files).then(function (copied) { + if (copied.error) { + console.error(copied.error); + throw new Error(copied.error); + } + + if (copied.errors && copied.errors.length) { + console.error("Errors copying assets..."); + copied.errors.forEach(function (err) { + console.error(err); + }); + } + + return desi; + }) || PromiseA.resolve(desi); }).then(runDesi).then(function (desi) { return readFrontmatter(desi.content.root.concat(desi.content.themes.concat(desi.content.collections))).then(function () { return desi; @@ -343,26 +432,28 @@ desi.content.root.forEach(function (page) { var name = path.basename(page.path, path.extname(page.path)) + , nindex ; //if (-1 === desi.data.navigation.indexOf(name) && 'index' !== name) - if (-1 === desi.data.navigation.indexOf(name)) { + nindex = desi.data.navigation.indexOf(name); + if (-1 === nindex) { return; } - desi.navigation.push({ - title: page.yml && page.yml.title || name.replace(/^./, function ($1) { return $1.toUpperCase(); }) - , href: '/' + name - , path: '/' + name + desi.navigation[nindex] = { + title: page.yml && page.yml.title || firstCap(name) + , href: desi.urls.base_path + '/' + name + , path: desi.urls.base_path + '/' + name , name: name , active: false // placeholder - }); + }; }); - desi.content.root.forEach(function (page) { page.yml = page.yml || {}; - page.yml.layout = page.yml.layout || 'default'; + // TODO make default layout configurable + page.yml.layout = page.yml.layout || '_root'; if (!page.relativePath) { page.relativePath = path.dirname(page.path); @@ -371,6 +462,11 @@ page.relativePath = page.relativePath.replace(desi.config.rootdir, '').replace(/^\//, ''); page.path = path.join(page.relativePath, page.name); + + // TODO make bare root routes configurable + page.yml.permalink = page.yml.permalink || page.path.replace(/\.\w+$/, ''); + + page.yml.title = page.yml.title || firstCap(page.name.replace(/\.\w+$/, '')); }); desi.content.collections = desi.content.collections.filter(function (article) { @@ -395,13 +491,24 @@ // TODO read the config for this collection for how to create premalink if (!yml.permalink) { + if (page.name) { + page.htmlname = page.name.replace(/\.\w+$/, '.html'); + } page.path = page.path || path.join(page.relativePath, page.name); + page.htmlpath = page.path.replace(/\.\w+$/, '.html'); // TODO strip '_root' or whatever // strip .html, .md, .jade, etc - yml.permalink = path.join(desi.urls.base_path, path.basename(page.path, path.extname(page.path))); + yml.permalink = page.htmlpath; + console.info('1', yml.permalink); } + + if (!/\.html?$/.test(yml.permalink)) { + console.info(page.yml.permalink); + yml.permalink = path.join(yml.permalink, 'index.html'); + } + //yml.permalinkBase = path.join(path.dirname(yml.permalink), path.basename(yml.permalink, path.extname(yml.permalink))); - yml.permalink = path.join(path.dirname(yml.permalink), path.basename(yml.permalink, path.extname(yml.permalink))); + //yml.permalink = path.join(path.dirname(yml.permalink), path.basename(yml.permalink, path.extname(yml.permalink))); if (!page.yml.uuid) { // TODO only do this if it's going to be saved @@ -439,7 +546,7 @@ //entity.second = entity.published_at.second; // The root index is the one exception - if (/^\/?index$/.test(entity.yml.permalink)) { + if (/^\/?index(\.html?)?$/.test(entity.yml.permalink)) { entity.yml.permalink = ''; console.info('found index', entity); } @@ -500,7 +607,7 @@ set = yearsArr[yindex]; if (!set.months[mindex]) { - set.months[mindex] = { month: f.month, pages: [] }; + set.months[mindex] = { month: months[parseInt(f.month, 10)], pages: [] }; } set = set.months[mindex]; @@ -554,16 +661,20 @@ // TODO less / sass / etc compiled.push({ contents: entity.body || entity.contents, path: path.join(desi.config.compiled_path, 'themes', entity.path) }); if (/stylesheets.*\.css/.test(entity.path) && (!/google/.test(entity.path) || /obsid/.test(entity.path))) { + // TODO XXX move to a partial desi.assets.push( - '' + '' ); } } + desi.navigation.filter(function (n) { + return n; + }); console.log(desi.navigation); function compileContentEntity(entity, i, arr) { console.log("compiling " + (i + 1) + "/" + arr.length + " " + (entity.path || entity.name)); - var child = '' + var previous = '' , layers , view ; @@ -572,7 +683,6 @@ view = { page: entity.yml // data for just *this* page - , content: child // processed content for just *this* page //, data: desi.data // data.yml // https://github.com/janl/mustache.js/issues/415 , data: num2str(desi.data) @@ -590,6 +700,7 @@ // TODO concat theme, widget, and site assets , assets: desi.assets.join('\n') }; + //console.log('rel:', view.relative_url); view.site.author = desi.data.author; view.site.navigation = JSON.parse(JSON.stringify(desi.navigation)); @@ -601,14 +712,20 @@ }); // backwards compat view.site['navigation?to_pages'] = view.site.navigation; + view.site['navigation?to__root'] = view.site.navigation; + view.data.navigation = view.site.navigation; + view.data['navigation?to_pages'] = view.site.navigation; + view.data['navigation?to__root'] = view.site.navigation; layers.forEach(function (current) { // TODO meta.layout var body = (current.body || current.contents || '').trim() , html + , curview = {} ; - current.path = current.path || entity.relativePath + '/' + entity.name; + // TODO move to normalization + current.path = current.path || (entity.relativePath + '/' + entity.name); if (/\.(html|htm)$/.test(current.path)) { html = body; @@ -622,29 +739,42 @@ console.error('unknown parser for ' + (entity.path)); } - view.content = child; + view.content = previous; + view.page.content = previous; - child = Mustache.render(html, view, desi.partials); + // to prevent perfect object equality (and potential template caching) + Object.keys(view).forEach(function (key) { + curview[key] = view[key]; + }); + previous = Mustache.render(html, curview, desi.partials); }); - // TODO add html meta-refresh redirects - compiled.push({ contents: child, path: path.join(desi.config.compiled_path, entity.yml.permalink, 'index.html') }); + console.log({ contents: previous }); + + // NOTE: by now, all permalinks should be in the format /path/to/page.html or /path/to/page/index.html + compiled.push({ contents: previous, path: path.join(desi.config.compiled_path, entity.yml.permalink/*, 'index.html'*/) }); entity.yml.redirects = entity.yml.redirects || []; - if (entity.yml.permalink) { - entity.yml.redirects.push(entity.yml.permalink + '.html'); + if (/\/index.html$/.test(entity.yml.permalink)) { + entity.yml.redirects.push(entity.yml.permalink.replace(/\/index.html$/, '.html')); + } else { + entity.yml.redirects.push(entity.yml.permalink.replace(/\.html?$/, '/index.html')); } entity.yml.redirects.forEach(function (redirect) { - child = + var content + ; + + // TODO move to partial + content = '' + '
' + 'This page has moved to a ' + entity.yml.title + '.
' @@ -652,7 +782,7 @@ + '' ; - compiled.push({ contents: child, path: path.join(desi.config.compiled_path, redirect) }); + compiled.push({ contents: content, path: path.join(desi.config.compiled_path, redirect) }); }); } @@ -680,9 +810,9 @@ // because some servers / proxies are terrible at handling large uploads (>= 100k) // (vagrant? or express? one of the two is CRAZY slow) - console.info('saving compiled files'); + console.info('saving compiled files', desi.compiled); while (compiled.length) { - batches.push(compiled.splice(0, 1)); + batches.push(compiled.splice(0, 500)); } now = Date.now(); @@ -716,6 +846,7 @@ }).catch(function (e) { console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..'); console.error(e.message); + console.error(e); throw e; }); }('undefined' !== typeof exports && exports || window)); diff --git a/lib/deardesi-browser.js b/lib/deardesi-browser.js index d565b6a..96c411a 100644 --- a/lib/deardesi-browser.js +++ b/lib/deardesi-browser.js @@ -246,6 +246,20 @@ }); }; + fsapi.copy = function (files) { + var body = { files: files }; + body = JSON.stringify(body); // this is more or less instant for a few MiB of posts + return request.post('/api/fs/copy', body).then(function (resp) { + var response = JSON.parse(resp) + ; + + // not accurate for utf8/unicode, but close enough + response.size = body.length; + return response; + }); + + }; + fsapi.putFiles = function (files) { var body = { files: files }; body = JSON.stringify(body); // this is more or less instant for a few MiB of posts diff --git a/lib/fsapi.js b/lib/fsapi.js index 740a01e..6be7d67 100644 --- a/lib/fsapi.js +++ b/lib/fsapi.js @@ -9,6 +9,7 @@ var PromiseA = require('bluebird').Promise , safeResolve = require('./deardesi-utils').safeResolve , sha1sum = function (str) { return require('secret-utils').hashsum('sha1', str); } , mkdirp = PromiseA.promisify(require('mkdirp')) + , fsExtra = PromiseA.promisifyAll(require('fs.extra')) ; function strip(prefix, pathname) { @@ -157,6 +158,92 @@ function getfs(blogdir, filepaths) { }); } +function makeAllDirs(dirpaths) { + var errors = [] + ; + + return forEachAsync(dirpaths, function (pathname) { + return mkdirp(pathname).catch(function (e) { + // TODO exclude attempting to write files to this dir? + errors.push({ + type: 'directory' + + , directory: pathname + + , message: e.message + , code: e.code + , errno: e.errno + , status: e.status + , syscall: e.syscall + }); + + }); + }).then(function () { + return errors; + }); +} + +function copyfs(blogdir, files) { + // TODO switch format to { source: ..., dest: ..., opts: ... } ? + var results = { errors: [] } + , dirpaths = {} + , sources = Object.keys(files) + ; + + return forEachAsync(sources, function (source) { + /* + var nsource = safeResolve(blogdir, source) + ; + */ + + var dest = safeResolve(blogdir, files[source]) + , pathname = path.dirname(dest) + //, filename = path.basename(dest) + ; + + dirpaths[pathname] = true; + + return Promise.resolve(); + }).then(function () { + // TODO is it better to do this lazy-like or as a batch? + // I figure as batch when there may be hundreds of files, + // likely within 2 or 3 directories + return makeAllDirs(Object.keys(dirpaths)).then(function (errors) { + errors.forEach(function (e) { + results.errors.push(e); + }); + }); + }).then(function () { + // TODO allow delete? + return forEachAsync(sources, function (source) { + return fsExtra.copyAsync(safeResolve(blogdir, source), safeResolve(blogdir, files[source])).catch(function (e) { + results.errors.push({ + type: 'file' + + , source: source + , destination: files[source] + + , message: e.message + , code: e.code + , errno: e.errno + , status: e.status + , syscall: e.syscall + }); + }); + }); + }).catch(function (e) { + results.error = { + message: e.message + , code: e.code + , errno: e.errno + , status: e.status + , syscall: e.syscall + }; + }).then(function () { + return results; + }); +} + function putfs(blogdir, files) { var results = { errors: [] } , dirpaths = {} @@ -246,6 +333,7 @@ walkDirs('blog', ['posts'], { contents: false }).then(function (stats) { */ module.exports.walk = { walkDirs: walkDirs, walkDir: walkDir }; +module.exports.copyfs = copyfs; module.exports.getfs = getfs; module.exports.putfs = putfs; module.exports.walkDir = walkDir; diff --git a/package.json b/package.json index cff9f74..3553419 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "connect-send-json": "^1.0.0", "escape-string-regexp": "^1.0.2", "foreachasync": "^5.0.5", + "fs.extra": "^1.2.1", "json2yaml": "^1.0.3", "markdown": "^0.5.0", "markdown-it": "^3.0.2", diff --git a/server.js b/server.js index 115b3ce..da3e311 100644 --- a/server.js +++ b/server.js @@ -10,6 +10,7 @@ var connect = require('connect') , send = require('connect-send-json') , app = connect() + , fsapi = require('./lib/fsapi') , walk = require('./lib/fsapi').walk , getfs = require('./lib/fsapi').getfs , putfs = require('./lib/fsapi').putfs @@ -23,15 +24,7 @@ var connect = require('connect') app .use(send.json()) .use(query()) - .use(function (req, res, next) { - console.log('before parse'); - next(); - }) .use(bodyParser.json({ limit: 10 * 1024 * 1024 })) // 10mb - .use(function (req, res, next) { - console.log('after parse'); - next(); - }) .use(require('compression')()) .use('/api/fs/walk', function (req, res, next) { if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) { @@ -112,6 +105,25 @@ app res.json(results); }); }) + .use('/api/fs/copy', function (req, res, next) { + if (!(/^POST|PUT$/i.test(req.method) || /^POST|PUT$/i.test(req.query._method))) { + next(); + return; + } + + var opts = {} + , files = req.body.files + ; + + if ('object' !== typeof files || !Object.keys(files).length) { + res.json({ error: "please specify POST w/ req.body.files" }); + return; + } + + return fsapi.copyfs(blogdir, files, opts).then(function (results) { + res.json(results); + }); + }) .use('/api/fs', function (req, res, next) { next(); @@ -119,12 +131,12 @@ app }) .use('/api/fs/static', serveStatic('.')) - .use(serveStatic(blogdir)) .use(serveStatic('.')) + .use(serveStatic(blogdir)) ; module.exports = app; -require('http').createServer().on('request', app).listen(80, function () { - console.log('listening 80'); +require('http').createServer().on('request', app).listen(process.argv[2] || 65080, function () { + console.log('listening ' + (process.argv[2] || 65080)); });