coolaj86
/
desirae.js
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

batch save all posts

This commit is contained in:
AJ ONeal 2015-01-08 22:09:22 +00:00
parent 3ed837c743
commit cb0a2fe189
4 changed files with 238 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

364
deardesi.js
View File

@ -25,75 +25,26 @@
})); }));
} }
function getLayout(desi, themename, layout, arr) { function readFrontmatter(things) {
arr = arr || []; return forEachAsync(things, function (file) {
var parts = frontmatter.parse(file.contents)
var layoutdir = 'layouts'
, themepath
, file
; ;
if (!themename) { if (!file.sha1) {
themename = desi.config.themes.default; // TODO sha1sum
}
if (!layout) {
layout = 'post.html';
} }
file.yml = parts.yml;
file.frontmatter = parts.frontmatter;
file.body = parts.body;
themepath = themename + '/' + layoutdir + '/' + layout; if (!parts.yml) {
console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
desi.content.themes.some(function (theme) {
// TODO what if it isn't html?
if (theme.path === themepath || theme.path.match(themepath + '\\.html')) {
file = theme;
arr.push(theme);
return true;
} }
}); });
if (!file) {
console.error("could not find " + themepath);
return;
} }
// TODO handle possible circular dep condition page -> post -> page function getDirty(cacheByPath, cacheBySha1, thingies, deps) {
console.info(file);
if (file.yml && file.yml.layout) {
return getLayout(desi, themename, file.yml.layout, arr);
} else {
// return the chain page -> posts -> default -> twitter
return arr;
}
}
function runDesi(desi, development) {
var cache = desi.cache
//, config = desi.config
, cacheByPath = {}
, cacheBySha1 = {}
, dfiles
, dthemes
;
desi.urls = desi.config.urls = {};
if (development) {
desi.urls.base_path = desi.config.development.base_path;
desi.urls.url = desi.config.development.url;
desi.urls.development_url = desi.config.development.url;
} else {
desi.config.base_path = desi.urls.base_path = desi.config.production.base_path;
desi.urls.url = desi.config.production.url;
desi.urls.production_url = desi.config.production.url;
}
cache.sources = cache.sources || [];
cache.sources.forEach(function (source) {
cacheByPath[source.path] = source;
cacheBySha1[source.sha1] = source;
});
function getDirty(thingies, deps) {
var byDirty = {} var byDirty = {}
; ;
@ -104,8 +55,6 @@
, fdate , fdate
; ;
console.log('files', key);
console.log(files);
files.forEach(function (file) { files.forEach(function (file) {
var pathname = path.join(file.relativePath + '/' + file.name) var pathname = path.join(file.relativePath + '/' + file.name)
; ;
@ -149,95 +98,197 @@
return byDirty; return byDirty;
} }
dthemes = getDirty(desi.meta.themes); function getLayout(desi, themename, layout, arr) {
console.log('dthemes'); arr = arr || [];
console.log(dthemes);
dfiles = getDirty(desi.meta.collections, dthemes); var layoutdir = 'layouts'
console.log('dfiles'); , themepath
console.log(dfiles); , file
;
if (!themename) {
themename = desi.config.themes.default;
}
if (!layout) {
layout = 'post.html';
}
themepath = themename + '/' + layoutdir + '/' + layout;
desi.content.themes.some(function (theme) {
// TODO what if it isn't html?
if (theme.path === themepath || theme.path.match(themepath + '\\.html')) {
file = theme;
arr.push(theme);
return true;
}
});
if (!file) {
console.error("could not find " + themepath);
return;
}
// TODO handle possible circular dep condition page -> post -> page
if (file.yml && file.yml.layout) {
return getLayout(desi, themename, file.yml.layout, arr);
} else {
// return the chain page -> posts -> default -> twitter
return arr;
}
}
function runDesi(desi, development) {
var cache = desi.cache
//, config = desi.config
, cacheByPath = {}
, cacheBySha1 = {}
, dfiles
, dthemes
, droot
;
desi.urls = desi.config.urls = {};
if (development) {
desi.urls.base_path = desi.config.development.base_path;
desi.urls.url = desi.config.development.url;
desi.urls.development_url = desi.config.development.url;
} else {
desi.config.base_path = desi.urls.base_path = desi.config.production.base_path;
desi.urls.url = desi.config.production.url;
desi.urls.production_url = desi.config.production.url;
}
cache.sources = cache.sources || [];
cache.sources.forEach(function (source) {
cacheByPath[source.path] = source;
cacheBySha1[source.sha1] = source;
});
dthemes = getDirty(cacheByPath, cacheBySha1, desi.meta.themes);
droot = getDirty(cacheByPath, cacheBySha1, [desi.meta.root], dthemes);
dfiles = getDirty(cacheByPath, cacheBySha1, desi.meta.collections, dthemes);
return PromiseA.all([
fsapi.getContents(Object.keys(droot))
, fsapi.getContents(Object.keys(dfiles))
, fsapi.getContents(Object.keys(dthemes))
]).then(function (arr) {
// TODO XXX display errors in html
function noErrors(o) {
if (!o.error) {
return true;
}
console.warn("Couldn't get file contents for " + o.path);
console.warn(o.error);
}
desi.content = {
root: arr[0].filter(noErrors)
, collections: arr[1].filter(noErrors)
, themes: arr[2].filter(noErrors)
};
return fsapi.getContents(Object.keys(dthemes)).then(function (tContent) {
return fsapi.getContents(Object.keys(dfiles)).then(function (cContent) {
desi.content = { collections: cContent, themes: tContent };
return desi; return desi;
}); });
});
} }
console.log(''); console.log('');
console.log(''); console.log('');
console.log('getting config, data, caches...'); console.info('getting config, data, caches...');
return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (things) {
var config = things[0] return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (arr) {
, data = things[1] var config = arr[0]
, cache = things[2] , data = arr[1]
, partials = things[3] , cache = arr[2]
, partials = arr[3]
, collectionnames = Object.keys(config.collections)
, themenames = Object.keys(config.themes)
.filter(function (k) { return 'default' !== k; })
//.map(function (n) { return path.join(n, 'layouts'); })
; ;
console.log('loaded config, data, caches.'); console.info('loaded config, data, caches.');
console.log(things); console.log(arr);
console.log('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never')); console.info('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
var collectionnames = Object.keys(config.collections)
;
return fsapi.getMeta( // TODO make document configurability
collectionnames config.rootdir = config.rootdir || '_root';
, { dotfiles: false return PromiseA.all([
, extensions: ['md', 'markdown', 'htm', 'html', 'jade'] fsapi.getMeta(
}
).then(function (collections) {
var themenames = Object.keys(config.themes).filter(function (k) { return 'default' !== k; })
;
console.log('collections');
console.log(collections);
return fsapi.getMeta(
themenames themenames
, { dotfiles: false , { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml'] , extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml']
} }
).then(function (themes) { )
console.log('themes'); , fsapi.getMeta(
console.log(themes); [config.rootdir]
return { config: config, data: data, cache: cache, meta: { collections: collections, themes: themes }, partials: partials }; , { dotfiles: false
}); , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
}); }
}).then(runDesi).then(function (desi) { )
console.log('desi.content'); , fsapi.getMeta(
console.log(desi.content); collectionnames
, { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade']
}
)
]).then(function (things) {
function noErrors(map) {
Object.keys(map).forEach(function (path) {
map[path] = map[path].filter(function (m) {
if (!m.error && m.size) {
return true;
}
function readMeta(things) { if (!m.size) {
return forEachAsync(things, function (file) { console.warn("Ignoring 0 byte file " + (m.path || m.name));
//console.log('file.contents'); return false;
//console.log(file.contents); }
var parts = frontmatter.parse(file.contents)
console.warn("Couldn't get stats for " + (m.path || m.name));
console.warn(m.error);
});
});
return map;
}
var themes = noErrors(things[0])
, root = noErrors(things[1])[config.rootdir]
, collections = noErrors(things[2])
; ;
if (!file.sha1) { return {
// TODO sha1sum config: config
, data: data
, cache: cache
, meta: {
themes: themes
, collections: collections
, root: root
} }
, partials: partials
file.yml = parts.yml; };
file.frontmatter = parts.frontmatter;
file.body = parts.body;
if (!parts.yml) {
console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
}
return Promise.resolve();
}); });
} }).then(runDesi).then(function (desi) {
return readFrontmatter(desi.content.root.concat(desi.content.themes.concat(desi.content.collections))).then(function () {
return readMeta(desi.content.themes).then(function () {
return readMeta(desi.content.collections).then(function () {
return desi; return desi;
}); });
});
}).then(function (desi) { }).then(function (desi) {
// TODO add missing metadata and resave file // TODO add missing metadata and resave file
desi.content.collections = desi.content.collections.filter(function (article) {
if (!article.yml) {
console.warn("no frontmatter for " + article.name);
console.warn(article.name);
return;
}
return true;
});
desi.content.collections.forEach(function (article) { desi.content.collections.forEach(function (article) {
if (!article.yml.permalink) { if (!article.yml.permalink) {
// TODO read the config for this collection // TODO read the config for this collection
@ -263,12 +314,13 @@
var compiled = [] var compiled = []
; ;
desi.content.collections.forEach(function (article) { desi.content.collections.forEach(function (article, i) {
console.log("compiling " + (i + 1) + "/" + desi.content.collections.length + " " + (article.path || article.name));
// TODO process tags and categories and such // TODO process tags and categories and such
console.log(article.yml.title); //console.log(article.yml.title);
//console.log(article.yml.theme); //console.log(article.yml.theme);
//console.log(article.yml.layout); //console.log(article.yml.layout);
console.log(article.yml.permalink); //console.log(article.yml.permalink);
var child = '' var child = ''
@ -276,13 +328,7 @@
, view , view
; ;
console.log(article.path || (article.relativePath + '/' + article.name));
//console.log(article.frontmatter);
console.log(article.yml);
layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]); layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]);
console.log('LAYERS');
console.log(layers);
view = { view = {
page: article.yml // data for just *this* page page: article.yml // data for just *this* page
@ -316,10 +362,8 @@
parent.path = parent.path || article.relativePath + '/' + article.name; parent.path = parent.path || article.relativePath + '/' + article.name;
if (/\.(html|htm)$/.test(parent.path)) { if (/\.(html|htm)$/.test(parent.path)) {
console.log('thinks its html');
html = body; html = body;
} else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) { } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
console.log('parsing markdown...');
html = marked(body); html = marked(body);
} else { } else {
console.error('unknown parser for ' + (article.path)); console.error('unknown parser for ' + (article.path));
@ -331,10 +375,6 @@
}); });
console.warn('view data.author contains objects?');
console.warn(JSON.stringify(view.data.author, null, ' '));
console.warn(typeof view.data.author.twitter_id);
console.warn(view.data.author);
// TODO add html meta-refresh redirects // TODO add html meta-refresh redirects
compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) }); compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) });
if (Array.isArray(article.yml.redirects)) { if (Array.isArray(article.yml.redirects)) {
@ -360,13 +400,43 @@
return desi; return desi;
}).then(function (desi) { }).then(function (desi) {
var compiled = desi.compiled var compiled = desi.compiled
, batches = []
, now
; ;
console.info('das compiled files'); if (!compiled.length) {
console.info(compiled); console.info("No files were deemed worthy to compile. Done");
return fsapi.putFiles(compiled).then(function (saved) { return;
console.info('files saved'); }
console.info(saved);
// because some servers / proxies are terrible at handling large uploads (>= 100k)
// (vagrant? or express? one of the two is CRAZY slow)
console.info('saving compiled files');
while (compiled.length) {
batches.push(compiled.splice(0, 1));
}
now = Date.now();
console.info('compiled files');
return forEachAsync(batches, function (files) {
return fsapi.putFiles(files).then(function (saved) {
if (saved.error) {
console.error(saved.error);
}
if (!saved.errors || !saved.errors.length) {
return;
}
saved.errors.forEach(function (e) {
console.error(e);
});
//console.info('saved ' + files.length + ' files');
//console.log(saved);
});
}).then(function () {
// TODO update cache
console.info('done', ((Date.now() - now) / 1000).toFixed(3));
}); });
}).catch(function (e) { }).catch(function (e) {
console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..'); console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..');

15
lib/deardesi-browser.js
View File

@ -179,7 +179,10 @@
req.open('POST', url); req.open('POST', url);
req.setRequestHeader("Content-Type", "application/json;charset=UTF-8"); req.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
// Make the request // Make the request
req.send(JSON.stringify(body, null, ' ')); if ('string' !== typeof body) {
body = JSON.stringify(body);
}
req.send(body);
}); });
}; };
@ -239,16 +242,16 @@
var partials = exports.YAML.parse(resp) var partials = exports.YAML.parse(resp)
; ;
console.info('partials');
console.info(partials);
return partials; return partials;
}); });
}; };
fsapi.putFiles = function (files) { fsapi.putFiles = function (files) {
return request.post('/api/fs/files', { var body = { files: files };
files: files body = JSON.stringify(body); // this isn't the slow part
}).then(function (resp) { console.log(files[0]);
console.info('total size:', body.length / (1024 * 1024)); // see filesize
return request.post('/api/fs/files', body).then(function (resp) {
return JSON.parse(resp); return JSON.parse(resp);
}); });
}; };

1
package.json
View File

@ -32,6 +32,7 @@
"bluebird": "^2.5.3", "bluebird": "^2.5.3",
"body-parser": "^1.10.1", "body-parser": "^1.10.1",
"circular-json": "^0.1.6", "circular-json": "^0.1.6",
"compression": "^1.3.0",
"connect": "^3.3.3", "connect": "^3.3.3",
"connect-query": "^0.2.0", "connect-query": "^0.2.0",
"connect-send-json": "^1.0.0", "connect-send-json": "^1.0.0",

11
server.js
View File

@ -23,7 +23,16 @@ var connect = require('connect')
app app
.use(send.json()) .use(send.json())
.use(query()) .use(query())
.use(bodyParser.json()) .use(function (req, res, next) {
console.log('before parse');
next();
})
.use(bodyParser.json({ limit: 10 * 1024 * 1024 })) // 10mb
.use(function (req, res, next) {
console.log('after parse');
next();
})
.use(require('compression')())
.use('/api/fs/walk', function (req, res, next) { .use('/api/fs/walk', function (req, res, next) {
if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) { if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) {
next(); next();