coolaj86
/
desirae.js
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

batch save all posts

This commit is contained in:
AJ ONeal 2015-01-08 22:09:22 +00:00
parent 3ed837c743
commit cb0a2fe189
4 changed files with 238 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

360
deardesi.js
View File

@ -25,6 +25,79 @@
}));
}
function readFrontmatter(things) {
return forEachAsync(things, function (file) {
var parts = frontmatter.parse(file.contents)
;
if (!file.sha1) {
// TODO sha1sum
}
file.yml = parts.yml;
file.frontmatter = parts.frontmatter;
file.body = parts.body;
if (!parts.yml) {
console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
}
});
}
function getDirty(cacheByPath, cacheBySha1, thingies, deps) {
var byDirty = {}
;
Object.keys(thingies).forEach(function (key) {
var files = thingies[key]
, cached
, cdate
, fdate
;
files.forEach(function (file) {
var pathname = path.join(file.relativePath + '/' + file.name)
;
// TODO legitimately checkout layout dependencies
if (deps && Object.keys(deps).length) {
byDirty[pathname] = file;
return;
}
if (!cacheByPath[pathname]) {
if (cacheBySha1[file.sha1]) {
// TODO rename
}
byDirty[pathname] = file;
return;
}
cached = cacheByPath[pathname];
cached.visited = true;
if (cached.sha1 && file.sha1) {
if (file.sha1 && cached.sha1 !== file.sha1) {
byDirty[pathname] = file;
return;
}
cdate = cached.lastModifiedDate && new Date(cached.lastModifiedDate);
fdate = file.lastModifiedDate && new Date(file.lastModifiedDate);
if (!cdate || !fdate || cdate !== fdate) {
byDirty[pathname] = file;
}
}
});
});
return byDirty;
}
function getLayout(desi, themename, layout, arr) {
arr = arr || [];
@ -58,7 +131,6 @@
}
// TODO handle possible circular dep condition page -> post -> page
console.info(file);
if (file.yml && file.yml.layout) {
return getLayout(desi, themename, file.yml.layout, arr);
} else {
@ -74,6 +146,7 @@
, cacheBySha1 = {}
, dfiles
, dthemes
, droot
;
desi.urls = desi.config.urls = {};
@ -93,151 +166,129 @@
cacheBySha1[source.sha1] = source;
});
function getDirty(thingies, deps) {
var byDirty = {}
;
dthemes = getDirty(cacheByPath, cacheBySha1, desi.meta.themes);
droot = getDirty(cacheByPath, cacheBySha1, [desi.meta.root], dthemes);
dfiles = getDirty(cacheByPath, cacheBySha1, desi.meta.collections, dthemes);
Object.keys(thingies).forEach(function (key) {
var files = thingies[key]
, cached
, cdate
, fdate
;
return PromiseA.all([
fsapi.getContents(Object.keys(droot))
, fsapi.getContents(Object.keys(dfiles))
, fsapi.getContents(Object.keys(dthemes))
]).then(function (arr) {
// TODO XXX display errors in html
function noErrors(o) {
if (!o.error) {
return true;
}
console.log('files', key);
console.log(files);
files.forEach(function (file) {
var pathname = path.join(file.relativePath + '/' + file.name)
;
console.warn("Couldn't get file contents for " + o.path);
console.warn(o.error);
}
desi.content = {
root: arr[0].filter(noErrors)
, collections: arr[1].filter(noErrors)
, themes: arr[2].filter(noErrors)
};
// TODO legitimately checkout layout dependencies
if (deps && Object.keys(deps).length) {
byDirty[pathname] = file;
return;
}
if (!cacheByPath[pathname]) {
if (cacheBySha1[file.sha1]) {
// TODO rename
}
byDirty[pathname] = file;
return;
}
cached = cacheByPath[pathname];
cached.visited = true;
if (cached.sha1 && file.sha1) {
if (file.sha1 && cached.sha1 !== file.sha1) {
byDirty[pathname] = file;
return;
}
cdate = cached.lastModifiedDate && new Date(cached.lastModifiedDate);
fdate = file.lastModifiedDate && new Date(file.lastModifiedDate);
if (!cdate || !fdate || cdate !== fdate) {
byDirty[pathname] = file;
}
}
});
});
return byDirty;
}
dthemes = getDirty(desi.meta.themes);
console.log('dthemes');
console.log(dthemes);
dfiles = getDirty(desi.meta.collections, dthemes);
console.log('dfiles');
console.log(dfiles);
return fsapi.getContents(Object.keys(dthemes)).then(function (tContent) {
return fsapi.getContents(Object.keys(dfiles)).then(function (cContent) {
desi.content = { collections: cContent, themes: tContent };
return desi;
});
return desi;
});
}
console.log('');
console.log('');
console.log('getting config, data, caches...');
return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (things) {
var config = things[0]
, data = things[1]
, cache = things[2]
, partials = things[3]
console.info('getting config, data, caches...');
return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (arr) {
var config = arr[0]
, data = arr[1]
, cache = arr[2]
, partials = arr[3]
, collectionnames = Object.keys(config.collections)
, themenames = Object.keys(config.themes)
.filter(function (k) { return 'default' !== k; })
//.map(function (n) { return path.join(n, 'layouts'); })
;
console.log('loaded config, data, caches.');
console.log(things);
console.log('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
var collectionnames = Object.keys(config.collections)
;
console.info('loaded config, data, caches.');
console.log(arr);
console.info('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
return fsapi.getMeta(
collectionnames
, { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade']
}
).then(function (collections) {
var themenames = Object.keys(config.themes).filter(function (k) { return 'default' !== k; })
;
console.log('collections');
console.log(collections);
return fsapi.getMeta(
// TODO make document configurability
config.rootdir = config.rootdir || '_root';
return PromiseA.all([
fsapi.getMeta(
themenames
, { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml']
}
).then(function (themes) {
console.log('themes');
console.log(themes);
return { config: config, data: data, cache: cache, meta: { collections: collections, themes: themes }, partials: partials };
});
)
, fsapi.getMeta(
[config.rootdir]
, { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade']
}
)
, fsapi.getMeta(
collectionnames
, { dotfiles: false
, extensions: ['md', 'markdown', 'htm', 'html', 'jade']
}
)
]).then(function (things) {
function noErrors(map) {
Object.keys(map).forEach(function (path) {
map[path] = map[path].filter(function (m) {
if (!m.error && m.size) {
return true;
}
if (!m.size) {
console.warn("Ignoring 0 byte file " + (m.path || m.name));
return false;
}
console.warn("Couldn't get stats for " + (m.path || m.name));
console.warn(m.error);
});
});
return map;
}
var themes = noErrors(things[0])
, root = noErrors(things[1])[config.rootdir]
, collections = noErrors(things[2])
;
return {
config: config
, data: data
, cache: cache
, meta: {
themes: themes
, collections: collections
, root: root
}
, partials: partials
};
});
}).then(runDesi).then(function (desi) {
console.log('desi.content');
console.log(desi.content);
function readMeta(things) {
return forEachAsync(things, function (file) {
//console.log('file.contents');
//console.log(file.contents);
var parts = frontmatter.parse(file.contents)
;
if (!file.sha1) {
// TODO sha1sum
}
file.yml = parts.yml;
file.frontmatter = parts.frontmatter;
file.body = parts.body;
if (!parts.yml) {
console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
}
return Promise.resolve();
});
}
return readMeta(desi.content.themes).then(function () {
return readMeta(desi.content.collections).then(function () {
return desi;
});
return readFrontmatter(desi.content.root.concat(desi.content.themes.concat(desi.content.collections))).then(function () {
return desi;
});
}).then(function (desi) {
// TODO add missing metadata and resave file
desi.content.collections = desi.content.collections.filter(function (article) {
if (!article.yml) {
console.warn("no frontmatter for " + article.name);
console.warn(article.name);
return;
}
return true;
});
desi.content.collections.forEach(function (article) {
if (!article.yml.permalink) {
// TODO read the config for this collection
@ -263,12 +314,13 @@
var compiled = []
;
desi.content.collections.forEach(function (article) {
desi.content.collections.forEach(function (article, i) {
console.log("compiling " + (i + 1) + "/" + desi.content.collections.length + " " + (article.path || article.name));
// TODO process tags and categories and such
console.log(article.yml.title);
//console.log(article.yml.title);
//console.log(article.yml.theme);
//console.log(article.yml.layout);
console.log(article.yml.permalink);
//console.log(article.yml.permalink);
var child = ''
@ -276,13 +328,7 @@
, view
;
console.log(article.path || (article.relativePath + '/' + article.name));
//console.log(article.frontmatter);
console.log(article.yml);
layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]);
console.log('LAYERS');
console.log(layers);
view = {
page: article.yml // data for just *this* page
@ -316,10 +362,8 @@
parent.path = parent.path || article.relativePath + '/' + article.name;
if (/\.(html|htm)$/.test(parent.path)) {
console.log('thinks its html');
html = body;
} else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
console.log('parsing markdown...');
html = marked(body);
} else {
console.error('unknown parser for ' + (article.path));
@ -331,10 +375,6 @@
});
console.warn('view data.author contains objects?');
console.warn(JSON.stringify(view.data.author, null, ' '));
console.warn(typeof view.data.author.twitter_id);
console.warn(view.data.author);
// TODO add html meta-refresh redirects
compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) });
if (Array.isArray(article.yml.redirects)) {
@ -360,13 +400,43 @@
return desi;
}).then(function (desi) {
var compiled = desi.compiled
, batches = []
, now
;
console.info('das compiled files');
console.info(compiled);
return fsapi.putFiles(compiled).then(function (saved) {
console.info('files saved');
console.info(saved);
if (!compiled.length) {
console.info("No files were deemed worthy to compile. Done");
return;
}
// because some servers / proxies are terrible at handling large uploads (>= 100k)
// (vagrant? or express? one of the two is CRAZY slow)
console.info('saving compiled files');
while (compiled.length) {
batches.push(compiled.splice(0, 1));
}
now = Date.now();
console.info('compiled files');
return forEachAsync(batches, function (files) {
return fsapi.putFiles(files).then(function (saved) {
if (saved.error) {
console.error(saved.error);
}
if (!saved.errors || !saved.errors.length) {
return;
}
saved.errors.forEach(function (e) {
console.error(e);
});
//console.info('saved ' + files.length + ' files');
//console.log(saved);
});
}).then(function () {
// TODO update cache
console.info('done', ((Date.now() - now) / 1000).toFixed(3));
});
}).catch(function (e) {
console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..');

15
lib/deardesi-browser.js
View File

@ -179,7 +179,10 @@
req.open('POST', url);
req.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
// Make the request
req.send(JSON.stringify(body, null, ' '));
if ('string' !== typeof body) {
body = JSON.stringify(body);
}
req.send(body);
});
};
@ -239,16 +242,16 @@
var partials = exports.YAML.parse(resp)
;
console.info('partials');
console.info(partials);
return partials;
});
};
fsapi.putFiles = function (files) {
return request.post('/api/fs/files', {
files: files
}).then(function (resp) {
var body = { files: files };
body = JSON.stringify(body); // this isn't the slow part
console.log(files[0]);
console.info('total size:', body.length / (1024 * 1024)); // see filesize
return request.post('/api/fs/files', body).then(function (resp) {
return JSON.parse(resp);
});
};

1
package.json
View File

@ -32,6 +32,7 @@
"bluebird": "^2.5.3",
"body-parser": "^1.10.1",
"circular-json": "^0.1.6",
"compression": "^1.3.0",
"connect": "^3.3.3",
"connect-query": "^0.2.0",
"connect-send-json": "^1.0.0",

11
server.js
View File

@ -23,7 +23,16 @@ var connect = require('connect')
app
.use(send.json())
.use(query())
.use(bodyParser.json())
.use(function (req, res, next) {
console.log('before parse');
next();
})
.use(bodyParser.json({ limit: 10 * 1024 * 1024 })) // 10mb
.use(function (req, res, next) {
console.log('after parse');
next();
})
.use(require('compression')())
.use('/api/fs/walk', function (req, res, next) {
if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) {
next();