batch save all posts

deardesi.js

@@ -25,75 +25,26 @@
     }));
   }
 
-  function getLayout(desi, themename, layout, arr) {
+  function readFrontmatter(things) {
-    arr = arr || [];
+    return forEachAsync(things, function (file) {
-
+      var parts = frontmatter.parse(file.contents)
-    var layoutdir = 'layouts'
-      , themepath
-      , file
         ;
 
-    if (!themename) {
+      if (!file.sha1) {
-      themename = desi.config.themes.default;
+        // TODO sha1sum
-    }
-    if (!layout) {
-      layout = 'post.html';
       }
 
+      file.yml = parts.yml;
+      file.frontmatter = parts.frontmatter;
+      file.body = parts.body;
 
-    themepath = themename + '/' + layoutdir + '/' + layout;
+      if (!parts.yml) {
-
+        console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
-    desi.content.themes.some(function (theme) {
-      // TODO what if it isn't html?
-      if (theme.path === themepath || theme.path.match(themepath + '\\.html')) {
-        file = theme;
-        arr.push(theme);
-        return true;
       }
     });
-
-    if (!file) {
-      console.error("could not find " + themepath);
-      return;
   }
 
-    // TODO handle possible circular dep condition page -> post -> page
+  function getDirty(cacheByPath, cacheBySha1, thingies, deps) {
-    console.info(file);
-    if (file.yml && file.yml.layout) {
-      return getLayout(desi, themename, file.yml.layout, arr);
-    } else {
-      // return the chain page -> posts -> default -> twitter
-      return arr;
-    }
-  }
-
-  function runDesi(desi, development) {
-    var cache = desi.cache
-      //, config = desi.config
-      , cacheByPath = {}
-      , cacheBySha1 = {}
-      , dfiles
-      , dthemes
-      ;
-
-    desi.urls = desi.config.urls = {};
-    if (development) {
-      desi.urls.base_path = desi.config.development.base_path;
-      desi.urls.url = desi.config.development.url;
-      desi.urls.development_url = desi.config.development.url;
-    } else {
-      desi.config.base_path = desi.urls.base_path = desi.config.production.base_path;
-      desi.urls.url = desi.config.production.url;
-      desi.urls.production_url = desi.config.production.url;
-    }
-
-    cache.sources = cache.sources || [];
-    cache.sources.forEach(function (source) {
-      cacheByPath[source.path] = source;
-      cacheBySha1[source.sha1] = source;
-    });
-
-    function getDirty(thingies, deps) {
     var byDirty = {}
       ;
 
@@ -104,8 +55,6 @@
         , fdate
         ;
 
-        console.log('files', key);
-        console.log(files);
       files.forEach(function (file) {
         var pathname = path.join(file.relativePath + '/' + file.name)
           ;
@@ -149,95 +98,197 @@
     return byDirty;
   }
 
-    dthemes = getDirty(desi.meta.themes);
+  function getLayout(desi, themename, layout, arr) {
-    console.log('dthemes');
+    arr = arr || [];
-    console.log(dthemes);
 
-    dfiles = getDirty(desi.meta.collections, dthemes);
+    var layoutdir = 'layouts'
-    console.log('dfiles');
+      , themepath
-    console.log(dfiles);
+      , file
+      ;
+
+    if (!themename) {
+      themename = desi.config.themes.default;
+    }
+    if (!layout) {
+      layout = 'post.html';
+    }
+
+
+    themepath = themename + '/' + layoutdir + '/' + layout;
+
+    desi.content.themes.some(function (theme) {
+      // TODO what if it isn't html?
+      if (theme.path === themepath || theme.path.match(themepath + '\\.html')) {
+        file = theme;
+        arr.push(theme);
+        return true;
+      }
+    });
+
+    if (!file) {
+      console.error("could not find " + themepath);
+      return;
+    }
+
+    // TODO handle possible circular dep condition page -> post -> page
+    if (file.yml && file.yml.layout) {
+      return getLayout(desi, themename, file.yml.layout, arr);
+    } else {
+      // return the chain page -> posts -> default -> twitter
+      return arr;
+    }
+  }
+
+  function runDesi(desi, development) {
+    var cache = desi.cache
+      //, config = desi.config
+      , cacheByPath = {}
+      , cacheBySha1 = {}
+      , dfiles
+      , dthemes
+      , droot
+      ;
+
+    desi.urls = desi.config.urls = {};
+    if (development) {
+      desi.urls.base_path = desi.config.development.base_path;
+      desi.urls.url = desi.config.development.url;
+      desi.urls.development_url = desi.config.development.url;
+    } else {
+      desi.config.base_path = desi.urls.base_path = desi.config.production.base_path;
+      desi.urls.url = desi.config.production.url;
+      desi.urls.production_url = desi.config.production.url;
+    }
+
+    cache.sources = cache.sources || [];
+    cache.sources.forEach(function (source) {
+      cacheByPath[source.path] = source;
+      cacheBySha1[source.sha1] = source;
+    });
+
+    dthemes = getDirty(cacheByPath, cacheBySha1, desi.meta.themes);
+    droot = getDirty(cacheByPath, cacheBySha1, [desi.meta.root], dthemes);
+    dfiles = getDirty(cacheByPath, cacheBySha1, desi.meta.collections, dthemes);
+    
+    return PromiseA.all([
+      fsapi.getContents(Object.keys(droot))
+    , fsapi.getContents(Object.keys(dfiles))
+    , fsapi.getContents(Object.keys(dthemes))
+    ]).then(function (arr) {
+      // TODO XXX display errors in html
+      function noErrors(o) {
+        if (!o.error) {
+          return true;
+        }
+
+        console.warn("Couldn't get file contents for " + o.path);
+        console.warn(o.error);
+      }
+
+      desi.content = {
+        root: arr[0].filter(noErrors)
+      , collections: arr[1].filter(noErrors)
+      , themes: arr[2].filter(noErrors)
+      };
 
-    return fsapi.getContents(Object.keys(dthemes)).then(function (tContent) {
-      return fsapi.getContents(Object.keys(dfiles)).then(function (cContent) {
-        desi.content = { collections: cContent, themes: tContent };
       return desi;
     });
-    });
   }
 
   console.log('');
   console.log('');
-  console.log('getting config, data, caches...');
+  console.info('getting config, data, caches...');
-  return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (things) {
+
-    var config = things[0]
+  return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (arr) {
-      , data = things[1]
+    var config = arr[0]
-      , cache = things[2]
+      , data = arr[1]
-      , partials = things[3]
+      , cache = arr[2]
+      , partials = arr[3]
+      , collectionnames = Object.keys(config.collections)
+      , themenames = Object.keys(config.themes)
+          .filter(function (k) { return 'default' !== k; })
+          //.map(function (n) { return path.join(n, 'layouts'); })
       ;
 
-    console.log('loaded config, data, caches.');
+    console.info('loaded config, data, caches.');
-    console.log(things);
+    console.log(arr);
-    console.log('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
+    console.info('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
-    var collectionnames = Object.keys(config.collections)
-      ;
 
-    return fsapi.getMeta(
+    // TODO make document configurability
-      collectionnames
+    config.rootdir = config.rootdir || '_root';
-    , { dotfiles: false
+    return PromiseA.all([
-      , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+      fsapi.getMeta(
-      }
-    ).then(function (collections) {
-      var themenames = Object.keys(config.themes).filter(function (k) { return 'default' !== k; })
-        ;
-
-      console.log('collections');
-      console.log(collections);
-      return fsapi.getMeta(
         themenames
       , { dotfiles: false 
         , extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml']
         }
-      ).then(function (themes) {
+      )
-        console.log('themes');
+    , fsapi.getMeta(
-        console.log(themes);
+        [config.rootdir]
-        return { config: config, data: data, cache: cache, meta: { collections: collections, themes: themes }, partials: partials };
+      , { dotfiles: false
-      });
+        , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
-    });
+        }
-  }).then(runDesi).then(function (desi) {
+      )
-    console.log('desi.content');
+    , fsapi.getMeta(
-    console.log(desi.content);
+        collectionnames
+      , { dotfiles: false
+        , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+        }
+      )
+    ]).then(function (things) {
+      function noErrors(map) {
+        Object.keys(map).forEach(function (path) {
+          map[path] = map[path].filter(function (m) {
+            if (!m.error && m.size) {
+              return true;
+            }
 
-    function readMeta(things) {
+            if (!m.size) {
-      return forEachAsync(things, function (file) {
+              console.warn("Ignoring 0 byte file " + (m.path || m.name));
-        //console.log('file.contents');
+              return false;
-        //console.log(file.contents);
+            }
-        var parts = frontmatter.parse(file.contents)
+
+            console.warn("Couldn't get stats for " + (m.path || m.name));
+            console.warn(m.error);
+          });
+        });
+
+        return map;
+      }
+
+      var themes = noErrors(things[0])
+        , root = noErrors(things[1])[config.rootdir]
+        , collections = noErrors(things[2])
         ;
 
-        if (!file.sha1) {
+      return {
-          // TODO sha1sum
+        config: config
+      , data: data
+      , cache: cache
+      , meta: { 
+          themes: themes
+        , collections: collections
+        , root: root
         }
-
+      , partials: partials
-        file.yml = parts.yml;
+      };
-        file.frontmatter = parts.frontmatter;
-        file.body = parts.body;
-
-        if (!parts.yml) {
-          console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
-        }
-
-        return Promise.resolve();
     });
-    }
+  }).then(runDesi).then(function (desi) {
-
+    return readFrontmatter(desi.content.root.concat(desi.content.themes.concat(desi.content.collections))).then(function () {
-    return readMeta(desi.content.themes).then(function () {
-      return readMeta(desi.content.collections).then(function () {
       return desi;
     });
-    });
-
   }).then(function (desi) {
     // TODO add missing metadata and resave file
+    desi.content.collections = desi.content.collections.filter(function (article) {
+      if (!article.yml) {
+        console.warn("no frontmatter for " + article.name);
+        console.warn(article.name);
+        return;
+      }
+
+      return true;
+    });
+
     desi.content.collections.forEach(function (article) {
       if (!article.yml.permalink) {
         // TODO read the config for this collection
@@ -263,12 +314,13 @@
     var compiled = []
       ;
 
-    desi.content.collections.forEach(function (article) {
+    desi.content.collections.forEach(function (article, i) {
+      console.log("compiling " + (i + 1) + "/" + desi.content.collections.length + " " + (article.path || article.name));
       // TODO process tags and categories and such
-      console.log(article.yml.title);
+      //console.log(article.yml.title);
       //console.log(article.yml.theme);
       //console.log(article.yml.layout);
-      console.log(article.yml.permalink);
+      //console.log(article.yml.permalink);
 
 
       var child = ''
@@ -276,13 +328,7 @@
         , view
         ;
 
-
-      console.log(article.path || (article.relativePath + '/' + article.name));
-      //console.log(article.frontmatter);
-      console.log(article.yml);
       layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]);
-      console.log('LAYERS');
-      console.log(layers);
 
       view = {
         page: article.yml // data for just *this* page
@@ -316,10 +362,8 @@
         parent.path = parent.path || article.relativePath + '/' + article.name;
 
         if (/\.(html|htm)$/.test(parent.path)) {
-          console.log('thinks its html');
           html = body;
         } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
-          console.log('parsing markdown...');
           html = marked(body);
         } else {
           console.error('unknown parser for ' + (article.path));
@@ -331,10 +375,6 @@
 
       });
 
-      console.warn('view data.author contains objects?');
-      console.warn(JSON.stringify(view.data.author, null, '  '));
-      console.warn(typeof view.data.author.twitter_id);
-      console.warn(view.data.author);
       // TODO add html meta-refresh redirects
       compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) });
       if (Array.isArray(article.yml.redirects)) {
@@ -360,13 +400,43 @@
     return desi;
   }).then(function (desi) {
     var compiled = desi.compiled
+      , batches = []
+      , now
       ;
 
-    console.info('das compiled files');
+    if (!compiled.length) {
-    console.info(compiled);
+      console.info("No files were deemed worthy to compile. Done");
-    return fsapi.putFiles(compiled).then(function (saved) {
+      return;
-      console.info('files saved');
+    }
-      console.info(saved);
+
+    // because some servers / proxies are terrible at handling large uploads (>= 100k)
+    // (vagrant? or express? one of the two is CRAZY slow)
+    console.info('saving compiled files');
+    while (compiled.length) {
+      batches.push(compiled.splice(0, 1));
+    }
+
+    now = Date.now();
+    console.info('compiled files');
+    return forEachAsync(batches, function (files) {
+      return fsapi.putFiles(files).then(function (saved) {
+        if (saved.error) {
+          console.error(saved.error);
+        }
+
+        if (!saved.errors || !saved.errors.length) {
+          return;
+        }
+
+        saved.errors.forEach(function (e) {
+          console.error(e);
+        });
+        //console.info('saved ' + files.length + ' files');
+        //console.log(saved);
+      });
+    }).then(function () {
+      // TODO update cache
+      console.info('done', ((Date.now() - now) / 1000).toFixed(3));
     });
   }).catch(function (e) {
     console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..');

lib/deardesi-browser.js

@@ -179,7 +179,10 @@
       req.open('POST', url);
       req.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
       // Make the request
-      req.send(JSON.stringify(body, null, '  '));
+      if ('string' !== typeof body) {
+        body = JSON.stringify(body);
+      }
+      req.send(body);
     });
   };
 
@@ -239,16 +242,16 @@
       var partials = exports.YAML.parse(resp)
         ;
 
-      console.info('partials');
-      console.info(partials);
       return partials;
     });
   };
 
   fsapi.putFiles = function (files) {
-    return request.post('/api/fs/files', {
+    var body = { files: files };
-      files: files
+    body = JSON.stringify(body); // this isn't the slow part
-    }).then(function (resp) {
+    console.log(files[0]);
+    console.info('total size:', body.length / (1024 * 1024)); // see filesize
+    return request.post('/api/fs/files', body).then(function (resp) {
       return JSON.parse(resp);
     });
   };

package.json

@@ -32,6 +32,7 @@
     "bluebird": "^2.5.3",
     "body-parser": "^1.10.1",
     "circular-json": "^0.1.6",
+    "compression": "^1.3.0",
     "connect": "^3.3.3",
     "connect-query": "^0.2.0",
     "connect-send-json": "^1.0.0",

server.js

@@ -23,7 +23,16 @@ var connect     = require('connect')
 app
   .use(send.json())
   .use(query())
-  .use(bodyParser.json())
+  .use(function (req, res, next) {
+    console.log('before parse');
+    next();
+  })
+  .use(bodyParser.json({ limit: 10 * 1024 * 1024 })) // 10mb
+  .use(function (req, res, next) {
+    console.log('after parse');
+    next();
+  })
+  .use(require('compression')())
   .use('/api/fs/walk', function (req, res, next) {
       if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) {
         next();