batch save all posts

deardesi.js

@@ -25,6 +25,79 @@
     }));
   }
 
+  function readFrontmatter(things) {
+    return forEachAsync(things, function (file) {
+      var parts = frontmatter.parse(file.contents)
+        ;
+
+      if (!file.sha1) {
+        // TODO sha1sum
+      }
+
+      file.yml = parts.yml;
+      file.frontmatter = parts.frontmatter;
+      file.body = parts.body;
+
+      if (!parts.yml) {
+        console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
+      }
+    });
+  }
+
+  function getDirty(cacheByPath, cacheBySha1, thingies, deps) {
+    var byDirty = {}
+      ;
+
+    Object.keys(thingies).forEach(function (key) {
+      var files = thingies[key]
+        , cached
+        , cdate
+        , fdate
+        ;
+
+      files.forEach(function (file) {
+        var pathname = path.join(file.relativePath + '/' + file.name)
+          ;
+
+
+        // TODO legitimately checkout layout dependencies
+        if (deps && Object.keys(deps).length) {
+          byDirty[pathname] = file;
+          return;
+        }
+
+        if (!cacheByPath[pathname]) {
+          if (cacheBySha1[file.sha1]) {
+            // TODO rename
+          }
+
+          byDirty[pathname] = file;
+          return;
+        }
+
+        cached = cacheByPath[pathname];
+        cached.visited = true;
+
+        if (cached.sha1 && file.sha1) {
+          if (file.sha1 && cached.sha1 !== file.sha1) {
+            byDirty[pathname] = file;
+            return;
+          }
+
+          cdate = cached.lastModifiedDate && new Date(cached.lastModifiedDate);
+          fdate = file.lastModifiedDate && new Date(file.lastModifiedDate);
+
+          if (!cdate || !fdate || cdate !== fdate) {
+            byDirty[pathname] = file;
+          }
+        }
+
+      });
+    });
+
+    return byDirty;
+  }
+
   function getLayout(desi, themename, layout, arr) {
     arr = arr || [];
 
@@ -58,7 +131,6 @@
     }
 
     // TODO handle possible circular dep condition page -> post -> page
-    console.info(file);
     if (file.yml && file.yml.layout) {
       return getLayout(desi, themename, file.yml.layout, arr);
     } else {
@@ -74,6 +146,7 @@
       , cacheBySha1 = {}
       , dfiles
       , dthemes
+      , droot
       ;
 
     desi.urls = desi.config.urls = {};
@@ -93,151 +166,129 @@
       cacheBySha1[source.sha1] = source;
     });
 
-    function getDirty(thingies, deps) {
+    dthemes = getDirty(cacheByPath, cacheBySha1, desi.meta.themes);
-      var byDirty = {}
+    droot = getDirty(cacheByPath, cacheBySha1, [desi.meta.root], dthemes);
-        ;
+    dfiles = getDirty(cacheByPath, cacheBySha1, desi.meta.collections, dthemes);
     
-      Object.keys(thingies).forEach(function (key) {
+    return PromiseA.all([
-        var files = thingies[key]
+      fsapi.getContents(Object.keys(droot))
-          , cached
+    , fsapi.getContents(Object.keys(dfiles))
-          , cdate
+    , fsapi.getContents(Object.keys(dthemes))
-          , fdate
+    ]).then(function (arr) {
-          ;
+      // TODO XXX display errors in html
+      function noErrors(o) {
+        if (!o.error) {
+          return true;
+        }
 
-        console.log('files', key);
+        console.warn("Couldn't get file contents for " + o.path);
-        console.log(files);
+        console.warn(o.error);
-        files.forEach(function (file) {
+      }
-          var pathname = path.join(file.relativePath + '/' + file.name)
-            ;
 
+      desi.content = {
+        root: arr[0].filter(noErrors)
+      , collections: arr[1].filter(noErrors)
+      , themes: arr[2].filter(noErrors)
+      };
 
-          // TODO legitimately checkout layout dependencies
+      return desi;
-          if (deps && Object.keys(deps).length) {
-            byDirty[pathname] = file;
-            return;
-          }
-
-          if (!cacheByPath[pathname]) {
-            if (cacheBySha1[file.sha1]) {
-              // TODO rename
-            }
-
-            byDirty[pathname] = file;
-            return;
-          }
-
-          cached = cacheByPath[pathname];
-          cached.visited = true;
-
-          if (cached.sha1 && file.sha1) {
-            if (file.sha1 && cached.sha1 !== file.sha1) {
-              byDirty[pathname] = file;
-              return;
-            }
-
-            cdate = cached.lastModifiedDate && new Date(cached.lastModifiedDate);
-            fdate = file.lastModifiedDate && new Date(file.lastModifiedDate);
-
-            if (!cdate || !fdate || cdate !== fdate) {
-              byDirty[pathname] = file;
-            }
-          }
-
-        });
-      });
-
-      return byDirty;
-    }
-
-    dthemes = getDirty(desi.meta.themes);
-    console.log('dthemes');
-    console.log(dthemes);
-
-    dfiles = getDirty(desi.meta.collections, dthemes);
-    console.log('dfiles');
-    console.log(dfiles);
-    
-    return fsapi.getContents(Object.keys(dthemes)).then(function (tContent) {
-      return fsapi.getContents(Object.keys(dfiles)).then(function (cContent) {
-        desi.content = { collections: cContent, themes: tContent };
-        return desi;
-      });
     });
   }
 
   console.log('');
   console.log('');
-  console.log('getting config, data, caches...');
+  console.info('getting config, data, caches...');
-  return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (things) {
+
-    var config = things[0]
+  return PromiseA.all([fsapi.getConfig(), fsapi.getData(), fsapi.getCache(), fsapi.getPartials()]).then(function (arr) {
-      , data = things[1]
+    var config = arr[0]
-      , cache = things[2]
+      , data = arr[1]
-      , partials = things[3]
+      , cache = arr[2]
+      , partials = arr[3]
+      , collectionnames = Object.keys(config.collections)
+      , themenames = Object.keys(config.themes)
+          .filter(function (k) { return 'default' !== k; })
+          //.map(function (n) { return path.join(n, 'layouts'); })
       ;
 
-    console.log('loaded config, data, caches.');
+    console.info('loaded config, data, caches.');
-    console.log(things);
+    console.log(arr);
-    console.log('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
+    console.info('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
-    var collectionnames = Object.keys(config.collections)
-      ;
 
-    return fsapi.getMeta(
+    // TODO make document configurability
-      collectionnames
+    config.rootdir = config.rootdir || '_root';
-    , { dotfiles: false
+    return PromiseA.all([
-      , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+      fsapi.getMeta(
-      }
-    ).then(function (collections) {
-      var themenames = Object.keys(config.themes).filter(function (k) { return 'default' !== k; })
-        ;
-
-      console.log('collections');
-      console.log(collections);
-      return fsapi.getMeta(
         themenames
       , { dotfiles: false 
         , extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml']
         }
-      ).then(function (themes) {
+      )
-        console.log('themes');
+    , fsapi.getMeta(
-        console.log(themes);
+        [config.rootdir]
-        return { config: config, data: data, cache: cache, meta: { collections: collections, themes: themes }, partials: partials };
+      , { dotfiles: false
-      });
+        , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+        }
+      )
+    , fsapi.getMeta(
+        collectionnames
+      , { dotfiles: false
+        , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+        }
+      )
+    ]).then(function (things) {
+      function noErrors(map) {
+        Object.keys(map).forEach(function (path) {
+          map[path] = map[path].filter(function (m) {
+            if (!m.error && m.size) {
+              return true;
+            }
+
+            if (!m.size) {
+              console.warn("Ignoring 0 byte file " + (m.path || m.name));
+              return false;
+            }
+
+            console.warn("Couldn't get stats for " + (m.path || m.name));
+            console.warn(m.error);
+          });
+        });
+
+        return map;
+      }
+
+      var themes = noErrors(things[0])
+        , root = noErrors(things[1])[config.rootdir]
+        , collections = noErrors(things[2])
+        ;
+
+      return {
+        config: config
+      , data: data
+      , cache: cache
+      , meta: { 
+          themes: themes
+        , collections: collections
+        , root: root
+        }
+      , partials: partials
+      };
     });
   }).then(runDesi).then(function (desi) {
-    console.log('desi.content');
+    return readFrontmatter(desi.content.root.concat(desi.content.themes.concat(desi.content.collections))).then(function () {
-    console.log(desi.content);
+      return desi;
-
-    function readMeta(things) {
-      return forEachAsync(things, function (file) {
-        //console.log('file.contents');
-        //console.log(file.contents);
-        var parts = frontmatter.parse(file.contents)
-          ;
-
-        if (!file.sha1) {
-          // TODO sha1sum
-        }
-
-        file.yml = parts.yml;
-        file.frontmatter = parts.frontmatter;
-        file.body = parts.body;
-
-        if (!parts.yml) {
-          console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
-        }
-
-        return Promise.resolve();
-      });
-    }
-
-    return readMeta(desi.content.themes).then(function () {
-      return readMeta(desi.content.collections).then(function () {
-        return desi;
-      });
     });
-
   }).then(function (desi) {
     // TODO add missing metadata and resave file
+    desi.content.collections = desi.content.collections.filter(function (article) {
+      if (!article.yml) {
+        console.warn("no frontmatter for " + article.name);
+        console.warn(article.name);
+        return;
+      }
+
+      return true;
+    });
+
     desi.content.collections.forEach(function (article) {
       if (!article.yml.permalink) {
         // TODO read the config for this collection
@@ -263,12 +314,13 @@
     var compiled = []
       ;
 
-    desi.content.collections.forEach(function (article) {
+    desi.content.collections.forEach(function (article, i) {
+      console.log("compiling " + (i + 1) + "/" + desi.content.collections.length + " " + (article.path || article.name));
       // TODO process tags and categories and such
-      console.log(article.yml.title);
+      //console.log(article.yml.title);
       //console.log(article.yml.theme);
       //console.log(article.yml.layout);
-      console.log(article.yml.permalink);
+      //console.log(article.yml.permalink);
 
 
       var child = ''
@@ -276,13 +328,7 @@
         , view
         ;
 
-
-      console.log(article.path || (article.relativePath + '/' + article.name));
-      //console.log(article.frontmatter);
-      console.log(article.yml);
       layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]);
-      console.log('LAYERS');
-      console.log(layers);
 
       view = {
         page: article.yml // data for just *this* page
@@ -316,10 +362,8 @@
         parent.path = parent.path || article.relativePath + '/' + article.name;
 
         if (/\.(html|htm)$/.test(parent.path)) {
-          console.log('thinks its html');
           html = body;
         } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
-          console.log('parsing markdown...');
           html = marked(body);
         } else {
           console.error('unknown parser for ' + (article.path));
@@ -331,10 +375,6 @@
 
       });
 
-      console.warn('view data.author contains objects?');
-      console.warn(JSON.stringify(view.data.author, null, '  '));
-      console.warn(typeof view.data.author.twitter_id);
-      console.warn(view.data.author);
       // TODO add html meta-refresh redirects
       compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) });
       if (Array.isArray(article.yml.redirects)) {
@@ -360,13 +400,43 @@
     return desi;
   }).then(function (desi) {
     var compiled = desi.compiled
+      , batches = []
+      , now
       ;
 
-    console.info('das compiled files');
+    if (!compiled.length) {
-    console.info(compiled);
+      console.info("No files were deemed worthy to compile. Done");
-    return fsapi.putFiles(compiled).then(function (saved) {
+      return;
-      console.info('files saved');
+    }
-      console.info(saved);
+
+    // because some servers / proxies are terrible at handling large uploads (>= 100k)
+    // (vagrant? or express? one of the two is CRAZY slow)
+    console.info('saving compiled files');
+    while (compiled.length) {
+      batches.push(compiled.splice(0, 1));
+    }
+
+    now = Date.now();
+    console.info('compiled files');
+    return forEachAsync(batches, function (files) {
+      return fsapi.putFiles(files).then(function (saved) {
+        if (saved.error) {
+          console.error(saved.error);
+        }
+
+        if (!saved.errors || !saved.errors.length) {
+          return;
+        }
+
+        saved.errors.forEach(function (e) {
+          console.error(e);
+        });
+        //console.info('saved ' + files.length + ' files');
+        //console.log(saved);
+      });
+    }).then(function () {
+      // TODO update cache
+      console.info('done', ((Date.now() - now) / 1000).toFixed(3));
     });
   }).catch(function (e) {
     console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..');

lib/deardesi-browser.js

@@ -179,7 +179,10 @@
       req.open('POST', url);
       req.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
       // Make the request
-      req.send(JSON.stringify(body, null, '  '));
+      if ('string' !== typeof body) {
+        body = JSON.stringify(body);
+      }
+      req.send(body);
     });
   };
 
@@ -239,16 +242,16 @@
       var partials = exports.YAML.parse(resp)
         ;
 
-      console.info('partials');
-      console.info(partials);
       return partials;
     });
   };
 
   fsapi.putFiles = function (files) {
-    return request.post('/api/fs/files', {
+    var body = { files: files };
-      files: files
+    body = JSON.stringify(body); // this isn't the slow part
-    }).then(function (resp) {
+    console.log(files[0]);
+    console.info('total size:', body.length / (1024 * 1024)); // see filesize
+    return request.post('/api/fs/files', body).then(function (resp) {
       return JSON.parse(resp);
     });
   };

package.json

@@ -32,6 +32,7 @@
     "bluebird": "^2.5.3",
     "body-parser": "^1.10.1",
     "circular-json": "^0.1.6",
+    "compression": "^1.3.0",
     "connect": "^3.3.3",
     "connect-query": "^0.2.0",
     "connect-send-json": "^1.0.0",

server.js

@@ -23,7 +23,16 @@ var connect     = require('connect')
 app
   .use(send.json())
   .use(query())
-  .use(bodyParser.json())
+  .use(function (req, res, next) {
+    console.log('before parse');
+    next();
+  })
+  .use(bodyParser.json({ limit: 10 * 1024 * 1024 })) // 10mb
+  .use(function (req, res, next) {
+    console.log('after parse');
+    next();
+  })
+  .use(require('compression')())
   .use('/api/fs/walk', function (req, res, next) {
       if (!(/^GET$/i.test(req.method) || /^GET$/i.test(req.query._method))) {
         next();