Rendered first page. Yay!

deardesi.js

@@ -11,206 +11,262 @@
     , sha1sum       = exports.sha1sum       || require('./lib/deardesi-node').sha1sum
     , frontmatter   = exports.Frontmatter   || require('./lib/frontmatter').Frontmatter
     , safeResolve   = exports.safeResolve   || require('./lib/deardesi-utils').safeResolve
-    , getStats      = exports.getStats      || require('./lib/deardesi-node').getStats
+    , fsapi         = exports.fsapi         || require('./lib/deardesi-node').fsapi
-    , getContents   = exports.getContents   || require('./lib/deardesi-node').getContents
     ;
 
-  function getCollections(blogbase, ignorable, collectionnames) {
+  function runDesi(desi) {
-    var collectiondir
+    var config = desi.config
-      , collectiondirs = []
+      , cache = desi.cache
-      , lost = []
+      , cacheByPath = {}
-      , found = []
+      , cacheBySha1 = {}
-      , errors = []
+      , dfiles
+      , dthemes
       ;
 
-
+    cache.sources = cache.sources || [];
-    collectionnames.forEach(function (collectionname) {
+    cache.sources.forEach(function (source) {
-      collectiondir = safeResolve(_dirname, collectionname);
+      cacheByPath[source.path] = source;
-
+      cacheBySha1[source.sha1] = source;
-      if (!collectiondir) {
-        return PromiseA.reject(new Error("Please update your config.yml: " + collectionname + " is outside of '" + _dirname + "'"));
-      }
-
-      collectiondirs.push({ name: collectionname, path: collectiondir });
     });
 
-
+    function getDirty(thingies, deps) {
-    return getFolders(collectiondirs, { recursive: true, limit: 5, stats: true }).then(function (stats) {
+      var byDirty = {}
-      collectiondirs.forEach(function (collection) {
-        if (!stats[collection.path]) {
-          errors.push({
-            collection: collection
-          , message: "server did not return success or error for " + collection.path + ':\n' + JSON.stringify(stats)
-          });
-        }
-        else if (!stats[collection.path].type) {
-          lost.push(collection);
-        }
-        else if ('directory' !== stats[collection.path].type) {
-          errors.push({
-            collection: collection
-          , message: collection.path + " is not a directory (might be a symbolic link)"
-          });
-        } else {
-          found.push(collection);
-        }
-      });
-
-      return {
-        lost: lost
-      , found: found
-      , errors: errors
-      };
-    });
-  }
-
-  function showCollectionNotes(notes) {
-    if (notes.lost.length) {
-      console.warn("WARNING: these collections you specified couldn't be found");
-      notes.lost.forEach(function (node) {
-        console.warn('? ' + node.name);
-      });
-      console.log('');
-    }
-
-    if (notes.found.length) {
-      console.log("Will compile these collections");
-      notes.found.forEach(function (node) {
-        console.log('+ ' + node.name);
-      });
-      console.log('');
-    }
-  }
-
-  function getLayouts() {
-    // TODO
-  }
-
-  function updatePage(pagedir, node, lstat, data) {
-    var parts = frontmatter.parse(data)
-      , meta
-      , html
-      , view
-      ;
-
-    if (!parts.yml) {
-      console.error("Could not parse frontmatter for " + node);
-      console.error(parts.frontmatter);
-      return;
-    }
-
-    if (/\.(html|htm)$/.test(node)) {
-      html = parts.body.trim();
-    } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(node)) {
-      console.log('parsing markdown...');
-      html = marked(parts.body.trim());
-    } else {
-      console.error('unknown parser for ' + node);
-    }
-
-    meta = {
-      mtime: lstat.mtime
-    , ymlsum: sha1sum(parts.frontmatter.trim())
-    , textsum: sha1sum(parts.body.trim())
-    , htmlsum: sha1sum(html)
-    , filesum: sha1sum(data)
-    , filename: node
-    , filepath: pagedir
-    };
-
-    /*
-    // TODO
-    db.getCached(meta).error(function () {
-      // TODO rebuild and save
-    });
-    */
-
-    // TODO meta.layout
-    view = {
-      page: parts.yml
-    , content: html
-    };
-
-    console.log(node);
-    console.log(parts.frontmatter);
-    console.log(parts.yml); //Mustache.render(pagetpl, view));
-    //console.log(meta.mtime.valueOf(), meta.ymlsum, meta.textsum, node);
-
-    return meta;
-  }
-
-  function templatePosts() {
-    var pagetpl
-      , defaulttpl
-      ;
-
-    // TODO declare path to theme
-    pagetpl = frontmatter.parse(fs.readFileSync(path.join(config.theme, 'layouts', 'page.html'), 'utf8'));
-    defaulttpl = frontmatter.parse(fs.readFileSync(path.join(config.theme, 'layouts', 'default.html'), 'utf8'));
-
-
-  }
-
-  function getCollection() {
-  }
-
-  console.log('');
-  console.log('');
-  console.log('loading caches...');
-
-  getMetaCache().then(function (db) {
-    console.log('last update: ' + (db.lastUpdate && new Date(db.lastUpdate) || 'never'));
-
-    console.log('checking for local updates...');
-
-
-    // TODO get layouts here
-    return getCollections('.', Object.keys(config.collections)).then(function (notes) {
-      showCollectionNotes(notes);
-
-      return notes.found;
-    }).then(function (found) {
-      var metas = []
         ;
 
-      return forEachAsync(found, function (collection) {
+      Object.keys(thingies).forEach(function (key) {
-        begintime = Date.now();
+        var files = thingies[key]
-        console.log('begin', ((begintime - starttime) / 1000).toFixed(4));
+          , cached
+          , cdate
+          , fdate
+          ;
 
-        return fs.readdirAsync(collection.path).then(function (nodes) {
+        console.log('files', key);
+        console.log(files);
+        files.forEach(function (file) {
+          var pathname = path.join(file.relativePath + '/' + file.name)
+            ;
 
-          // TODO look for companion yml file aside html|md|jade
+
-          nodes = nodes.filter(function (node) {
+          // TODO legitimately checkout layout dependencies
-            // TODO have handlers accept or reject extensions in the order they are registered
+          if (deps && Object.keys(deps).length) {
-            if (!/\.(htm|html|md|markdown|mdown|mkdn|mkd|jade)$/.test(node)) {
+            byDirty[pathname] = file;
-              console.warn("ignoring " + collection.name + '/' + node + " (unknown filetype processor)");
+            return;
-              return false;
+          }
+
+          if (!cacheByPath[pathname]) {
+            if (cacheBySha1[file.sha1]) {
+              // TODO rename
             }
 
-            return true;
+            byDirty[pathname] = file;
-          });
+            return;
+          }
 
-          return forEachAsync(nodes, function (pagename) {
+          cached = cacheByPath[pathname];
-            var pagepath = path.join(collection.path, pagename)
+          cached.visited = true;
-              ;
 
-            // TODO: support walking deep
+          if (cached.sha1 && file.sha1) {
-            // TODO: test .html, .md, etc
+            if (file.sha1 && cached.sha1 !== file.sha1) {
-            return fs.lstatAsync(pagepath).then(function (lstat) {
+              byDirty[pathname] = file;
-              // no funny business allowed
+              return;
-              if (!lstat.isFile()) {
+            }
-                return;
-              }
 
-              return fs.readFileAsync(nodepath, 'utf8').then(function (data) {
+            cdate = cached.lastModifiedDate && new Date(cached.lastModifiedDate);
-                updatePage(pagedir, node, lstat, data);
+            fdate = file.lastModifiedDate && new Date(file.lastModifiedDate);
-              });
+
-            });
+            if (!cdate || !fdate || cdate !== fdate) {
-          });
+              byDirty[pathname] = file;
-        }).then(function () {
+            }
-          console.log('doneish', ((Date.now() - begintime) / 1000).toFixed(4));
+          }
+
+        });
+      });
+
+      return byDirty;
+    }
+
+    dthemes = getDirty(desi.meta.themes);
+    console.log('dthemes');
+    console.log(dthemes);
+
+    dfiles = getDirty(desi.meta.collections, dthemes);
+    console.log('dfiles');
+    console.log(dfiles);
+    
+    return fsapi.getContents(Object.keys(dthemes)).then(function (tContent) {
+      return fsapi.getContents(Object.keys(dfiles)).then(function (cContent) {
+        desi.content = { collections: cContent, themes: tContent };
+        return desi;
+      });
+    });
+  }
+
+  console.log('');
+  console.log('');
+  console.log('getting config...');
+  return fsapi.getConfig().then(function (config) {
+    console.log('loading caches...');
+    return fsapi.getCache().then(function (cache) {
+      console.log('cache');
+      console.log(cache);
+      console.log('last update: ' + (cache.lastUpdate && new Date(cache.lastUpdate) || 'never'));
+      var collectionnames = Object.keys(config.collections)
+        ;
+
+      return fsapi.getMeta(
+        collectionnames
+      , { dotfiles: false
+        , extensions: ['md', 'markdown', 'htm', 'html', 'jade']
+        }
+      ).then(function (collections) {
+        var themenames = Object.keys(config.themes).filter(function (k) { return 'default' !== k; })
+          ;
+
+        console.log('collections');
+        console.log(collections);
+        return fsapi.getMeta(
+          themenames
+        , { dotfiles: false 
+          , extensions: ['md', 'markdown', 'htm', 'html', 'jade', 'css', 'js', 'yml']
+          }
+        ).then(function (themes) {
+          console.log('themes');
+          console.log(themes);
+          return { config: config, cache: cache, meta: { collections: collections, themes: themes } };
         });
       });
     });
+  }).then(runDesi).then(function (desi) {
+    console.log('desi.content');
+    console.log(desi.content);
+
+    function readMeta(things) {
+      return forEachAsync(things, function (file) {
+        //console.log('file.contents');
+        //console.log(file.contents);
+        var parts = frontmatter.parse(file.contents)
+          ;
+
+        if (!file.sha1) {
+          // TODO sha1sum
+        }
+
+        file.yml = parts.yml;
+        file.frontmatter = parts.frontmatter;
+        file.body = parts.body;
+
+        if (!parts.yml) {
+          console.warn("No frontmatter for " + (file.path || (file.relativePath + '/' + file.name)));
+        }
+
+        return Promise.resolve();
+      });
+    }
+
+    return readMeta(desi.content.themes).then(function () {
+      return readMeta(desi.content.collections).then(function () {
+        return desi;
+      });
+    });
+  }).then(function (desi) {
+    function getLayout(themename, layout, arr) {
+      arr = arr || [];
+
+      var layoutdir = 'layouts'
+        , themepath
+        , file
+        ;
+
+      if (!themename) {
+        themename = desi.config.themes.default;
+      }
+      if (!layout) {
+        layout = 'post.html';
+      }
+
+
+      themepath = themename + '/' + layoutdir + '/' + layout;
+
+      desi.content.themes.some(function (theme) {
+        // TODO what if it isn't html?
+        if (theme.path === themepath || theme.path.match(themepath + '\\.html')) {
+          file = theme;
+          arr.push(theme);
+          return true;
+        }
+      });
+
+      if (!file) {
+        console.error("could not find " + themepath);
+        return;
+      }
+
+      // TODO handle possible circular dep condition page -> post -> page
+      console.info(file);
+      if (file.yml && file.yml.layout) {
+        return getLayout(themename, file.yml.layout, arr);
+      } else {
+        // return the chain page -> posts -> default -> twitter
+        return arr;
+      }
+    }
+
+    desi.content.collections.forEach(function (article) {
+      // TODO process tags and categories and such
+      console.log(article.yml.title);
+      //console.log(article.yml.theme);
+      //console.log(article.yml.layout);
+      console.log(article.yml.permalink);
+
+
+      var child = ''
+        , layers
+        ;
+
+
+      console.log(article.path || (article.relativePath + '/' + article.name));
+      //console.log(article.frontmatter);
+      console.log(article.yml);
+      layers = getLayout(article.yml.theme, article.yml.layout, [article]);
+      console.log('LAYERS');
+      console.log(layers);
+      layers.forEach(function (parent) {
+        // TODO meta.layout
+        var view
+          , body = (parent.body || parent.contents || '').trim()
+          , html
+          ;
+
+        parent.path = parent.path || article.relativePath + '/' + article.name;
+
+        if (/\.(html|htm)$/.test(parent.path)) {
+          console.log('thinks its html');
+          html = body;
+        } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
+          console.log('parsing markdown...');
+          html = marked(body);
+        } else {
+          console.error('unknown parser for ' + (article.path));
+        }
+
+        view = {
+          page: article.yml // data for just *this* page
+        , content: child    // processed content for just *this* page
+        , data: {}          // data.yml
+        , collection: {}    // data for just *this* collection
+        , categories: []    // *all* categories in all collections
+        , tags: []          // *all* tags in all collections
+        };
+
+        child = Mustache.render(html, view);
+      });
+
+      console.log('child');
+      console.log(child);
+      //console.log(meta.mtime.valueOf(), meta.ymlsum, meta.textsum, node);
+    });
+  }).catch(function (e) {
+    console.error('The Badness is upon us...');
+    throw e;
   });
 }('undefined' !== typeof exports && exports || window));

lib/deardesi-browser.js

@@ -2,6 +2,9 @@
 ;(function (exports) {
   'use strict';
 
+  window.YAML = window.YAML || {};
+  window.YAML.parse = exports.jsyaml.load || require('jsyaml').load;
+
   // Chrome, Firefox, and even MSIE11+ all support crypto
   var crypto = window.crypto || window.msCrypto
     , algos
@@ -108,9 +111,12 @@
   exports.sha1sum = sha1sum;
 }('undefined' !== typeof exports && exports || window));
 
-;(function () {
+;(function (exports) {
   'use strict';
 
+  var fsapi
+    ;
+
   function request() {
   }
   request.get = function (url/*, query*/) {
@@ -177,18 +183,58 @@
     });
   };
 
-  exports.getCollectionMeta = function (collections) {
+  
-    return request.post('/api/fs/walk?_method=GET&dotfiles=true&extensions=md,markdown,jade,htm,html', {
+  exports.fsapi = fsapi = {};
+  fsapi.getMeta = function (collections, opts) {
+    opts = opts || {};
+
+    var extensions = ''
+      , dotfiles = ''
+      ;
+
+    if (Array.isArray(opts.extensions)) {
+      extensions = '&extensions=' + opts.extensions.join(','); // md,markdown,jade,htm,html
+    }
+    if (opts.dotfiles) {
+      dotfiles = '&dotfiles=true';
+    }
+
+    return request.post('/api/fs/walk?_method=GET' + dotfiles + extensions, {
       dirs: collections
     }).then(function (resp) {
-      return JSON.parse(resp.responseText);
+      return JSON.parse(resp);
+    });
+  };
+
+  fsapi.getContents = function (filepaths) {
+    return request.post('/api/fs/files?_method=GET', {
+      paths: filepaths
+    }).then(function (resp) {
+      return JSON.parse(resp);
+    });
+  };
+
+  fsapi.getConfig = function () {
+    return request.get('/config.yml').then(function (resp) {
+      console.log('config');
+      console.log(exports.YAML.parse(resp));
+      return exports.YAML.parse(resp);
+    });
+  };
+
+  fsapi.getCache = function () {
+    return request.get('/cache.json').then(function (resp) {
+      return JSON.parse(resp);
+    }).catch(function () {
+      return {};
+    });
+  };
+
+  fsapi.putFiles = function (files) {
+    return request.put('/api/fs/files', {
+      files: files
+    }).then(function (resp) {
+      return JSON.parse(resp);
     });
   };
-  /*
-  exports.getStats
-  exports.getContents
-  exports.getMetaCache
-  exports.getContentCache
-  */
-  //require('./db').create(path.join(_dirname, 'db.json'))
 }('undefined' !== typeof exports && exports || window));

lib/frontmatter.js

@@ -2,10 +2,8 @@
 ;(function (exports) {
   'use strict';
 
-  var YAML = {}
+  window.YAML = window.YAML || {};
-    ;
+  window.YAML.parse = exports.jsyaml.load || require('jsyaml').load;
-
-  YAML.parse = exports.jsyaml.load || require('jsyaml').load;
   //YAML.parse = require('yaml').eval;
   //YAML.parse2 = require('yamljs').parse;
 
@@ -69,7 +67,7 @@
     // strip frontmatter from text, if any
     // including trailing '---' (which is accounted for by the added '\n')
     if (yml) {
-      len = fm.split(/\n/).length;
+      len = fm.split(/\n/).length + 1;
     } else {
       len = 0;
     }
@@ -79,14 +77,16 @@
 
   function parseText(text) {
     var fm = readFrontMatter(text)
-      , body = separateText(text, fm)
+      , body = fm && separateText(text, fm)
       , yml
       ;
 
-    try {
+    if (fm) {
-      yml = YAML.parse(fm);
+      try {
-    } catch(e) {
+        yml = window.YAML.parse(fm);
-      //
+      } catch(e) {
+        //
+      }
     }
 
     return {

lib/fsapi.js

@@ -87,9 +87,14 @@ function walkDir(parent, sub, opts) {
         files.push(file);
 
         if (opts.contents) {
-          return fs.readFileAsync(path.join(root, stat.name), 'utf8').then(function (contents) {
+          return fs.readFileAsync(path.join(root, stat.name), null).then(function (buffer) {
+            var contents = buffer.toString('utf8')
+              ;
+
             file.contents = contents;
             file.sha1 = sha1sum(contents);
+            file.size = buffer.length;
+            file.type = undefined;
           });
         }
       }
@@ -136,7 +141,7 @@ function getfs(blogdir, filepaths) {
       return fs.readFileAsync(pathname, null).then(function (buffer) {
         files.push({
           path: filepath
-        , size: buffer.byteLength
+        , size: buffer.length
         , lastModifiedDate: stat.mtime.toISOString()
         , contents: buffer.toString('utf8')
         , sha1: sha1sum(buffer)