collate articles

deardesi.js

@@ -13,8 +13,68 @@
     //, safeResolve   = exports.safeResolve   || require('./lib/deardesi-utils').safeResolve
     , fsapi         = exports.fsapi         || require('./lib/deardesi-node').fsapi
     //, UUID          = exports.uuid          || require('node-uuid')
+    , months
     ;
 
+  months = {
+    1: 'January'
+  , 2: 'February'
+  , 3: 'March'
+  , 4: 'April'
+  , 5: 'May'
+  , 6: 'June'
+  , 7: 'July'
+  , 8: 'August'
+  , 9: 'September'
+  , 10: 'October'
+  , 11: 'November'
+  , 12: 'December'
+  };
+
+  function pad(str) {
+    str = str.toString();
+    if (str.length < 2) {
+      return '0' + str;
+    }
+
+    return str;
+  }
+
+  function toLocaleDate(d) {
+    return d.getFullYear() + '-' + (d.getMonth() + 1) + '-' + d.getDate()
+      + ' '
+      + (d.getHours() % 12) + ':' + pad(d.getMinutes()) + ':' + pad(d.getSeconds())
+      ;
+  }
+
+  function fromLocaleDate(str) {
+    // handles ISO and ISO-ish dates
+    var m = str.match(/(\d\d\d\d)-(\d{1,2})-(\d{1,2})([T\s](\d{1,2}):(\d{1,2})(:(\d{1,2}))?)?/)
+      ;
+
+    if (!m) {
+      return [];
+    }
+
+    m.year   = m[1];
+    m.month  = m[2];
+    m.day    = m[3];
+
+    m.hour   = m[4] = pad(m[5] || '00');  // hours
+    m.minute = m[5] = pad(m[6] || '00');  // minutes
+    m.second = m[6] = pad(m[8] || '00');  // seconds
+ 
+    if (m[4] > 12) {
+      m.twelve_hour = m[7] = m[4] - 12;   // 12-hour
+      m.meridian    = m[8] = 'pm';        // am/pm
+    } else {
+      m.twelve_hour = m[7] = m[4]; 
+      m.meridian    = m[8] = 'am';
+    }
+
+    return m;
+  }
+
   // See https://github.com/janl/mustache.js/issues/415
   function num2str(obj) {
     return JSON.parse(JSON.stringify(obj, function (key, val) {
@@ -279,6 +339,40 @@
     });
   }).then(function (desi) {
     // TODO add missing metadata and resave file
+    desi.navigation = [];
+
+    desi.content.root.forEach(function (page) {
+      var name = path.basename(page.path, path.extname(page.path))
+        ;
+
+      //if (-1 === desi.data.navigation.indexOf(name) && 'index' !== name)
+      if (-1 === desi.data.navigation.indexOf(name)) {
+        return;
+      }
+
+      desi.navigation.push({
+        title: page.yml && page.yml.title || name.replace(/^./, function ($1) { return $1.toUpperCase(); })
+      , href: '/' + name
+      , path: '/' + name
+      , name: name
+      , active: false // placeholder
+      });
+    });
+
+
+    desi.content.root.forEach(function (page) {
+      page.yml = page.yml || {};
+      page.yml.layout = page.yml.layout || 'default';
+
+      if (!page.relativePath) {
+        page.relativePath = path.dirname(page.path);
+        page.name = page.name || path.basename(page.path);
+      }
+
+      page.relativePath = page.relativePath.replace(desi.config.rootdir, '').replace(/^\//, '');
+      page.path = path.join(page.relativePath, page.name);
+    });
+
     desi.content.collections = desi.content.collections.filter(function (article) {
       if (!article.yml) {
         console.warn("no frontmatter for " + article.name);
@@ -286,52 +380,190 @@
         return;
       }
 
+      if (!article.body || !article.body.trim()) {
+        console.warn('Ignoring empty content file ' + (article.path || article.name));
+        return;
+      }
+
       return true;
     });
 
-    desi.content.collections.forEach(function (article) {
-      if (!article.yml.permalink) {
-        // TODO read the config for this collection
-        article.yml.permalink = path.join(desi.urls.base_path, article.title);
-      }
 
-      if (!article.yml.uuid) {
+    function normalizeFrontmatter(page) {
+      var yml = page.yml
+        ;
+
+      // TODO read the config for this collection for how to create premalink
+      if (!yml.permalink) {
+        page.path = page.path || path.join(page.relativePath, page.name);
+        // TODO strip '_root' or whatever
+        // strip .html, .md, .jade, etc
+        yml.permalink = path.join(desi.urls.base_path, path.basename(page.path, path.extname(page.path)));
+      }
+      //yml.permalinkBase = path.join(path.dirname(yml.permalink), path.basename(yml.permalink, path.extname(yml.permalink)));
+      yml.permalink = path.join(path.dirname(yml.permalink), path.basename(yml.permalink, path.extname(yml.permalink)));
+
+      if (!page.yml.uuid) {
         // TODO only do this if it's going to be saved
-        // article.yml.uuid = UUID.v4();
+        // page.yml.uuid = UUID.v4();
       }
 
-      if (!article.yml.date) {
+      if (!page.yml.date) {
-        article.yml.date = article.createdDate || article.lastModifiedDate;
+        // TODO tell YAML parser to keep the date a string
+        page.yml.date = new Date(page.yml.created_at || page.yml.time || page.createdDate || page.lastModifiedDate).toISOString();
+      }
+      if ('object' === typeof page.yml.date) {
+        page.yml.date = page.yml.date.toISOString();
       }
 
-      if (!article.yml.updated_at) {
+      if (!page.yml.updated_at) {
-        article.yml.updated_at = article.lastModifiedDate;
+        page.yml.updated_at = page.lastModifiedDate;
       }
-    });
+    }
+
+    function normalizeContentEntity(entity) {
+      // The root index is the one exception
+      if (/^\/?index$/.test(entity.yml.permalink)) {
+        console.info('found index', entity);
+        entity.yml.permalink = '';
+      }
+
+      entity.url            = desi.urls.url + path.join(desi.urls.base_path, entity.yml.permalink, 'index.html');
+      entity.cananical_url  = desi.urls.url + path.join(desi.urls.base_path, entity.yml.permalink, 'index.html');
+      entity.relative_url   = path.join(desi.urls.base_path, entity.yml.permalink, 'index.html');
+      entity.published_at   = fromLocaleDate(entity.yml.date);
+      entity.year = entity.published_at.year;
+      entity.month = entity.published_at.month;
+      entity.day = entity.published_at.day;
+      entity.hour = entity.published_at.hour;
+      entity.twelve_hour = entity.published_at.twelve_hour;
+      entity.meridian = entity.published_at.meridian;
+      entity.minute = entity.published_at.minute;
+      entity.title = entity.yml.title;
+      // let's just agree that that's too far
+      //entity.second = entity.published_at.second;
+    }
+
+    function byDate(a, b) {
+      if (a.year > b.year) {
+        return -1;
+      } else if (a.year < b.year) {
+        return 1;
+      }
+
+      if (a.month > b.month) {
+        return -1;
+      } else if (a.month < b.month) {
+        return 1;
+      }
+
+      if (a.day > b.day) {
+        return -1;
+      } else if (a.day < b.day) {
+        return 1;
+      }
+
+      if (a.hour > b.hour) {
+        return -1;
+      } else if (a.hour < b.hour) {
+        return 1;
+      }
+
+      if (a.minute > b.minute) {
+        return -1;
+      } else if (a.minute < b.minute) {
+        return 1;
+      }
+
+      if (a.title.toLowerCase() <= b.title.toLowerCase()) {
+        return -1;
+      }
+
+      return 1;
+    }
+
+    function collate(entities) {
+      var yearsArr = []
+        ;
+
+      entities.forEach(function (f) {
+        var set
+          , yindex = 3000 - f.year
+          , mindex = 12 - f.month
+          ;
+
+        if (!yearsArr[yindex]) {
+          console.log(f.year);
+          yearsArr[yindex] = { year: f.year, months: [] };
+        }
+        set = yearsArr[yindex];
+
+        if (!set.months[mindex]) {
+          set.months[mindex] = { month: f.month, pages: [] };
+        }
+        set = set.months[mindex];
+
+        set.pages.push(f);
+      });
+
+      yearsArr = yearsArr.filter(function (y) {
+        if (!y) {
+          return false;
+        }
+
+        y.months = y.months.filter(function (m) {
+          return m && m.pages.length;
+        });
+
+        if (!y.months.length) {
+          return false;
+        }
+
+        return true;
+      });
+
+      console.log(yearsArr);
+      return { years: yearsArr };
+    }
+
+    desi.content.root.forEach(normalizeFrontmatter);
+    // TODO process tags and categories and such
+    desi.content.collections.forEach(normalizeFrontmatter);
+
+    desi.content.root.forEach(normalizeContentEntity);
+    desi.content.collections.forEach(normalizeContentEntity);
+
+    desi.content.collections.sort(byDate);
+    desi.collated = collate(desi.content.collections);
+    console.info('desi.collated');
+    console.info(desi.collated);
 
     return desi;
   }).then(function (desi) {
     var compiled = []
       ;
 
-    desi.content.collections.forEach(function (article, i) {
+    /*
-      console.log("compiling " + (i + 1) + "/" + desi.content.collections.length + " " + (article.path || article.name));
+    function compileScriptEntity(entity, i, arr) {
-      // TODO process tags and categories and such
+    }
-      //console.log(article.yml.title);
+    */
-      //console.log(article.yml.theme);
+    function compileThemeEntity(entity, i, arr) {
-      //console.log(article.yml.layout);
+      console.log("compiling " + (i + 1) + "/" + arr.length + " " + (entity.path || entity.name));
-      //console.log(article.yml.permalink);
+      // TODO less / sass / etc
-
+      compiled.push({ contents: entity.body || entity.contents, path: path.join(desi.config.compiled_path, 'themes', entity.path) });
+    }
+    function compileContentEntity(entity, i, arr) {
+      console.log("compiling " + (i + 1) + "/" + arr.length + " " + (entity.path || entity.name));
 
       var child = ''
         , layers
         , view
         ;
 
-      layers = getLayout(desi, article.yml.theme, article.yml.layout, [article]);
+      layers = getLayout(desi, entity.yml.theme, entity.yml.layout, [entity]);
 
       view = {
-        page: article.yml // data for just *this* page
+        page: entity.yml // data for just *this* page
       , content: child    // processed content for just *this* page
       //, data: desi.data   // data.yml
       // https://github.com/janl/mustache.js/issues/415
@@ -340,18 +572,16 @@
       , categories: []    // *all* categories in all collections
       , tags: []          // *all* tags in all collections
       , site: num2str(desi.site || {})
-      , url: path.join(desi.urls.url, desi.urls.base_path, article.yml.permalink)
+      , url: entity.cananical_url
-      , canonical_url: path.join(desi.urls.url, desi.urls.base_path, article.yml.permalink)
+      , canonical_url: entity.cananical_url 
-      , relative_url: path.join(desi.urls.base_path, article.yml.permalink)
+      , relative_url: entity.relative_url
       , urls: desi.urls
+      , previous: arr[i - 1]
+      , next: arr[i + 1]
+      , posts: { collated: desi.collated }
       };
       view.site.author = desi.data.author;
-      view.site['navigation?to_pages'] = desi.data.navigation.map(function (nav) {
+      view.site['navigation?to_pages'] = desi.navigation.slice(0);
-        var title = nav.replace(/^./, function ($1) { return $1.toUpperCase(); })
-          ;
-
-        return { path: '/' + nav, active: false, title: /*TODO*/ title };
-      });
 
       layers.forEach(function (parent) {
         // TODO meta.layout
@@ -359,49 +589,64 @@
           , html
           ;
 
-        parent.path = parent.path || article.relativePath + '/' + article.name;
+        parent.path = parent.path || entity.relativePath + '/' + entity.name;
 
         if (/\.(html|htm)$/.test(parent.path)) {
           html = body;
         } else if (/\.(md|markdown|mdown|mkdn|mkd|mdwn|mdtxt|mdtext)$/.test(parent.path)) {
           html = marked(body);
         } else {
-          console.error('unknown parser for ' + (article.path));
+          console.error('unknown parser for ' + (entity.path));
         }
 
         view.content = child;
 
         child = Mustache.render(html, view, desi.partials);
-
       });
 
       // TODO add html meta-refresh redirects
-      compiled.push({ contents: child, path: path.join(desi.config.compiled_path, article.yml.permalink) });
+      compiled.push({ contents: child, path: path.join(desi.config.compiled_path, entity.yml.permalink, 'index.html') });
-      if (Array.isArray(article.yml.redirects)) {
+      entity.yml.redirects = entity.yml.redirects || [];
+      if (entity.yml.permalink) {
+        entity.yml.redirects.push(entity.yml.permalink + '.html');
+      }
+      entity.yml.redirects.forEach(function (redirect) {
         child = 
           '<html>'
             + '<head>'
-              + '<title>Redirecting to ' + article.yml.title + '</title>'
+              + '<title>Redirecting to ' + entity.yml.title + '</title>'
-              + '<meta http-equiv="refresh" content="0;URL=\'' + path.join(desi.urls.url, article.yml.permalink) + '\'" />'
+              + '<meta http-equiv="refresh" content="0;URL=\''
+                + desi.urls.url + path.join(entity.yml.permalink)
+              + '\'" />'
             + '</head>'
             + '<body>'
-              + '<p>This page has moved to a <a href="' + path.join(desi.urls.url, article.yml.permalink) +'">' + article.yml.title + '</a>.</p>'
+              + '<p>This page has moved to a <a href="'
+                + desi.urls.url + path.join(entity.yml.permalink)
+              +'">'
+                + entity.yml.title
+              + '</a>.</p>'
             + '</body>'
         + '</html>'
         ;
 
-        compiled.push({ contents: child, url: view.url, path: path.join(desi.config.compiled, article.yml.permalink) });
+        compiled.push({ contents: child, path: path.join(desi.config.compiled_path, redirect) });
-      }
+      });
+    }
 
-
+    console.info('compiling root pages');
-    });
+    desi.content.root.forEach(compileContentEntity);
+    console.info('compiling article pages');
+    desi.content.collections.forEach(compileContentEntity);
+    console.info('compiling theme assets');
+    desi.content.themes.filter(function (f) { return !/\blayouts\b/.test(f.path); }).forEach(compileThemeEntity);
 
     desi.compiled = compiled;
     return desi;
   }).then(function (desi) {
-    var compiled = desi.compiled
+    var compiled = desi.compiled.slice(0)
       , batches = []
       , now
+      , size = 0
       ;
 
     if (!compiled.length) {
@@ -420,6 +665,8 @@
     console.info('compiled files');
     return forEachAsync(batches, function (files) {
       return fsapi.putFiles(files).then(function (saved) {
+        size += saved.size;
+
         if (saved.error) {
           console.error(saved.error);
         }
@@ -436,7 +683,11 @@
       });
     }).then(function () {
       // TODO update cache
-      console.info('done', ((Date.now() - now) / 1000).toFixed(3));
+      console.info('wrote ' + desi.compiled.length 
+        + ' files (' + (size / (1024 * 1024)).toFixed(2)
+        + ' MiB) in '
+        + ((Date.now() - now) / 1000).toFixed(3) + 's'
+      );
     });
   }).catch(function (e) {
     console.error('A great and uncatchable error has befallen the land. Read ye here for das detalles..');

lib/deardesi-browser.js

@@ -248,11 +248,14 @@
 
   fsapi.putFiles = function (files) {
     var body = { files: files };
-    body = JSON.stringify(body); // this isn't the slow part
+    body = JSON.stringify(body); // this is more or less instant for a few MiB of posts
-    console.log(files[0]);
-    console.info('total size:', body.length / (1024 * 1024)); // see filesize
     return request.post('/api/fs/files', body).then(function (resp) {
-      return JSON.parse(resp);
+      var response = JSON.parse(resp)
+        ;
+
+      // not accurate for utf8/unicode, but close enough
+      response.size = body.length;
+      return response;
     });
   };
 }('undefined' !== typeof exports && exports || window));

partials.yml

@@ -9,24 +9,50 @@ pages_list: |
 #     'pages': [{}, {}, ..] }, ..] }, ..]
 posts_collate: |
   {{# years }}
+    <h2>{{ year }}</h2>
     {{> posts_collate_year }}
   {{/ years }}
 
 posts_collate_year: |
   {{# months }}
-  <h2>{{ year }}</h2>
+    <h3>{{ month }}</h3>
     {{> posts_collate_month }}
   {{/ months }}
 
 posts_collate_month: |
-  {{# pages }}
-  <h3>{{ month }}</h3>
   <ul>
+  {{# pages }}
     {{> posts_collate_pages }}
-  </ul>
   {{/ pages }}
+  </ul>
 
 posts_collate_pages: |
-  <li
+  <li><span>{{ year }}-{{ month }}-{{ day }}</span> » 
-    ><span>date</span> » <a href="{{url}}">{{ title }}</a
+    <a href="{{{url}}}"
-  ></li>
+      >{{ title }}</a></li>
+
+google_prettify: |
+  <script src="http://cdnjs.cloudflare.com/ajax/libs/prettify/188.0.0/prettify.js"></script>
+  <script>
+    var pres = document.getElementsByTagName("pre");
+    for (var i=0; i < pres.length; ++i) {
+      pres[i].className = "prettyprint {{#linenums}}linenums{{/linenums}}";
+    }
+    prettyPrint();
+  </script>
+  <!-- end Google Prettify -->
+
+google_ads: |
+  <script type="text/javascript">
+
+    var _gaq = _gaq || [];
+    _gaq.push(['_setAccount', {{#id}}'{{.}}'{{/id}}]);
+    _gaq.push(['_trackPageview']);
+
+    (function() {
+      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+    })();
+
+  </script>