Loading the database compacts it right away

lib/datastore.js

@@ -1,9 +1,3 @@
-/**
- * The datastore itself
- * TODO
- * Update and removes should only modify the corresponding part of the database (or use much faster append-only format)
- */
-
 var fs = require('fs')
   , path = require('path')
   , customUtils = require('./customUtils')
@@ -24,11 +18,12 @@ function Datastore (filename) {
 
 /**
  * Load the database
- * For now this means pulling data out of the data file or creating it
+ * This means pulling data out of the data file or creating it if it doesn't exist
- * if it doesn't exist
+ * Also, all data is persisted right away, which has the effect of compacting the database file
+ * This operation is very quick at startup for a big collection (60ms for ~10k docs)
  * @param {Function} cb Optional callback, signature: err
  *
- * @api private
+ * @api private Use loadDatabase
  */
 Datastore.prototype._loadDatabase = function (cb) {
   var callback = cb || function () {}
@@ -44,7 +39,7 @@ Datastore.prototype._loadDatabase = function (cb) {
         fs.readFile(self.filename, 'utf8', function (err, rawData) {
           if (err) { return callback(err); }
           self.data = Datastore.treatRawData(rawData);
-          return callback();
+          self.persistCachedDatabase(callback);
         });
       }
     });
@@ -90,6 +85,28 @@ Datastore.treatRawData = function (rawData) {
 };
 
 
+/**
+ * Persist cached database
+ * This serves as a compaction function since the cache always contains only the number of documents in the collection
+ * while the data file is append-only so it may grow larger
+ * @param {Function} cb Optional callback, signature: err
+ */
+Datastore.prototype.persistCachedDatabase = function (cb) {
+  var callback = cb || function () {}
+    , toPersist = ''
+    ;
+
+  this.data.forEach(function (doc) {
+    toPersist += model.serialize(doc) + '\n';
+  });
+
+  if (toPersist.length === 0) { return callback(); }
+
+  fs.writeFile(this.filename, toPersist, function (err) { return callback(err); });
+
+};
+
+
 /**
  * Insert a new document
  * @param {Function} cb Optional callback, signature: err, insertedDoc
@@ -173,6 +190,7 @@ Datastore.prototype.findOne = function (query, callback) {
 
 /**
  * Persist new state for the given newDocs (can be update or removal)
+ * Use an append-only format
  * @param {Array} newDocs Can be empty if no doc was updated/removed
  * @param {Function} cb Optional, signature: err
  */

test/db.test.js

@@ -36,7 +36,7 @@ describe('Database', function () {
   });
 
 
-  describe('Loading the database data from file', function () {
+  describe('Loading the database data from file and persistence', function () {
 
     it('Every line represents a document', function () {
       var now = new Date()
@@ -124,7 +124,35 @@ describe('Database', function () {
       _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true);
     });
 
-  });   // ==== End of 'Loading the database data from file' ==== //
+    it('Compact database on load', function (done) {
+      d.insert({ a: 2 }, function () {
+        d.insert({ a: 4 }, function () {
+          d.remove({ a: 2 }, {}, function () {
+            // Here, the underlying file is 3 lines long for only one document
+            var data = fs.readFileSync(d.filename, 'utf8').split('\n')
+              , filledCount = 0;
+
+            data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } });
+            filledCount.should.equal(3);
+
+            d.loadDatabase(function (err) {
+              assert.isNull(err);
+
+              // Now, the file has been compacted and is only 1 line long
+              var data = fs.readFileSync(d.filename, 'utf8').split('\n')
+                , filledCount = 0;
+
+              data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } });
+              filledCount.should.equal(1);
+
+              done();
+            });
+          })
+        });
+      });
+    });
+
+  });   // ==== End of 'Loading the database data from file and persistence' ==== //
 
 
   describe('Insert', function () {
@@ -737,7 +765,7 @@ describe('Database', function () {
 
               // Even after a reload the database state hasn't changed
               d.loadDatabase(function (err) {
-                assert.isUndefined(err);
+                assert.isNull(err);
 
                 d.find({}, function (err, docs) {
                   docs.sort(function (a, b) { return a.a - b.a; });
@@ -770,7 +798,7 @@ describe('Database', function () {
 
                 // Even after a reload the database state hasn't changed
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
                   d.find({}, function (err, docs) {
                     docs.sort(function (a, b) { return a.a - b.a; });
@@ -891,7 +919,7 @@ describe('Database', function () {
 
                 // Even after a reload the database state hasn't changed
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
                   d.find({}, function (err, docs) {
                     docs.sort(function (a, b) { return a.a - b.a; });
@@ -922,7 +950,7 @@ describe('Database', function () {
 
                 // Even after a reload the database state hasn't changed
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
                   d.find({}, function (err, docs) {
                     docs.length.should.equal(1);