From f694e7ce2aa765b4805eaef7ff1481223cb817f5 Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Sun, 26 May 2013 11:40:57 +0200 Subject: [PATCH] Loading the database compacts it right away --- lib/datastore.js | 38 ++++++++++++++++++++++++++++---------- test/db.test.js | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/lib/datastore.js b/lib/datastore.js index 77b7923..aa4ea2b 100644 --- a/lib/datastore.js +++ b/lib/datastore.js @@ -1,9 +1,3 @@ -/** - * The datastore itself - * TODO - * Update and removes should only modify the corresponding part of the database (or use much faster append-only format) - */ - var fs = require('fs') , path = require('path') , customUtils = require('./customUtils') @@ -24,11 +18,12 @@ function Datastore (filename) { /** * Load the database - * For now this means pulling data out of the data file or creating it - * if it doesn't exist + * This means pulling data out of the data file or creating it if it doesn't exist + * Also, all data is persisted right away, which has the effect of compacting the database file + * This operation is very quick at startup for a big collection (60ms for ~10k docs) * @param {Function} cb Optional callback, signature: err * - * @api private + * @api private Use loadDatabase */ Datastore.prototype._loadDatabase = function (cb) { var callback = cb || function () {} @@ -44,7 +39,7 @@ Datastore.prototype._loadDatabase = function (cb) { fs.readFile(self.filename, 'utf8', function (err, rawData) { if (err) { return callback(err); } self.data = Datastore.treatRawData(rawData); - return callback(); + self.persistCachedDatabase(callback); }); } }); @@ -90,6 +85,28 @@ Datastore.treatRawData = function (rawData) { }; +/** + * Persist cached database + * This serves as a compaction function since the cache always contains only the number of documents in the collection + * while the data file is append-only so it may grow larger + * @param {Function} cb Optional callback, signature: err + */ +Datastore.prototype.persistCachedDatabase = function (cb) { + var callback = cb || function () {} + , toPersist = '' + ; + + this.data.forEach(function (doc) { + toPersist += model.serialize(doc) + '\n'; + }); + + if (toPersist.length === 0) { return callback(); } + + fs.writeFile(this.filename, toPersist, function (err) { return callback(err); }); + +}; + + /** * Insert a new document * @param {Function} cb Optional callback, signature: err, insertedDoc @@ -173,6 +190,7 @@ Datastore.prototype.findOne = function (query, callback) { /** * Persist new state for the given newDocs (can be update or removal) + * Use an append-only format * @param {Array} newDocs Can be empty if no doc was updated/removed * @param {Function} cb Optional, signature: err */ diff --git a/test/db.test.js b/test/db.test.js index b1d3738..4a33ca1 100644 --- a/test/db.test.js +++ b/test/db.test.js @@ -36,7 +36,7 @@ describe('Database', function () { }); - describe('Loading the database data from file', function () { + describe('Loading the database data from file and persistence', function () { it('Every line represents a document', function () { var now = new Date() @@ -124,7 +124,35 @@ describe('Database', function () { _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true); }); - }); // ==== End of 'Loading the database data from file' ==== // + it('Compact database on load', function (done) { + d.insert({ a: 2 }, function () { + d.insert({ a: 4 }, function () { + d.remove({ a: 2 }, {}, function () { + // Here, the underlying file is 3 lines long for only one document + var data = fs.readFileSync(d.filename, 'utf8').split('\n') + , filledCount = 0; + + data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } }); + filledCount.should.equal(3); + + d.loadDatabase(function (err) { + assert.isNull(err); + + // Now, the file has been compacted and is only 1 line long + var data = fs.readFileSync(d.filename, 'utf8').split('\n') + , filledCount = 0; + + data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } }); + filledCount.should.equal(1); + + done(); + }); + }) + }); + }); + }); + + }); // ==== End of 'Loading the database data from file and persistence' ==== // describe('Insert', function () { @@ -737,7 +765,7 @@ describe('Database', function () { // Even after a reload the database state hasn't changed d.loadDatabase(function (err) { - assert.isUndefined(err); + assert.isNull(err); d.find({}, function (err, docs) { docs.sort(function (a, b) { return a.a - b.a; }); @@ -770,7 +798,7 @@ describe('Database', function () { // Even after a reload the database state hasn't changed d.loadDatabase(function (err) { - assert.isUndefined(err); + assert.isNull(err); d.find({}, function (err, docs) { docs.sort(function (a, b) { return a.a - b.a; }); @@ -891,7 +919,7 @@ describe('Database', function () { // Even after a reload the database state hasn't changed d.loadDatabase(function (err) { - assert.isUndefined(err); + assert.isNull(err); d.find({}, function (err, docs) { docs.sort(function (a, b) { return a.a - b.a; }); @@ -922,7 +950,7 @@ describe('Database', function () { // Even after a reload the database state hasn't changed d.loadDatabase(function (err) { - assert.isUndefined(err); + assert.isNull(err); d.find({}, function (err, docs) { docs.length.should.equal(1);