From dcb1e16b92c810d04478dcafc4e1a90f924c8fe4 Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Fri, 12 Jul 2013 15:07:11 +0200 Subject: [PATCH] Only the persistence module requires fs now, better separation of tasks --- benchmarks/commonUtilities.js | 3 +- lib/customUtils.js | 18 +----- lib/datastore.js | 92 +------------------------------ lib/persistence.js | 101 ++++++++++++++++++++++++++++++++++ test/db.test.js | 3 +- 5 files changed, 109 insertions(+), 108 deletions(-) diff --git a/benchmarks/commonUtilities.js b/benchmarks/commonUtilities.js index 788486b..9612006 100644 --- a/benchmarks/commonUtilities.js +++ b/benchmarks/commonUtilities.js @@ -6,6 +6,7 @@ var customUtils = require('../lib/customUtils') , fs = require('fs') , path = require('path') , Datastore = require('../lib/datastore') + , Persistence = require('../lib/persistence') , executeAsap // process.nextTick or setImmediate depending on your Node version ; @@ -50,7 +51,7 @@ module.exports.getConfiguration = function (benchDb) { * Ensure the workspace exists and the db datafile is empty */ module.exports.prepareDb = function (filename, cb) { - customUtils.ensureDirectoryExists(path.dirname(filename), function () { + Persistence.ensureDirectoryExists(path.dirname(filename), function () { fs.exists(filename, function (exists) { if (exists) { fs.unlink(filename, cb); diff --git a/lib/customUtils.js b/lib/customUtils.js index 4ecfd78..069f36f 100644 --- a/lib/customUtils.js +++ b/lib/customUtils.js @@ -1,22 +1,7 @@ -var fs = require('fs') - , crypto = require('crypto') +var crypto = require('crypto') , path = require('path') - , mkdirp = require('mkdirp') ; - -/** - * Check if a directory exists and create it on the fly if it is not the case - * cb is optional, signature: err - */ -function ensureDirectoryExists (dir, cb) { - var callback = cb || function () {} - ; - - mkdirp(dir, function (err) { return callback(err); }); -} - - /** * Return a random alphanumerical string of length len * There is a very small probability (less than 1/1,000,000) for the length to be less than len @@ -66,6 +51,5 @@ function getNWAppFilename (appName, relativeFilename) { } -module.exports.ensureDirectoryExists = ensureDirectoryExists; module.exports.uid = uid; module.exports.getNWAppFilename = getNWAppFilename; diff --git a/lib/datastore.js b/lib/datastore.js index c2b1500..451eaa8 100644 --- a/lib/datastore.js +++ b/lib/datastore.js @@ -1,6 +1,4 @@ -var fs = require('fs') - , path = require('path') - , customUtils = require('./customUtils') +var customUtils = require('./customUtils') , model = require('./model') , async = require('async') , Executor = require('./executor') @@ -243,97 +241,13 @@ Datastore.prototype.getCandidates = function (query) { }; -/** - * Load the database - * This means pulling data out of the data file or creating it if it doesn't exist - * Also, all data is persisted right away, which has the effect of compacting the database file - * This operation is very quick at startup for a big collection (60ms for ~10k docs) - * @param {Function} cb Optional callback, signature: err - * - * @api private Use loadDatabase - */ -Datastore.prototype._loadDatabase = function (cb) { - var callback = cb || function () {} - , self = this - ; - - self.resetIndexes(); - self.datafileSize = 0; - - // In-memory only datastore - if (self.inMemoryOnly) { return callback(null); } - - async.waterfall([ - function (cb) { - customUtils.ensureDirectoryExists(path.dirname(self.filename), function (err) { - fs.exists(self.filename, function (exists) { - if (!exists) { return fs.writeFile(self.filename, '', 'utf8', function (err) { cb(err); }); } - - fs.readFile(self.filename, 'utf8', function (err, rawData) { - if (err) { return cb(err); } - var treatedData = Datastore.treatRawData(rawData); - - try { - self.resetIndexes(treatedData); - } catch (e) { - self.resetIndexes(); // Rollback any index which didn't fail - self.datafileSize = 0; - return cb(e); - } - - self.datafileSize = treatedData.length; - self.persistence.persistCachedDatabase(cb); - }); - }); - }); - } - ], function (err) { - if (err) { return callback(err); } - - self.executor.processBuffer(); - return callback(null); - }); -}; Datastore.prototype.loadDatabase = function () { - this.executor.push({ this: this, fn: this._loadDatabase, arguments: arguments }, true); -}; - - -/** - * From a database's raw data, return the corresponding - * machine understandable collection - */ -Datastore.treatRawData = function (rawData) { - var data = rawData.split('\n') - , dataById = {} - , res = [] - , i; - - for (i = 0; i < data.length; i += 1) { - var doc; - - try { - doc = model.deserialize(data[i]); - if (doc._id) { - if (doc.$$deleted === true) { - delete dataById[doc._id]; - } else { - dataById[doc._id] = doc; - } - } - } catch (e) { - } - } - - Object.keys(dataById).forEach(function (k) { - res.push(dataById[k]); - }); - - return res; + this.executor.push({ this: this.persistence, fn: this.persistence._loadDatabase, arguments: arguments }, true); }; +Datastore.treatRawData = Persistence.treatRawData; diff --git a/lib/persistence.js b/lib/persistence.js index a720fc1..a50e989 100644 --- a/lib/persistence.js +++ b/lib/persistence.js @@ -6,6 +6,8 @@ var fs = require('fs') , path = require('path') , model = require('./model') , customUtils = require('./customUtils') + , async = require('async') + , mkdirp = require('mkdirp') ; @@ -18,6 +20,18 @@ function Persistence (options) { } +/** + * Check if a directory exists and create it on the fly if it is not the case + * cb is optional, signature: err + */ +Persistence.ensureDirectoryExists = function (dir, cb) { + var callback = cb || function () {} + ; + + mkdirp(dir, function (err) { return callback(err); }); +} + + /** * Persist cached database * This serves as a compaction function since the cache always contains only the number of documents in the collection @@ -68,5 +82,92 @@ Persistence.prototype.persistNewState = function (newDocs, cb) { }; +/** + * From a database's raw data, return the corresponding + * machine understandable collection + */ +Persistence.treatRawData = function (rawData) { + var data = rawData.split('\n') + , dataById = {} + , res = [] + , i; + + for (i = 0; i < data.length; i += 1) { + var doc; + + try { + doc = model.deserialize(data[i]); + if (doc._id) { + if (doc.$$deleted === true) { + delete dataById[doc._id]; + } else { + dataById[doc._id] = doc; + } + } + } catch (e) { + } + } + + Object.keys(dataById).forEach(function (k) { + res.push(dataById[k]); + }); + + return res; +}; + + +/** + * Load the database + * This means pulling data out of the data file or creating it if it doesn't exist + * Also, all data is persisted right away, which has the effect of compacting the database file + * This operation is very quick at startup for a big collection (60ms for ~10k docs) + * @param {Function} cb Optional callback, signature: err + * + * @api private Use loadDatabase + */ +Persistence.prototype._loadDatabase = function (cb) { + var callback = cb || function () {} + , self = this + ; + + self.db.resetIndexes(); + self.db.datafileSize = 0; + + // In-memory only datastore + if (self.db.inMemoryOnly) { return callback(null); } + + async.waterfall([ + function (cb) { + Persistence.ensureDirectoryExists(path.dirname(self.db.filename), function (err) { + fs.exists(self.db.filename, function (exists) { + if (!exists) { return fs.writeFile(self.db.filename, '', 'utf8', function (err) { cb(err); }); } + + fs.readFile(self.db.filename, 'utf8', function (err, rawData) { + if (err) { return cb(err); } + var treatedData = Persistence.treatRawData(rawData); + + try { + self.db.resetIndexes(treatedData); + } catch (e) { + self.db.resetIndexes(); // Rollback any index which didn't fail + self.db.datafileSize = 0; + return cb(e); + } + + self.db.datafileSize = treatedData.length; + self.db.persistence.persistCachedDatabase(cb); + }); + }); + }); + } + ], function (err) { + if (err) { return callback(err); } + + self.db.executor.processBuffer(); + return callback(null); + }); +}; + + // Interface module.exports = Persistence; diff --git a/test/db.test.js b/test/db.test.js index 6f65c87..1bf0d86 100644 --- a/test/db.test.js +++ b/test/db.test.js @@ -8,6 +8,7 @@ var should = require('chai').should() , Datastore = require('../lib/datastore') , customUtils = require('../lib/customUtils') , model = require('../lib/model') + , Persistence = require('../lib/persistence') ; @@ -21,7 +22,7 @@ describe('Database', function () { async.waterfall([ function (cb) { - customUtils.ensureDirectoryExists(path.dirname(testDb), function () { + Persistence.ensureDirectoryExists(path.dirname(testDb), function () { fs.exists(testDb, function (exists) { if (exists) { fs.unlink(testDb, cb);