Only the persistence module requires fs now, better separation of tasks

pull/2/head
Louis Chatriot 12 years ago
parent 4ae6b55816
commit dcb1e16b92
  1. 3
      benchmarks/commonUtilities.js
  2. 18
      lib/customUtils.js
  3. 92
      lib/datastore.js
  4. 101
      lib/persistence.js
  5. 3
      test/db.test.js

@ -6,6 +6,7 @@ var customUtils = require('../lib/customUtils')
, fs = require('fs') , fs = require('fs')
, path = require('path') , path = require('path')
, Datastore = require('../lib/datastore') , Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, executeAsap // process.nextTick or setImmediate depending on your Node version , executeAsap // process.nextTick or setImmediate depending on your Node version
; ;
@ -50,7 +51,7 @@ module.exports.getConfiguration = function (benchDb) {
* Ensure the workspace exists and the db datafile is empty * Ensure the workspace exists and the db datafile is empty
*/ */
module.exports.prepareDb = function (filename, cb) { module.exports.prepareDb = function (filename, cb) {
customUtils.ensureDirectoryExists(path.dirname(filename), function () { Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) { fs.exists(filename, function (exists) {
if (exists) { if (exists) {
fs.unlink(filename, cb); fs.unlink(filename, cb);

@ -1,22 +1,7 @@
var fs = require('fs') var crypto = require('crypto')
, crypto = require('crypto')
, path = require('path') , path = require('path')
, mkdirp = require('mkdirp')
; ;
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
function ensureDirectoryExists (dir, cb) {
var callback = cb || function () {}
;
mkdirp(dir, function (err) { return callback(err); });
}
/** /**
* Return a random alphanumerical string of length len * Return a random alphanumerical string of length len
* There is a very small probability (less than 1/1,000,000) for the length to be less than len * There is a very small probability (less than 1/1,000,000) for the length to be less than len
@ -66,6 +51,5 @@ function getNWAppFilename (appName, relativeFilename) {
} }
module.exports.ensureDirectoryExists = ensureDirectoryExists;
module.exports.uid = uid; module.exports.uid = uid;
module.exports.getNWAppFilename = getNWAppFilename; module.exports.getNWAppFilename = getNWAppFilename;

@ -1,6 +1,4 @@
var fs = require('fs') var customUtils = require('./customUtils')
, path = require('path')
, customUtils = require('./customUtils')
, model = require('./model') , model = require('./model')
, async = require('async') , async = require('async')
, Executor = require('./executor') , Executor = require('./executor')
@ -243,97 +241,13 @@ Datastore.prototype.getCandidates = function (query) {
}; };
/**
* Load the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*
* @api private Use loadDatabase
*/
Datastore.prototype._loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.resetIndexes();
self.datafileSize = 0;
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
customUtils.ensureDirectoryExists(path.dirname(self.filename), function (err) {
fs.exists(self.filename, function (exists) {
if (!exists) { return fs.writeFile(self.filename, '', 'utf8', function (err) { cb(err); }); }
fs.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
var treatedData = Datastore.treatRawData(rawData);
try {
self.resetIndexes(treatedData);
} catch (e) {
self.resetIndexes(); // Rollback any index which didn't fail
self.datafileSize = 0;
return cb(e);
}
self.datafileSize = treatedData.length;
self.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.executor.processBuffer();
return callback(null);
});
};
Datastore.prototype.loadDatabase = function () { Datastore.prototype.loadDatabase = function () {
this.executor.push({ this: this, fn: this._loadDatabase, arguments: arguments }, true); this.executor.push({ this: this.persistence, fn: this.persistence._loadDatabase, arguments: arguments }, true);
};
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Datastore.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, res = []
, i;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(data[i]);
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
}
} catch (e) {
}
}
Object.keys(dataById).forEach(function (k) {
res.push(dataById[k]);
});
return res;
}; };
Datastore.treatRawData = Persistence.treatRawData;

@ -6,6 +6,8 @@ var fs = require('fs')
, path = require('path') , path = require('path')
, model = require('./model') , model = require('./model')
, customUtils = require('./customUtils') , customUtils = require('./customUtils')
, async = require('async')
, mkdirp = require('mkdirp')
; ;
@ -18,6 +20,18 @@ function Persistence (options) {
} }
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
Persistence.ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
mkdirp(dir, function (err) { return callback(err); });
}
/** /**
* Persist cached database * Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection * This serves as a compaction function since the cache always contains only the number of documents in the collection
@ -68,5 +82,92 @@ Persistence.prototype.persistNewState = function (newDocs, cb) {
}; };
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Persistence.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, res = []
, i;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(data[i]);
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
}
} catch (e) {
}
}
Object.keys(dataById).forEach(function (k) {
res.push(dataById[k]);
});
return res;
};
/**
* Load the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*
* @api private Use loadDatabase
*/
Persistence.prototype._loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.db.resetIndexes();
self.db.datafileSize = 0;
// In-memory only datastore
if (self.db.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.db.filename), function (err) {
fs.exists(self.db.filename, function (exists) {
if (!exists) { return fs.writeFile(self.db.filename, '', 'utf8', function (err) { cb(err); }); }
fs.readFile(self.db.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
var treatedData = Persistence.treatRawData(rawData);
try {
self.db.resetIndexes(treatedData);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
self.db.datafileSize = 0;
return cb(e);
}
self.db.datafileSize = treatedData.length;
self.db.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
};
// Interface // Interface
module.exports = Persistence; module.exports = Persistence;

@ -8,6 +8,7 @@ var should = require('chai').should()
, Datastore = require('../lib/datastore') , Datastore = require('../lib/datastore')
, customUtils = require('../lib/customUtils') , customUtils = require('../lib/customUtils')
, model = require('../lib/model') , model = require('../lib/model')
, Persistence = require('../lib/persistence')
; ;
@ -21,7 +22,7 @@ describe('Database', function () {
async.waterfall([ async.waterfall([
function (cb) { function (cb) {
customUtils.ensureDirectoryExists(path.dirname(testDb), function () { Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
fs.exists(testDb, function (exists) { fs.exists(testDb, function (exists) {
if (exists) { if (exists) {
fs.unlink(testDb, cb); fs.unlink(testDb, cb);

Loading…
Cancel
Save