TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Only the persistence module requires fs now, better separation of tasks

Browse Source
pull/2/head
Louis Chatriot 12 years ago
parent 4ae6b55816
commit dcb1e16b92
5 changed files with 109 additions and 108 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      benchmarks/commonUtilities.js
  2. 18
      lib/customUtils.js
  3. 92
      lib/datastore.js
  4. 101
      lib/persistence.js
  5. 3
      test/db.test.js

3
benchmarks/commonUtilities.js
Unescape View File

@ -6,6 +6,7 @@ var customUtils = require('../lib/customUtils')
, fs = require('fs')
, path = require('path')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, executeAsap // process.nextTick or setImmediate depending on your Node version
;
@ -50,7 +51,7 @@ module.exports.getConfiguration = function (benchDb) {
* Ensure the workspace exists and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
customUtils.ensureDirectoryExists(path.dirname(filename), function () {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);

18
lib/customUtils.js
Unescape View File

@ -1,22 +1,7 @@
var fs = require('fs')
, crypto = require('crypto')
var crypto = require('crypto')
, path = require('path')
, mkdirp = require('mkdirp')
;
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
function ensureDirectoryExists (dir, cb) {
var callback = cb || function () {}
;
mkdirp(dir, function (err) { return callback(err); });
}
/**
* Return a random alphanumerical string of length len
* There is a very small probability (less than 1/1,000,000) for the length to be less than len
@ -66,6 +51,5 @@ function getNWAppFilename (appName, relativeFilename) {
}
module.exports.ensureDirectoryExists = ensureDirectoryExists;
module.exports.uid = uid;
module.exports.getNWAppFilename = getNWAppFilename;

92
lib/datastore.js
Unescape View File

@ -1,6 +1,4 @@
var fs = require('fs')
, path = require('path')
, customUtils = require('./customUtils')
var customUtils = require('./customUtils')
, model = require('./model')
, async = require('async')
, Executor = require('./executor')
@ -243,97 +241,13 @@ Datastore.prototype.getCandidates = function (query) {
};
/**
* Load the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*
* @api private Use loadDatabase
*/
Datastore.prototype._loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.resetIndexes();
self.datafileSize = 0;
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
customUtils.ensureDirectoryExists(path.dirname(self.filename), function (err) {
fs.exists(self.filename, function (exists) {
if (!exists) { return fs.writeFile(self.filename, '', 'utf8', function (err) { cb(err); }); }
fs.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
var treatedData = Datastore.treatRawData(rawData);
try {
self.resetIndexes(treatedData);
} catch (e) {
self.resetIndexes(); // Rollback any index which didn't fail
self.datafileSize = 0;
return cb(e);
}
self.datafileSize = treatedData.length;
self.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.executor.processBuffer();
return callback(null);
});
};
Datastore.prototype.loadDatabase = function () {
this.executor.push({ this: this, fn: this._loadDatabase, arguments: arguments }, true);
};
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Datastore.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, res = []
, i;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(data[i]);
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
}
} catch (e) {
}
}
Object.keys(dataById).forEach(function (k) {
res.push(dataById[k]);
});
return res;
this.executor.push({ this: this.persistence, fn: this.persistence._loadDatabase, arguments: arguments }, true);
};
Datastore.treatRawData = Persistence.treatRawData;

101
lib/persistence.js
Unescape View File

@ -6,6 +6,8 @@ var fs = require('fs')
, path = require('path')
, model = require('./model')
, customUtils = require('./customUtils')
, async = require('async')
, mkdirp = require('mkdirp')
;
@ -18,6 +20,18 @@ function Persistence (options) {
}
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
Persistence.ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
mkdirp(dir, function (err) { return callback(err); });
}
/**
* Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection
@ -68,5 +82,92 @@ Persistence.prototype.persistNewState = function (newDocs, cb) {
};
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Persistence.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, res = []
, i;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(data[i]);
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
}
} catch (e) {
}
}
Object.keys(dataById).forEach(function (k) {
res.push(dataById[k]);
});
return res;
};
/**
* Load the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*
* @api private Use loadDatabase
*/
Persistence.prototype._loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.db.resetIndexes();
self.db.datafileSize = 0;
// In-memory only datastore
if (self.db.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.db.filename), function (err) {
fs.exists(self.db.filename, function (exists) {
if (!exists) { return fs.writeFile(self.db.filename, '', 'utf8', function (err) { cb(err); }); }
fs.readFile(self.db.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
var treatedData = Persistence.treatRawData(rawData);
try {
self.db.resetIndexes(treatedData);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
self.db.datafileSize = 0;
return cb(e);
}
self.db.datafileSize = treatedData.length;
self.db.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
};
// Interface
module.exports = Persistence;

3
test/db.test.js
Unescape View File

@ -8,6 +8,7 @@ var should = require('chai').should()
, Datastore = require('../lib/datastore')
, customUtils = require('../lib/customUtils')
, model = require('../lib/model')
, Persistence = require('../lib/persistence')
;
@ -21,7 +22,7 @@ describe('Database', function () {
async.waterfall([
function (cb) {
customUtils.ensureDirectoryExists(path.dirname(testDb), function () {
Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
fs.exists(testDb, function (exists) {
if (exists) {
fs.unlink(testDb, cb);

Write Preview
Loading…
Cancel
Save