From d1e6d14b530e9c25e13fe396d11a240641260c1c Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Mon, 9 Nov 2015 11:22:24 +0100 Subject: [PATCH] Moved all crash safe operations to storage --- .../browser-specific/lib/storage.js | 6 +- lib/persistence.js | 83 +-- lib/storage.js | 90 +++- package.json | 1 + test/persistence.test.js | 475 +++++++++--------- 5 files changed, 343 insertions(+), 312 deletions(-) diff --git a/browser-version/browser-specific/lib/storage.js b/browser-version/browser-specific/lib/storage.js index 54c65cd..15248bb 100644 --- a/browser-version/browser-specific/lib/storage.js +++ b/browser-version/browser-specific/lib/storage.js @@ -36,7 +36,7 @@ function rename (filename, newFilename, callback) { function writeFile (filename, contents, options, callback) { if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); } - + // Options do not matter in browser setup if (typeof options === 'function') { callback = options; } @@ -47,7 +47,7 @@ function writeFile (filename, contents, options, callback) { function appendFile (filename, toAppend, options, callback) { if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); } - + // Options do not matter in browser setup if (typeof options === 'function') { callback = options; } @@ -61,7 +61,7 @@ function appendFile (filename, toAppend, options, callback) { function readFile (filename, options, callback) { if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); } - + // Options do not matter in browser setup if (typeof options === 'function') { callback = options; } diff --git a/lib/persistence.js b/lib/persistence.js index e0e0186..0b6f8ad 100644 --- a/lib/persistence.js +++ b/lib/persistence.js @@ -22,19 +22,14 @@ var storage = require('./storage') */ function Persistence (options) { var i, j, randomString; - + this.db = options.db; this.inMemoryOnly = this.db.inMemoryOnly; this.filename = this.db.filename; this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1; - - if (!this.inMemoryOnly && this.filename) { - if (this.filename.charAt(this.filename.length - 1) === '~') { - throw "The datafile name can't end with a ~, which is reserved for automatic backup files"; - } else { - this.tempFilename = this.filename + '~'; - this.oldFilename = this.filename + '~~'; - } + + if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') { + throw "The datafile name can't end with a ~, which is reserved for crash safe backup files"; } // After serialization and before deserialization hooks with some basic sanity checks @@ -54,7 +49,7 @@ function Persistence (options) { } } } - + // For NW apps, store data in the same directory where NW stores application data if (this.filename && options.nodeWebkitAppName) { console.log("=================================================================="); @@ -65,8 +60,6 @@ function Persistence (options) { console.log("See https://github.com/rogerwang/node-webkit/issues/500"); console.log("=================================================================="); this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename); - this.tempFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.tempFilename); - this.oldFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.oldFilename); } }; @@ -83,13 +76,6 @@ Persistence.ensureDirectoryExists = function (dir, cb) { }; -Persistence.ensureFileDoesntExist = function (file, callback) { - storage.exists(file, function (exists) { - if (!exists) { return callback(null); } - - storage.unlink(file, function (err) { return callback(err); }); - }); -}; /** @@ -137,7 +123,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) { , self = this ; - if (this.inMemoryOnly) { return callback(null); } + if (this.inMemoryOnly) { return callback(null); } this.db.getAllData().forEach(function (doc) { toPersist += self.afterSerialization(model.serialize(doc)) + '\n'; @@ -148,26 +134,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) { } }); - async.waterfall([ - async.apply(Persistence.ensureFileDoesntExist, self.tempFilename) - , async.apply(Persistence.ensureFileDoesntExist, self.oldFilename) - , function (cb) { - storage.exists(self.filename, function (exists) { - if (exists) { - storage.rename(self.filename, self.oldFilename, function (err) { return cb(err); }); - } else { - return cb(); - } - }); - } - , function (cb) { - storage.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); }); - } - , function (cb) { - storage.rename(self.tempFilename, self.filename, function (err) { return cb(err); }); - } - , async.apply(Persistence.ensureFileDoesntExist, self.oldFilename) - ], function (err) { if (err) { return callback(err); } else { return callback(null); } }) + storage.crashSafeWriteFile(this.filename, toPersist, callback); }; @@ -244,10 +211,10 @@ Persistence.prototype.treatRawData = function (rawData) { , indexes = {} , corruptItems = -1 // Last line of every data file is usually blank so not really corrupt ; - + for (i = 0; i < data.length; i += 1) { var doc; - + try { doc = model.deserialize(this.beforeDeserialization(data[i])); if (doc._id) { @@ -265,7 +232,7 @@ Persistence.prototype.treatRawData = function (rawData) { corruptItems += 1; } } - + // A bit lenient on corruption if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) { throw "More than 10% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss" @@ -279,30 +246,6 @@ Persistence.prototype.treatRawData = function (rawData) { }; -/** - * Ensure that this.filename contains the most up-to-date version of the data - * Even if a loadDatabase crashed before - */ -Persistence.prototype.ensureDatafileIntegrity = function (callback) { - var self = this ; - - storage.exists(self.filename, function (filenameExists) { - // Write was successful - if (filenameExists) { return callback(null); } - - storage.exists(self.oldFilename, function (oldFilenameExists) { - // New database - if (!oldFilenameExists) { - return storage.writeFile(self.filename, '', 'utf8', function (err) { callback(err); }); - } - - // Write failed, use old version - storage.rename(self.oldFilename, self.filename, function (err) { return callback(err); }); - }); - }); -}; - - /** * Load the database * 1) Create all indexes @@ -326,16 +269,16 @@ Persistence.prototype.loadDatabase = function (cb) { async.waterfall([ function (cb) { Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) { - self.ensureDatafileIntegrity(function (exists) { + storage.ensureDatafileIntegrity(self.filename, function (exists) { storage.readFile(self.filename, 'utf8', function (err, rawData) { if (err) { return cb(err); } - + try { var treatedData = self.treatRawData(rawData); } catch (e) { return cb(e); } - + // Recreate all indexes in the datafile Object.keys(treatedData.indexes).forEach(function (key) { self.db.indexes[key] = new Index(treatedData.indexes[key]); diff --git a/lib/storage.js b/lib/storage.js index e46b3ff..0dd94d0 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -4,12 +4,98 @@ * For a browser-side database it's localStorage when supported * * This version is the Node.js/Node Webkit version + * It's essentially fs, mkdirp and crash safe write and read functions */ var fs = require('fs') , mkdirp = require('mkdirp') + , async = require('async') + , storage = {} ; +storage.exists = fs.exists; +storage.rename = fs.rename; +storage.writeFile = fs.writeFile; +storage.unlink = fs.unlink; +storage.appendFile = fs.appendFile; +storage.readFile = fs.readFile; +storage.mkdirp = mkdirp; -module.exports = fs; -module.exports.mkdirp = mkdirp; + +/** + * Explicit name ... + */ +storage.ensureFileDoesntExist = function (file, callback) { + storage.exists(file, function (exists) { + if (!exists) { return callback(null); } + + storage.unlink(file, function (err) { return callback(err); }); + }); +}; + + +/** + * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost) + * @param {String} filename + * @param {String} data + * @param {Function} cb Optional callback, signature: err + */ +storage.crashSafeWriteFile = function (filename, data, cb) { + var callback = cb || function () {} + , tempFilename = filename + '~' + , oldFilename = filename + '~~' + ; + + async.waterfall([ + async.apply(storage.ensureFileDoesntExist, tempFilename) + , async.apply(storage.ensureFileDoesntExist, oldFilename) + , function (cb) { + storage.exists(filename, function (exists) { + if (exists) { + storage.rename(filename, oldFilename, function (err) { return cb(err); }); + } else { + return cb(); + } + }); + } + , function (cb) { + storage.writeFile(tempFilename, data, function (err) { return cb(err); }); + } + , function (cb) { + storage.rename(tempFilename, filename, function (err) { return cb(err); }); + } + , async.apply(storage.ensureFileDoesntExist, oldFilename) + ], function (err) { if (err) { return callback(err); } else { return callback(null); } }) +}; + + +/** + * Ensure the datafile contains all the data, even if there was a crash during a full file write + * @param {String} filename + * @param {Function} callback signature: err + */ +storage.ensureDatafileIntegrity = function (filename, callback) { + var tempFilename = filename + '~' + , oldFilename = filename + '~~' + ; + + storage.exists(filename, function (filenameExists) { + // Write was successful + if (filenameExists) { return callback(null); } + + storage.exists(oldFilename, function (oldFilenameExists) { + // New database + if (!oldFilenameExists) { + return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); }); + } + + // Write failed, use old version + storage.rename(oldFilename, filename, function (err) { return callback(err); }); + }); + }); +}; + + + +// Interface +module.exports = storage; diff --git a/package.json b/package.json index 43d25d1..d39232b 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dependencies": { "async": "0.2.10", "binary-search-tree": "0.2.4", + "localforage": "^1.3.0", "mkdirp": "~0.5.1", "underscore": "~1.4.4" }, diff --git a/test/persistence.test.js b/test/persistence.test.js index 045e847..a170208 100644 --- a/test/persistence.test.js +++ b/test/persistence.test.js @@ -9,8 +9,9 @@ var should = require('chai').should() , customUtils = require('../lib/customUtils') , Datastore = require('../lib/datastore') , Persistence = require('../lib/persistence') + , storage = require('../lib/storage') , child_process = require('child_process') - ; +; describe('Persistence', function () { @@ -32,22 +33,22 @@ describe('Persistence', function () { }); } , function (cb) { - d.loadDatabase(function (err) { - assert.isNull(err); - d.getAllData().length.should.equal(0); - return cb(); - }); - } + d.loadDatabase(function (err) { + assert.isNull(err); + d.getAllData().length.should.equal(0); + return cb(); + }); + } ], done); }); it('Every line represents a document', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ _id: "2", hello: 'world' }) + '\n' + - model.serialize({ _id: "3", nested: { today: now } }) + model.serialize({ _id: "2", hello: 'world' }) + '\n' + + model.serialize({ _id: "3", nested: { today: now } }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(3); @@ -59,10 +60,10 @@ describe('Persistence', function () { it('Badly formatted lines have no impact on the treated data', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - 'garbage\n' + - model.serialize({ _id: "3", nested: { today: now } }) + 'garbage\n' + + model.serialize({ _id: "3", nested: { today: now } }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); @@ -73,10 +74,10 @@ describe('Persistence', function () { it('Well formatted lines that have no _id are not included in the data', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ _id: "2", hello: 'world' }) + '\n' + - model.serialize({ nested: { today: now } }) + model.serialize({ _id: "2", hello: 'world' }) + '\n' + + model.serialize({ nested: { today: now } }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); @@ -87,10 +88,10 @@ describe('Persistence', function () { it('If two lines concern the same doc (= same _id), the last one is the good version', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ _id: "2", hello: 'world' }) + '\n' + - model.serialize({ _id: "1", nested: { today: now } }) + model.serialize({ _id: "2", hello: 'world' }) + '\n' + + model.serialize({ _id: "1", nested: { today: now } }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); @@ -101,11 +102,11 @@ describe('Persistence', function () { it('If a doc contains $$deleted: true, that means we need to remove it from the data', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ _id: "2", hello: 'world' }) + '\n' + - model.serialize({ _id: "1", $$deleted: true }) + '\n' + - model.serialize({ _id: "3", today: now }) + model.serialize({ _id: "2", hello: 'world' }) + '\n' + + model.serialize({ _id: "1", $$deleted: true }) + '\n' + + model.serialize({ _id: "3", today: now }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); @@ -116,29 +117,29 @@ describe('Persistence', function () { it('If a doc contains $$deleted: true, no error is thrown if the doc wasnt in the list before', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ _id: "2", $$deleted: true }) + '\n' + - model.serialize({ _id: "3", today: now }) + model.serialize({ _id: "2", $$deleted: true }) + '\n' + + model.serialize({ _id: "3", today: now }) , treatedData = d.persistence.treatRawData(rawData).data - ; + ; treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true); _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true); }); - + it('If a doc contains $$indexCreated, no error is thrown during treatRawData and we can get the index options', function () { var now = new Date() , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' + - model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' + - model.serialize({ _id: "3", today: now }) + model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' + + model.serialize({ _id: "3", today: now }) , treatedData = d.persistence.treatRawData(rawData).data , indexes = d.persistence.treatRawData(rawData).indexes - ; + ; Object.keys(indexes).length.should.equal(1); assert.deepEqual(indexes.test, { fieldName: "test", unique: true }); - + treatedData.sort(function (a, b) { return a._id - b._id; }); treatedData.length.should.equal(2); _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true); @@ -181,7 +182,7 @@ describe('Persistence', function () { var data = d.getAllData() , doc1 = _.find(data, function (doc) { return doc.a === 1; }) , doc2 = _.find(data, function (doc) { return doc.a === 2; }) - ; + ; assert.isNull(err); data.length.should.equal(2); doc1.a.should.equal(1); @@ -191,7 +192,7 @@ describe('Persistence', function () { var data = d.getAllData() , doc1 = _.find(data, function (doc) { return doc.a === 1; }) , doc2 = _.find(data, function (doc) { return doc.a === 2; }) - ; + ; assert.isNull(err); data.length.should.equal(2); doc1.a.should.equal(1); @@ -212,7 +213,7 @@ describe('Persistence', function () { var data = d.getAllData() , doc1 = _.find(data, function (doc) { return doc.a === 1; }) , doc2 = _.find(data, function (doc) { return doc.a === 2; }) - ; + ; assert.isNull(err); data.length.should.equal(2); doc1.a.should.equal(1); @@ -240,7 +241,7 @@ describe('Persistence', function () { var data = d.getAllData() , doc1 = _.find(data, function (doc) { return doc.a === 1; }) , doc2 = _.find(data, function (doc) { return doc.a === 2; }) - ; + ; assert.isNull(err); data.length.should.equal(2); doc1.a.should.equal(1); @@ -250,9 +251,9 @@ describe('Persistence', function () { assert.isNull(err); d.loadDatabase(function (err) { var data = d.getAllData() - , doc1 = _.find(data, function (doc) { return doc.a === 1; }) - , doc2 = _.find(data, function (doc) { return doc.a === 2; }) - , doc3 = _.find(data, function (doc) { return doc.a === 3; }) + , doc1 = _.find(data, function (doc) { return doc.a === 1; }) + , doc2 = _.find(data, function (doc) { return doc.a === 2; }) + , doc3 = _.find(data, function (doc) { return doc.a === 3; }) ; assert.isNull(err); data.length.should.equal(1); @@ -272,7 +273,7 @@ describe('Persistence', function () { var corruptTestFilename = 'workspace/corruptTest.db' , fakeData = '{"_id":"one","hello":"world"}\n' + 'Some corrupt data\n' + '{"_id":"two","hello":"earth"}\n' + '{"_id":"three","hello":"you"}\n' , d - ; + ; fs.writeFileSync(corruptTestFilename, fakeData, "utf8"); // Default corruptAlertThreshold @@ -285,7 +286,7 @@ describe('Persistence', function () { d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 1 }); d.loadDatabase(function (err) { assert.isNull(err); - + fs.writeFileSync(corruptTestFilename, fakeData, "utf8"); d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 0 }); d.loadDatabase(function (err) { @@ -295,32 +296,32 @@ describe('Persistence', function () { done(); }); }); - }); + }); }); - - + + describe('Serialization hooks', function () { var as = function (s) { return "before_" + s + "_after"; } - , bd = function (s) { return s.substring(7, s.length - 6); } - + , bd = function (s) { return s.substring(7, s.length - 6); } + it("Declaring only one hook will throw an exception to prevent data loss", function (done) { var hookTestFilename = 'workspace/hookTest.db' - Persistence.ensureFileDoesntExist(hookTestFilename, function () { + storage.ensureFileDoesntExist(hookTestFilename, function () { fs.writeFileSync(hookTestFilename, "Some content", "utf8"); - + (function () { new Datastore({ filename: hookTestFilename, autoload: true , afterSerialization: as - }); + }); }).should.throw(); - + // Data file left untouched fs.readFileSync(hookTestFilename, "utf8").should.equal("Some content"); - + (function () { new Datastore({ filename: hookTestFilename, autoload: true , beforeDeserialization: bd - }); + }); }).should.throw(); // Data file left untouched @@ -329,17 +330,17 @@ describe('Persistence', function () { done(); }); }); - + it("Declaring two hooks that are not reverse of one another will cause an exception to prevent data loss", function (done) { var hookTestFilename = 'workspace/hookTest.db' - Persistence.ensureFileDoesntExist(hookTestFilename, function () { + storage.ensureFileDoesntExist(hookTestFilename, function () { fs.writeFileSync(hookTestFilename, "Some content", "utf8"); - + (function () { new Datastore({ filename: hookTestFilename, autoload: true , afterSerialization: as , beforeDeserialization: function (s) { return s; } - }); + }); }).should.throw(); // Data file left untouched @@ -348,24 +349,24 @@ describe('Persistence', function () { done(); }); }); - + it("A serialization hook can be used to transform data before writing new state to disk", function (done) { var hookTestFilename = 'workspace/hookTest.db' - Persistence.ensureFileDoesntExist(hookTestFilename, function () { + storage.ensureFileDoesntExist(hookTestFilename, function () { var d = new Datastore({ filename: hookTestFilename, autoload: true - , afterSerialization: as - , beforeDeserialization: bd - }) - ; - + , afterSerialization: as + , beforeDeserialization: bd + }) + ; + d.insert({ hello: "world" }, function () { var _data = fs.readFileSync(hookTestFilename, 'utf8') , data = _data.split('\n') , doc0 = bd(data[0]) - ; - + ; + data.length.should.equal(2); - + data[0].substring(0, 7).should.equal('before_'); data[0].substring(data[0].length - 6).should.equal('_after'); @@ -378,10 +379,10 @@ describe('Persistence', function () { , data = _data.split('\n') , doc0 = bd(data[0]) , doc1 = bd(data[1]) - ; - + ; + data.length.should.equal(3); - + data[0].substring(0, 7).should.equal('before_'); data[0].substring(data[0].length - 6).should.equal('_after'); data[1].substring(0, 7).should.equal('before_'); @@ -401,10 +402,10 @@ describe('Persistence', function () { , doc0 = bd(data[0]) , doc1 = bd(data[1]) , idx = bd(data[2]) - ; - + ; + data.length.should.equal(4); - + data[0].substring(0, 7).should.equal('before_'); data[0].substring(data[0].length - 6).should.equal('_after'); data[1].substring(0, 7).should.equal('before_'); @@ -412,33 +413,33 @@ describe('Persistence', function () { doc0 = model.deserialize(doc0); Object.keys(doc0).length.should.equal(2); - doc0.hello.should.equal('world'); + doc0.hello.should.equal('world'); doc1 = model.deserialize(doc1); Object.keys(doc1).length.should.equal(2); doc1.p.should.equal('Mars'); - + idx = model.deserialize(idx); assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } }); - + done(); - }); + }); }); }); }); }); - + it("Use serialization hook when persisting cached database or compacting", function (done) { var hookTestFilename = 'workspace/hookTest.db' - Persistence.ensureFileDoesntExist(hookTestFilename, function () { + storage.ensureFileDoesntExist(hookTestFilename, function () { var d = new Datastore({ filename: hookTestFilename, autoload: true - , afterSerialization: as - , beforeDeserialization: bd - }) - ; + , afterSerialization: as + , beforeDeserialization: bd + }) + ; d.insert({ hello: "world" }, function () { - d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () { + d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () { d.ensureIndex({ fieldName: 'idefix' }, function () { var _data = fs.readFileSync(hookTestFilename, 'utf8') , data = _data.split('\n') @@ -446,21 +447,21 @@ describe('Persistence', function () { , doc1 = bd(data[1]) , idx = bd(data[2]) , _id - ; - + ; + data.length.should.equal(4); - + doc0 = model.deserialize(doc0); Object.keys(doc0).length.should.equal(2); - doc0.hello.should.equal('world'); - + doc0.hello.should.equal('world'); + doc1 = model.deserialize(doc1); Object.keys(doc1).length.should.equal(2); doc1.hello.should.equal('earth'); doc0._id.should.equal(doc1._id); _id = doc0._id; - + idx = model.deserialize(idx); assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } }); @@ -469,19 +470,19 @@ describe('Persistence', function () { , data = _data.split('\n') , doc0 = bd(data[0]) , idx = bd(data[1]) - ; - + ; + data.length.should.equal(3); - + doc0 = model.deserialize(doc0); Object.keys(doc0).length.should.equal(2); doc0.hello.should.equal('earth'); doc0._id.should.equal(_id); - + idx = model.deserialize(idx); assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix', unique: false, sparse: false } }); - + done(); }); }); @@ -489,15 +490,15 @@ describe('Persistence', function () { }); }); }); - + it("Deserialization hook is correctly used when loading data", function (done) { var hookTestFilename = 'workspace/hookTest.db' - Persistence.ensureFileDoesntExist(hookTestFilename, function () { + storage.ensureFileDoesntExist(hookTestFilename, function () { var d = new Datastore({ filename: hookTestFilename, autoload: true - , afterSerialization: as - , beforeDeserialization: bd - }) - ; + , afterSerialization: as + , beforeDeserialization: bd + }) + ; d.insert({ hello: "world" }, function (err, doc) { var _id = doc._id; @@ -507,22 +508,22 @@ describe('Persistence', function () { d.ensureIndex({ fieldName: 'idefix' }, function () { var _data = fs.readFileSync(hookTestFilename, 'utf8') , data = _data.split('\n') - ; + ; data.length.should.equal(6); // Everything is deserialized correctly, including deletes and indexes var d = new Datastore({ filename: hookTestFilename - , afterSerialization: as - , beforeDeserialization: bd - }) - ; + , afterSerialization: as + , beforeDeserialization: bd + }) + ; d.loadDatabase(function () { d.find({}, function (err, docs) { docs.length.should.equal(1); docs[0].hello.should.equal("earth"); docs[0]._id.should.equal(_id); - + Object.keys(d.indexes).length.should.equal(2); Object.keys(d.indexes).indexOf("idefix").should.not.equal(-1); @@ -531,111 +532,111 @@ describe('Persistence', function () { }); }); }); - }); + }); }); }); }); }); - + }); // ==== End of 'Serialization hooks' ==== // - + describe('Prevent dataloss when persisting data', function () { it('Creating a datastore with in memory as true and a bad filename wont cause an error', function () { new Datastore({ filename: 'workspace/bad.db~', inMemoryOnly: true }); }) - + it('Creating a persistent datastore with a bad filename will cause an error', function () { (function () { new Datastore({ filename: 'workspace/bad.db~' }); }).should.throw(); - }) - + }) + it('If no file exists, ensureDatafileIntegrity creates an empty datafile', function (done) { var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } }); - + if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); } if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); } - + fs.existsSync('workspace/it.db').should.equal(false); - fs.existsSync('workspace/it.db~~').should.equal(false); - - p.ensureDatafileIntegrity(function (err) { + fs.existsSync('workspace/it.db~~').should.equal(false); + + storage.ensureDatafileIntegrity(p.filename, function (err) { assert.isNull(err); - + fs.existsSync('workspace/it.db').should.equal(true); fs.existsSync('workspace/it.db~~').should.equal(false); - + fs.readFileSync('workspace/it.db', 'utf8').should.equal(''); - + done(); }); }); - + it('If only datafile exists, ensureDatafileIntegrity will use it', function (done) { var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } }); - + if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); } if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); } - + fs.writeFileSync('workspace/it.db', 'something', 'utf8'); fs.existsSync('workspace/it.db').should.equal(true); - fs.existsSync('workspace/it.db~~').should.equal(false); - - p.ensureDatafileIntegrity(function (err) { + fs.existsSync('workspace/it.db~~').should.equal(false); + + storage.ensureDatafileIntegrity(p.filename, function (err) { assert.isNull(err); fs.existsSync('workspace/it.db').should.equal(true); fs.existsSync('workspace/it.db~~').should.equal(false); - + fs.readFileSync('workspace/it.db', 'utf8').should.equal('something'); - + done(); }); }); - + it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) { var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } }); - + if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); } if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); } - + fs.writeFileSync('workspace/it.db~~', 'something', 'utf8'); - + fs.existsSync('workspace/it.db').should.equal(false); - fs.existsSync('workspace/it.db~~').should.equal(true); - - p.ensureDatafileIntegrity(function (err) { + fs.existsSync('workspace/it.db~~').should.equal(true); + + storage.ensureDatafileIntegrity(p.filename, function (err) { assert.isNull(err); - + fs.existsSync('workspace/it.db').should.equal(true); fs.existsSync('workspace/it.db~~').should.equal(false); - + fs.readFileSync('workspace/it.db', 'utf8').should.equal('something'); - + done(); }); }); - + it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 4 of persistence failed', function (done) { var theDb = new Datastore({ filename: 'workspace/it.db' }); - + if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); } if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); } - + fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8'); fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8'); - + fs.existsSync('workspace/it.db').should.equal(true); - fs.existsSync('workspace/it.db~~').should.equal(true); - - theDb.persistence.ensureDatafileIntegrity(function (err) { + fs.existsSync('workspace/it.db~~').should.equal(true); + + storage.ensureDatafileIntegrity(theDb.persistence.filename, function (err) { assert.isNull(err); - + fs.existsSync('workspace/it.db').should.equal(true); fs.existsSync('workspace/it.db~~').should.equal(true); - + fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}'); - + theDb.loadDatabase(function (err) { assert.isNull(err); theDb.find({}, function (err, docs) { @@ -647,22 +648,22 @@ describe('Persistence', function () { }); }); }); - + it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) { d.insert({ hello: 'world' }, function () { d.find({}, function (err, docs) { docs.length.should.equal(1); - + if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); } if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); } if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); } fs.existsSync(testDb).should.equal(false); - + fs.writeFileSync(testDb + '~', 'something', 'utf8'); fs.writeFileSync(testDb + '~~', 'something else', 'utf8'); fs.existsSync(testDb + '~').should.equal(true); fs.existsSync(testDb + '~~').should.equal(true); - + d.persistence.persistCachedDatabase(function (err) { var contents = fs.readFileSync(testDb, 'utf8'); assert.isNull(err); @@ -677,22 +678,22 @@ describe('Persistence', function () { }); }); }); - + it('After a persistCachedDatabase, there should be no temp or old filename', function (done) { d.insert({ hello: 'world' }, function () { d.find({}, function (err, docs) { docs.length.should.equal(1); - + if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); } if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); } if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); } fs.existsSync(testDb).should.equal(false); - + fs.writeFileSync(testDb + '~', 'bloup', 'utf8'); fs.writeFileSync(testDb + '~~', 'blap', 'utf8'); fs.existsSync(testDb + '~').should.equal(true); fs.existsSync(testDb + '~~').should.equal(true); - + d.persistence.persistCachedDatabase(function (err) { var contents = fs.readFileSync(testDb, 'utf8'); assert.isNull(err); @@ -707,19 +708,19 @@ describe('Persistence', function () { }); }); }); - + it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) { d.insert({ hello: 'world' }, function () { d.find({}, function (err, docs) { docs.length.should.equal(1); - + if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); } fs.writeFileSync(testDb + '~', 'blabla', 'utf8'); fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8'); fs.existsSync(testDb).should.equal(false); fs.existsSync(testDb + '~').should.equal(true); fs.existsSync(testDb + '~~').should.equal(true); - + d.persistence.persistCachedDatabase(function (err) { var contents = fs.readFileSync(testDb, 'utf8'); assert.isNull(err); @@ -734,22 +735,22 @@ describe('Persistence', function () { }); }); }); - + it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) { var dbFile = 'workspace/test2.db', theDb; - + if (fs.existsSync(dbFile)) { fs.unlinkSync(dbFile); } if (fs.existsSync(dbFile + '~')) { fs.unlinkSync(dbFile + '~'); } if (fs.existsSync(dbFile + '~~')) { fs.unlinkSync(dbFile + '~~'); } - + theDb = new Datastore({ filename: dbFile }); - + theDb.loadDatabase(function (err) { var contents = fs.readFileSync(dbFile, 'utf8'); assert.isNull(err); fs.existsSync(dbFile).should.equal(true); - fs.existsSync(dbFile + '~').should.equal(false); - fs.existsSync(dbFile + '~~').should.equal(false); + fs.existsSync(dbFile + '~').should.equal(false); + fs.existsSync(dbFile + '~~').should.equal(false); if (contents != "") { throw "Datafile contents not as expected"; } @@ -759,90 +760,90 @@ describe('Persistence', function () { it('Persistence works as expected when everything goes fine', function (done) { var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2; - + async.waterfall([ - async.apply(Persistence.ensureFileDoesntExist, dbFile) - , async.apply(Persistence.ensureFileDoesntExist, dbFile + '~') - , async.apply(Persistence.ensureFileDoesntExist, dbFile + '~~') - , function (cb) { - theDb = new Datastore({ filename: dbFile }); - theDb.loadDatabase(cb); - } - , function (cb) { - theDb.find({}, function (err, docs) { - assert.isNull(err); - docs.length.should.equal(0); - return cb(); - }); - } + async.apply(storage.ensureFileDoesntExist, dbFile) + , async.apply(storage.ensureFileDoesntExist, dbFile + '~') + , async.apply(storage.ensureFileDoesntExist, dbFile + '~~') + , function (cb) { + theDb = new Datastore({ filename: dbFile }); + theDb.loadDatabase(cb); + } + , function (cb) { + theDb.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(0); + return cb(); + }); + } , function (cb) { - theDb.insert({ a: 'hello' }, function (err, _doc1) { - assert.isNull(err); - doc1 = _doc1; - theDb.insert({ a: 'world' }, function (err, _doc2) { - assert.isNull(err); - doc2 = _doc2; - return cb(); - }); - }); + theDb.insert({ a: 'hello' }, function (err, _doc1) { + assert.isNull(err); + doc1 = _doc1; + theDb.insert({ a: 'world' }, function (err, _doc2) { + assert.isNull(err); + doc2 = _doc2; + return cb(); + }); + }); } , function (cb) { - theDb.find({}, function (err, docs) { - assert.isNull(err); - docs.length.should.equal(2); - _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); - _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); - return cb(); - }); + theDb.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(2); + _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); + _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); + return cb(); + }); } , function (cb) { - theDb.loadDatabase(cb); + theDb.loadDatabase(cb); } , function (cb) { // No change - theDb.find({}, function (err, docs) { - assert.isNull(err); - docs.length.should.equal(2); - _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); - _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); - return cb(); - }); + theDb.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(2); + _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); + _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); + return cb(); + }); } , function (cb) { - fs.existsSync(dbFile).should.equal(true); - fs.existsSync(dbFile + '~').should.equal(false); - fs.existsSync(dbFile + '~~').should.equal(false); - return cb(); + fs.existsSync(dbFile).should.equal(true); + fs.existsSync(dbFile + '~').should.equal(false); + fs.existsSync(dbFile + '~~').should.equal(false); + return cb(); } , function (cb) { - theDb2 = new Datastore({ filename: dbFile }); - theDb2.loadDatabase(cb); + theDb2 = new Datastore({ filename: dbFile }); + theDb2.loadDatabase(cb); } , function (cb) { // No change in second db - theDb2.find({}, function (err, docs) { - assert.isNull(err); - docs.length.should.equal(2); - _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); - _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); - return cb(); - }); + theDb2.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(2); + _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello'); + _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world'); + return cb(); + }); } , function (cb) { - fs.existsSync(dbFile).should.equal(true); - fs.existsSync(dbFile + '~').should.equal(false); - fs.existsSync(dbFile + '~~').should.equal(false); - return cb(); + fs.existsSync(dbFile).should.equal(true); + fs.existsSync(dbFile + '~').should.equal(false); + fs.existsSync(dbFile + '~~').should.equal(false); + return cb(); } ], done); }); - - + + // This test is a bit complicated since it depends on the time I/O actions take to execute // That depends on the machine and the load on the machine when the tests are run // It is timed for my machine with nothing else running but may not work as expected on others (it will not fail but may not be a proof) // Every new version of NeDB passes it on my machine before rtelease it('If system crashes during a loadDatabase, the former version is not lost', function (done) { var cp, N = 150000, toWrite = "", i; - + // Ensuring the state is clean if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); } if (fs.existsSync('workspace/lac.db~')) { fs.unlinkSync('workspace/lac.db~'); } @@ -852,46 +853,46 @@ describe('Persistence', function () { toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n'; } fs.writeFileSync('workspace/lac.db', toWrite, 'utf8'); - + // Loading it in a separate process that we will crash before finishing the loadDatabase cp = child_process.fork('test_lac/loadAndCrash.test') - + // Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile) setTimeout(function() { cp.kill('SIGINT'); - + // If the timing is correct, only the temp datafile contains data // The datafile was in the middle of being written and is empty - + // Let the process crash be finished then load database without a crash, and test we didn't lose data setTimeout(function () { var db = new Datastore({ filename: 'workspace/lac.db' }); db.loadDatabase(function (err) { assert.isNull(err); - + db.count({}, function (err, n) { // Data has not been lost assert.isNull(err); n.should.equal(150000); - + // State is clean, the temp datafile has been erased and the datafile contains all the data fs.existsSync('workspace/lac.db').should.equal(true); fs.existsSync('workspace/lac.db~').should.equal(false); - + done(); }); }); - }, 100); + }, 100); }, 2000); }); - + }); // ==== End of 'Prevent dataloss when persisting data' ==== describe('ensureFileDoesntExist', function () { - + it('Doesnt do anything if file already doesnt exist', function (done) { - Persistence.ensureFileDoesntExist('workspace/nonexisting', function (err) { + storage.ensureFileDoesntExist('workspace/nonexisting', function (err) { assert.isNull(err); fs.existsSync('workspace/nonexisting').should.equal(false); done(); @@ -901,14 +902,14 @@ describe('Persistence', function () { it('Deletes file if it exists', function (done) { fs.writeFileSync('workspace/existing', 'hello world', 'utf8'); fs.existsSync('workspace/existing').should.equal(true); - - Persistence.ensureFileDoesntExist('workspace/existing', function (err) { + + storage.ensureFileDoesntExist('workspace/existing', function (err) { assert.isNull(err); fs.existsSync('workspace/existing').should.equal(false); done(); }); }); - + }); // ==== End of 'ensureFileDoesntExist' ====