TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Moved all crash safe operations to storage

Browse Source
pull/2/head
Louis Chatriot 9 years ago
parent 26441f1aae
commit d1e6d14b53
5 changed files with 343 additions and 312 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      browser-version/browser-specific/lib/storage.js
  2. 83
      lib/persistence.js
  3. 90
      lib/storage.js
  4. 1
      package.json
  5. 475
      test/persistence.test.js

6
browser-version/browser-specific/lib/storage.js
Unescape View File

@ -36,7 +36,7 @@ function rename (filename, newFilename, callback) {
function writeFile (filename, contents, options, callback) {
if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options; }
@ -47,7 +47,7 @@ function writeFile (filename, contents, options, callback) {
function appendFile (filename, toAppend, options, callback) {
if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options; }
@ -61,7 +61,7 @@ function appendFile (filename, toAppend, options, callback) {
function readFile (filename, options, callback) {
if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options; }

83
lib/persistence.js
Unescape View File

@ -22,19 +22,14 @@ var storage = require('./storage')
*/
function Persistence (options) {
var i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename) {
if (this.filename.charAt(this.filename.length - 1) === '~') {
throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
} else {
this.tempFilename = this.filename + '~';
this.oldFilename = this.filename + '~~';
}
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw "The datafile name can't end with a ~, which is reserved for crash safe backup files";
}
// After serialization and before deserialization hooks with some basic sanity checks
@ -54,7 +49,7 @@ function Persistence (options) {
}
}
}
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log("==================================================================");
@ -65,8 +60,6 @@ function Persistence (options) {
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
this.tempFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.tempFilename);
this.oldFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.oldFilename);
}
};
@ -83,13 +76,6 @@ Persistence.ensureDirectoryExists = function (dir, cb) {
};
Persistence.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
/**
@ -137,7 +123,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
, self = this
;
if (this.inMemoryOnly) { return callback(null); }
if (this.inMemoryOnly) { return callback(null); }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
@ -148,26 +134,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
}
});
async.waterfall([
async.apply(Persistence.ensureFileDoesntExist, self.tempFilename)
, async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
, function (cb) {
storage.exists(self.filename, function (exists) {
if (exists) {
storage.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
}
, function (cb) {
storage.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
}
, async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
storage.crashSafeWriteFile(this.filename, toPersist, callback);
};
@ -244,10 +211,10 @@ Persistence.prototype.treatRawData = function (rawData) {
, indexes = {}
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(this.beforeDeserialization(data[i]));
if (doc._id) {
@ -265,7 +232,7 @@ Persistence.prototype.treatRawData = function (rawData) {
corruptItems += 1;
}
}
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw "More than 10% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss"
@ -279,30 +246,6 @@ Persistence.prototype.treatRawData = function (rawData) {
};
/**
* Ensure that this.filename contains the most up-to-date version of the data
* Even if a loadDatabase crashed before
*/
Persistence.prototype.ensureDatafileIntegrity = function (callback) {
var self = this ;
storage.exists(self.filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(self.oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
});
});
};
/**
* Load the database
* 1) Create all indexes
@ -326,16 +269,16 @@ Persistence.prototype.loadDatabase = function (cb) {
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
self.ensureDatafileIntegrity(function (exists) {
storage.ensureDatafileIntegrity(self.filename, function (exists) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);

90
lib/storage.js
Unescape View File

@ -4,12 +4,98 @@
* For a browser-side database it's localStorage when supported
*
* This version is the Node.js/Node Webkit version
* It's essentially fs, mkdirp and crash safe write and read functions
*/
var fs = require('fs')
, mkdirp = require('mkdirp')
, async = require('async')
, storage = {}
;
storage.exists = fs.exists;
storage.rename = fs.rename;
storage.writeFile = fs.writeFile;
storage.unlink = fs.unlink;
storage.appendFile = fs.appendFile;
storage.readFile = fs.readFile;
storage.mkdirp = mkdirp;
module.exports = fs;
module.exports.mkdirp = mkdirp;
/**
* Explicit name ...
*/
storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
/**
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
* @param {String} filename
* @param {String} data
* @param {Function} cb Optional callback, signature: err
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
async.waterfall([
async.apply(storage.ensureFileDoesntExist, tempFilename)
, async.apply(storage.ensureFileDoesntExist, oldFilename)
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.rename(filename, oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.ensureFileDoesntExist, oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
};
/**
* Ensure the datafile contains all the data, even if there was a crash during a full file write
* @param {String} filename
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(oldFilename, filename, function (err) { return callback(err); });
});
});
};
// Interface
module.exports = storage;

1
package.json
Unescape View File

@ -22,6 +22,7 @@
"dependencies": {
"async": "0.2.10",
"binary-search-tree": "0.2.4",
"localforage": "^1.3.0",
"mkdirp": "~0.5.1",
"underscore": "~1.4.4"
},

475
test/persistence.test.js
Unescape View File

@ -9,8 +9,9 @@ var should = require('chai').should()
, customUtils = require('../lib/customUtils')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, storage = require('../lib/storage')
, child_process = require('child_process')
;
;
describe('Persistence', function () {
@ -32,22 +33,22 @@ describe('Persistence', function () {
});
}
, function (cb) {
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return cb();
});
}
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return cb();
});
}
], done);
});
it('Every line represents a document', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "3", nested: { today: now } })
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "3", nested: { today: now } })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(3);
@ -59,10 +60,10 @@ describe('Persistence', function () {
it('Badly formatted lines have no impact on the treated data', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
'garbage\n' +
model.serialize({ _id: "3", nested: { today: now } })
'garbage\n' +
model.serialize({ _id: "3", nested: { today: now } })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
@ -73,10 +74,10 @@ describe('Persistence', function () {
it('Well formatted lines that have no _id are not included in the data', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ nested: { today: now } })
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ nested: { today: now } })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
@ -87,10 +88,10 @@ describe('Persistence', function () {
it('If two lines concern the same doc (= same _id), the last one is the good version', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "1", nested: { today: now } })
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "1", nested: { today: now } })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
@ -101,11 +102,11 @@ describe('Persistence', function () {
it('If a doc contains $$deleted: true, that means we need to remove it from the data', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "1", $$deleted: true }) + '\n' +
model.serialize({ _id: "3", today: now })
model.serialize({ _id: "2", hello: 'world' }) + '\n' +
model.serialize({ _id: "1", $$deleted: true }) + '\n' +
model.serialize({ _id: "3", today: now })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
@ -116,29 +117,29 @@ describe('Persistence', function () {
it('If a doc contains $$deleted: true, no error is thrown if the doc wasnt in the list before', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ _id: "2", $$deleted: true }) + '\n' +
model.serialize({ _id: "3", today: now })
model.serialize({ _id: "2", $$deleted: true }) + '\n' +
model.serialize({ _id: "3", today: now })
, treatedData = d.persistence.treatRawData(rawData).data
;
;
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
_.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
_.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true);
});
it('If a doc contains $$indexCreated, no error is thrown during treatRawData and we can get the index options', function () {
var now = new Date()
, rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' +
model.serialize({ _id: "3", today: now })
model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' +
model.serialize({ _id: "3", today: now })
, treatedData = d.persistence.treatRawData(rawData).data
, indexes = d.persistence.treatRawData(rawData).indexes
;
;
Object.keys(indexes).length.should.equal(1);
assert.deepEqual(indexes.test, { fieldName: "test", unique: true });
treatedData.sort(function (a, b) { return a._id - b._id; });
treatedData.length.should.equal(2);
_.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
@ -181,7 +182,7 @@ describe('Persistence', function () {
var data = d.getAllData()
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
;
;
assert.isNull(err);
data.length.should.equal(2);
doc1.a.should.equal(1);
@ -191,7 +192,7 @@ describe('Persistence', function () {
var data = d.getAllData()
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
;
;
assert.isNull(err);
data.length.should.equal(2);
doc1.a.should.equal(1);
@ -212,7 +213,7 @@ describe('Persistence', function () {
var data = d.getAllData()
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
;
;
assert.isNull(err);
data.length.should.equal(2);
doc1.a.should.equal(1);
@ -240,7 +241,7 @@ describe('Persistence', function () {
var data = d.getAllData()
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
;
;
assert.isNull(err);
data.length.should.equal(2);
doc1.a.should.equal(1);
@ -250,9 +251,9 @@ describe('Persistence', function () {
assert.isNull(err);
d.loadDatabase(function (err) {
var data = d.getAllData()
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
, doc3 = _.find(data, function (doc) { return doc.a === 3; })
, doc1 = _.find(data, function (doc) { return doc.a === 1; })
, doc2 = _.find(data, function (doc) { return doc.a === 2; })
, doc3 = _.find(data, function (doc) { return doc.a === 3; })
;
assert.isNull(err);
data.length.should.equal(1);
@ -272,7 +273,7 @@ describe('Persistence', function () {
var corruptTestFilename = 'workspace/corruptTest.db'
, fakeData = '{"_id":"one","hello":"world"}\n' + 'Some corrupt data\n' + '{"_id":"two","hello":"earth"}\n' + '{"_id":"three","hello":"you"}\n'
, d
;
;
fs.writeFileSync(corruptTestFilename, fakeData, "utf8");
// Default corruptAlertThreshold
@ -285,7 +286,7 @@ describe('Persistence', function () {
d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 1 });
d.loadDatabase(function (err) {
assert.isNull(err);
fs.writeFileSync(corruptTestFilename, fakeData, "utf8");
d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 0 });
d.loadDatabase(function (err) {
@ -295,32 +296,32 @@ describe('Persistence', function () {
done();
});
});
});
});
});
describe('Serialization hooks', function () {
var as = function (s) { return "before_" + s + "_after"; }
, bd = function (s) { return s.substring(7, s.length - 6); }
, bd = function (s) { return s.substring(7, s.length - 6); }
it("Declaring only one hook will throw an exception to prevent data loss", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
fs.writeFileSync(hookTestFilename, "Some content", "utf8");
(function () {
new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
});
});
}).should.throw();
// Data file left untouched
fs.readFileSync(hookTestFilename, "utf8").should.equal("Some content");
(function () {
new Datastore({ filename: hookTestFilename, autoload: true
, beforeDeserialization: bd
});
});
}).should.throw();
// Data file left untouched
@ -329,17 +330,17 @@ describe('Persistence', function () {
done();
});
});
it("Declaring two hooks that are not reverse of one another will cause an exception to prevent data loss", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
fs.writeFileSync(hookTestFilename, "Some content", "utf8");
(function () {
new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: function (s) { return s; }
});
});
}).should.throw();
// Data file left untouched
@ -348,24 +349,24 @@ describe('Persistence', function () {
done();
});
});
it("A serialization hook can be used to transform data before writing new state to disk", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
})
;
, afterSerialization: as
, beforeDeserialization: bd
})
;
d.insert({ hello: "world" }, function () {
var _data = fs.readFileSync(hookTestFilename, 'utf8')
, data = _data.split('\n')
, doc0 = bd(data[0])
;
;
data.length.should.equal(2);
data[0].substring(0, 7).should.equal('before_');
data[0].substring(data[0].length - 6).should.equal('_after');
@ -378,10 +379,10 @@ describe('Persistence', function () {
, data = _data.split('\n')
, doc0 = bd(data[0])
, doc1 = bd(data[1])
;
;
data.length.should.equal(3);
data[0].substring(0, 7).should.equal('before_');
data[0].substring(data[0].length - 6).should.equal('_after');
data[1].substring(0, 7).should.equal('before_');
@ -401,10 +402,10 @@ describe('Persistence', function () {
, doc0 = bd(data[0])
, doc1 = bd(data[1])
, idx = bd(data[2])
;
;
data.length.should.equal(4);
data[0].substring(0, 7).should.equal('before_');
data[0].substring(data[0].length - 6).should.equal('_after');
data[1].substring(0, 7).should.equal('before_');
@ -412,33 +413,33 @@ describe('Persistence', function () {
doc0 = model.deserialize(doc0);
Object.keys(doc0).length.should.equal(2);
doc0.hello.should.equal('world');
doc0.hello.should.equal('world');
doc1 = model.deserialize(doc1);
Object.keys(doc1).length.should.equal(2);
doc1.p.should.equal('Mars');
idx = model.deserialize(idx);
assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } });
done();
});
});
});
});
});
});
it("Use serialization hook when persisting cached database or compacting", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
})
;
, afterSerialization: as
, beforeDeserialization: bd
})
;
d.insert({ hello: "world" }, function () {
d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () {
d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () {
d.ensureIndex({ fieldName: 'idefix' }, function () {
var _data = fs.readFileSync(hookTestFilename, 'utf8')
, data = _data.split('\n')
@ -446,21 +447,21 @@ describe('Persistence', function () {
, doc1 = bd(data[1])
, idx = bd(data[2])
, _id
;
;
data.length.should.equal(4);
doc0 = model.deserialize(doc0);
Object.keys(doc0).length.should.equal(2);
doc0.hello.should.equal('world');
doc0.hello.should.equal('world');
doc1 = model.deserialize(doc1);
Object.keys(doc1).length.should.equal(2);
doc1.hello.should.equal('earth');
doc0._id.should.equal(doc1._id);
_id = doc0._id;
idx = model.deserialize(idx);
assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } });
@ -469,19 +470,19 @@ describe('Persistence', function () {
, data = _data.split('\n')
, doc0 = bd(data[0])
, idx = bd(data[1])
;
;
data.length.should.equal(3);
doc0 = model.deserialize(doc0);
Object.keys(doc0).length.should.equal(2);
doc0.hello.should.equal('earth');
doc0._id.should.equal(_id);
idx = model.deserialize(idx);
assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix', unique: false, sparse: false } });
done();
});
});
@ -489,15 +490,15 @@ describe('Persistence', function () {
});
});
});
it("Deserialization hook is correctly used when loading data", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
})
;
, afterSerialization: as
, beforeDeserialization: bd
})
;
d.insert({ hello: "world" }, function (err, doc) {
var _id = doc._id;
@ -507,22 +508,22 @@ describe('Persistence', function () {
d.ensureIndex({ fieldName: 'idefix' }, function () {
var _data = fs.readFileSync(hookTestFilename, 'utf8')
, data = _data.split('\n')
;
;
data.length.should.equal(6);
// Everything is deserialized correctly, including deletes and indexes
var d = new Datastore({ filename: hookTestFilename
, afterSerialization: as
, beforeDeserialization: bd
})
;
, afterSerialization: as
, beforeDeserialization: bd
})
;
d.loadDatabase(function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
docs[0].hello.should.equal("earth");
docs[0]._id.should.equal(_id);
Object.keys(d.indexes).length.should.equal(2);
Object.keys(d.indexes).indexOf("idefix").should.not.equal(-1);
@ -531,111 +532,111 @@ describe('Persistence', function () {
});
});
});
});
});
});
});
});
});
}); // ==== End of 'Serialization hooks' ==== //
describe('Prevent dataloss when persisting data', function () {
it('Creating a datastore with in memory as true and a bad filename wont cause an error', function () {
new Datastore({ filename: 'workspace/bad.db~', inMemoryOnly: true });
})
it('Creating a persistent datastore with a bad filename will cause an error', function () {
(function () { new Datastore({ filename: 'workspace/bad.db~' }); }).should.throw();
})
})
it('If no file exists, ensureDatafileIntegrity creates an empty datafile', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
fs.existsSync('workspace/it.db~~').should.equal(false);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('');
done();
});
});
it('If only datafile exists, ensureDatafileIntegrity will use it', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
fs.existsSync('workspace/it.db~~').should.equal(false);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
done();
});
});
it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(true);
p.ensureDatafileIntegrity(function (err) {
fs.existsSync('workspace/it.db~~').should.equal(true);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
done();
});
});
it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 4 of persistence failed', function (done) {
var theDb = new Datastore({ filename: 'workspace/it.db' });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
theDb.persistence.ensureDatafileIntegrity(function (err) {
fs.existsSync('workspace/it.db~~').should.equal(true);
storage.ensureDatafileIntegrity(theDb.persistence.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');
theDb.loadDatabase(function (err) {
assert.isNull(err);
theDb.find({}, function (err, docs) {
@ -647,22 +648,22 @@ describe('Persistence', function () {
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.writeFileSync(testDb + '~', 'something', 'utf8');
fs.writeFileSync(testDb + '~~', 'something else', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
@ -677,22 +678,22 @@ describe('Persistence', function () {
});
});
});
it('After a persistCachedDatabase, there should be no temp or old filename', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
@ -707,19 +708,19 @@ describe('Persistence', function () {
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
@ -734,22 +735,22 @@ describe('Persistence', function () {
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
var dbFile = 'workspace/test2.db', theDb;
if (fs.existsSync(dbFile)) { fs.unlinkSync(dbFile); }
if (fs.existsSync(dbFile + '~')) { fs.unlinkSync(dbFile + '~'); }
if (fs.existsSync(dbFile + '~~')) { fs.unlinkSync(dbFile + '~~'); }
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(function (err) {
var contents = fs.readFileSync(dbFile, 'utf8');
assert.isNull(err);
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
if (contents != "") {
throw "Datafile contents not as expected";
}
@ -759,90 +760,90 @@ describe('Persistence', function () {
it('Persistence works as expected when everything goes fine', function (done) {
var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2;
async.waterfall([
async.apply(Persistence.ensureFileDoesntExist, dbFile)
, async.apply(Persistence.ensureFileDoesntExist, dbFile + '~')
, async.apply(Persistence.ensureFileDoesntExist, dbFile + '~~')
, function (cb) {
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(cb);
}
, function (cb) {
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(0);
return cb();
});
}
async.apply(storage.ensureFileDoesntExist, dbFile)
, async.apply(storage.ensureFileDoesntExist, dbFile + '~')
, async.apply(storage.ensureFileDoesntExist, dbFile + '~~')
, function (cb) {
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(cb);
}
, function (cb) {
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(0);
return cb();
});
}
, function (cb) {
theDb.insert({ a: 'hello' }, function (err, _doc1) {
assert.isNull(err);
doc1 = _doc1;
theDb.insert({ a: 'world' }, function (err, _doc2) {
assert.isNull(err);
doc2 = _doc2;
return cb();
});
});
theDb.insert({ a: 'hello' }, function (err, _doc1) {
assert.isNull(err);
doc1 = _doc1;
theDb.insert({ a: 'world' }, function (err, _doc2) {
assert.isNull(err);
doc2 = _doc2;
return cb();
});
});
}
, function (cb) {
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
theDb.loadDatabase(cb);
theDb.loadDatabase(cb);
}
, function (cb) { // No change
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
}
, function (cb) {
theDb2 = new Datastore({ filename: dbFile });
theDb2.loadDatabase(cb);
theDb2 = new Datastore({ filename: dbFile });
theDb2.loadDatabase(cb);
}
, function (cb) { // No change in second db
theDb2.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb2.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
}
], done);
});
// This test is a bit complicated since it depends on the time I/O actions take to execute
// That depends on the machine and the load on the machine when the tests are run
// It is timed for my machine with nothing else running but may not work as expected on others (it will not fail but may not be a proof)
// Every new version of NeDB passes it on my machine before rtelease
it('If system crashes during a loadDatabase, the former version is not lost', function (done) {
var cp, N = 150000, toWrite = "", i;
// Ensuring the state is clean
if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); }
if (fs.existsSync('workspace/lac.db~')) { fs.unlinkSync('workspace/lac.db~'); }
@ -852,46 +853,46 @@ describe('Persistence', function () {
toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n';
}
fs.writeFileSync('workspace/lac.db', toWrite, 'utf8');
// Loading it in a separate process that we will crash before finishing the loadDatabase
cp = child_process.fork('test_lac/loadAndCrash.test')
// Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile)
setTimeout(function() {
cp.kill('SIGINT');
// If the timing is correct, only the temp datafile contains data
// The datafile was in the middle of being written and is empty
// Let the process crash be finished then load database without a crash, and test we didn't lose data
setTimeout(function () {
var db = new Datastore({ filename: 'workspace/lac.db' });
db.loadDatabase(function (err) {
assert.isNull(err);
db.count({}, function (err, n) {
// Data has not been lost
assert.isNull(err);
n.should.equal(150000);
// State is clean, the temp datafile has been erased and the datafile contains all the data
fs.existsSync('workspace/lac.db').should.equal(true);
fs.existsSync('workspace/lac.db~').should.equal(false);
done();
});
});
}, 100);
}, 100);
}, 2000);
});
}); // ==== End of 'Prevent dataloss when persisting data' ====
describe('ensureFileDoesntExist', function () {
it('Doesnt do anything if file already doesnt exist', function (done) {
Persistence.ensureFileDoesntExist('workspace/nonexisting', function (err) {
storage.ensureFileDoesntExist('workspace/nonexisting', function (err) {
assert.isNull(err);
fs.existsSync('workspace/nonexisting').should.equal(false);
done();
@ -901,14 +902,14 @@ describe('Persistence', function () {
it('Deletes file if it exists', function (done) {
fs.writeFileSync('workspace/existing', 'hello world', 'utf8');
fs.existsSync('workspace/existing').should.equal(true);
Persistence.ensureFileDoesntExist('workspace/existing', function (err) {
storage.ensureFileDoesntExist('workspace/existing', function (err) {
assert.isNull(err);
fs.existsSync('workspace/existing').should.equal(false);
done();
});
});
}); // ==== End of 'ensureFileDoesntExist' ====

Write Preview
Loading…
Cancel
Save