TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Redis AOF-style fsync

Browse Source
pull/2/head
Louis Chatriot 9 years ago
parent 37336c270e
commit 6884f83df8
3 changed files with 110 additions and 117 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 43
      lib/storage.js
  2. 15
      test/db.test.js
  3. 169
      test/persistence.test.js

43
lib/storage.js
Unescape View File

@ -10,6 +10,7 @@
var fs = require('fs')
, mkdirp = require('mkdirp')
, async = require('async')
, path = require('path')
, storage = {}
;
@ -35,15 +36,26 @@ storage.ensureFileDoesntExist = function (file, callback) {
/**
* Flush data in OS buffer to storage if corresponding option is set
* Flush data in OS buffer to storage if corresponding option is set
* @param {String} options.filename
* @param {Boolean} options.isDir Optional, defaults to false
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/
storage.flushToStorage = function (filename, callback) {
fs.open(filename, 'w', function (err, fd) {
storage.flushToStorage = function (options, callback) {
var filename, flags;
if (typeof options === 'string') {
filename = options;
flags = 'r+';
} else {
filename = options.filename;
flags = options.isDir ? 'r' : 'r+';
}
fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err); }
fs.fsync(fd, function (err) {
return callback(err);
if (err) { return callback(err); }
fs.close(fd, function (err) { return callback(err); });
});
@ -59,17 +71,14 @@ storage.flushToStorage = function (filename, callback) {
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
, tempFilename = filename + '~';
async.waterfall([
async.apply(storage.ensureFileDoesntExist, tempFilename)
, async.apply(storage.ensureFileDoesntExist, oldFilename)
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.rename(filename, oldFilename, function (err) { return cb(err); });
storage.flushToStorage(filename, function (err) { return cb(err); });
} else {
return cb();
}
@ -78,12 +87,12 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
//, async.apply(storage.flushToStorage, tempFilename)
, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
//, async.apply(storage.flushToStorage, filename)
, async.apply(storage.ensureFileDoesntExist, oldFilename)
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, async.apply(storage.ensureFileDoesntExist, tempFilename)
], function (err) { return callback(err); })
};
@ -94,22 +103,20 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
var tempFilename = filename + '~';
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(oldFilename, function (oldFilenameExists) {
storage.exists(tempFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(oldFilename, filename, function (err) { return callback(err); });
storage.rename(tempFilename, filename, function (err) { return callback(err); });
});
});
};

15
test/db.test.js
Unescape View File

@ -8,6 +8,7 @@ var should = require('chai').should()
, model = require('../lib/model')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, reloadTimeUpperBound = 60; // In ms, an upper bound for the reload time used to check createdAt and updatedAt
;
@ -285,7 +286,7 @@ describe('Database', function () {
insertedDoc.createdAt.should.equal(insertedDoc.updatedAt);
assert.isDefined(insertedDoc._id);
Object.keys(insertedDoc).length.should.equal(4);
assert.isBelow(Math.abs(insertedDoc.createdAt.getTime() - beginning), 30); // No more than 30ms should have elapsed (worst case, if there is a flush)
assert.isBelow(Math.abs(insertedDoc.createdAt.getTime() - beginning), reloadTimeUpperBound); // No more than 30ms should have elapsed (worst case, if there is a flush)
// Modifying results of insert doesn't change the cache
insertedDoc.bloup = "another";
@ -332,7 +333,7 @@ describe('Database', function () {
d.insert(newDoc, function (err, insertedDoc) {
Object.keys(insertedDoc).length.should.equal(4);
insertedDoc.createdAt.getTime().should.equal(234); // Not modified
assert.isBelow(insertedDoc.updatedAt.getTime() - beginning, 30); // Created
assert.isBelow(insertedDoc.updatedAt.getTime() - beginning, reloadTimeUpperBound); // Created
d.find({}, function (err, docs) {
assert.deepEqual(insertedDoc, docs[0]);
@ -354,7 +355,7 @@ describe('Database', function () {
d.insert(newDoc, function (err, insertedDoc) {
Object.keys(insertedDoc).length.should.equal(4);
insertedDoc.updatedAt.getTime().should.equal(234); // Not modified
assert.isBelow(insertedDoc.createdAt.getTime() - beginning, 30); // Created
assert.isBelow(insertedDoc.createdAt.getTime() - beginning, reloadTimeUpperBound); // Created
d.find({}, function (err, docs) {
assert.deepEqual(insertedDoc, docs[0]);
@ -961,8 +962,8 @@ describe('Database', function () {
var beginning = Date.now();
d = new Datastore({ filename: testDb, autoload: true, timestampData: true });
d.insert({ hello: 'world' }, function (err, insertedDoc) {
assert.isBelow(insertedDoc.updatedAt.getTime() - beginning, 30);
assert.isBelow(insertedDoc.createdAt.getTime() - beginning, 30);
assert.isBelow(insertedDoc.updatedAt.getTime() - beginning, reloadTimeUpperBound);
assert.isBelow(insertedDoc.createdAt.getTime() - beginning, reloadTimeUpperBound);
Object.keys(insertedDoc).length.should.equal(4);
// Wait 100ms before performing the update
@ -976,7 +977,7 @@ describe('Database', function () {
docs[0].createdAt.should.equal(insertedDoc.createdAt);
docs[0].hello.should.equal('mars');
assert.isAbove(docs[0].updatedAt.getTime() - beginning, 99); // updatedAt modified
assert.isBelow(docs[0].updatedAt.getTime() - step1, 30); // updatedAt modified
assert.isBelow(docs[0].updatedAt.getTime() - step1, reloadTimeUpperBound); // updatedAt modified
done();
});
@ -2396,7 +2397,7 @@ describe('Database', function () {
describe('Persisting indexes', function () {
it.only('Indexes are persisted to a separate file and recreated upon reload', function (done) {
it('Indexes are persisted to a separate file and recreated upon reload', function (done) {
var persDb = "workspace/persistIndexes.db"
, db
;

169
test/persistence.test.js
Unescape View File

@ -554,16 +554,16 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('');
@ -575,18 +575,18 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
fs.writeFileSync('workspace/it.db', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
@ -594,22 +594,22 @@ describe('Persistence', function () {
});
});
it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
it('If temp datafile exists and datafile doesnt, ensureDatafileIntegrity will use it (cannot happen except upon first use)', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');
fs.writeFileSync('workspace/it.db~', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
@ -617,23 +617,24 @@ describe('Persistence', function () {
});
});
it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 4 of persistence failed', function (done) {
// Technically it could also mean the write was successful but the rename wasn't, but there is in any case no guarantee that the data in the temp file is whole so we have to discard the whole file
it('If both temp and current datafiles exist, ensureDatafileIntegrity will use the datafile, as it means that the write of the temp file failed', function (done) {
var theDb = new Datastore({ filename: 'workspace/it.db' });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');
fs.writeFileSync('workspace/it.db~', '{"_id":"0","hello":"other"}', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
storage.ensureDatafileIntegrity(theDb.persistence.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');
@ -643,6 +644,8 @@ describe('Persistence', function () {
assert.isNull(err);
docs.length.should.equal(1);
docs[0].hello.should.equal("world");
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
done();
});
});
@ -656,20 +659,16 @@ describe('Persistence', function () {
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.writeFileSync(testDb + '~', 'something', 'utf8');
fs.writeFileSync(testDb + '~~', 'something else', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
@ -686,47 +685,41 @@ describe('Persistence', function () {
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
done();
});
});
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp datafile', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
@ -736,12 +729,11 @@ describe('Persistence', function () {
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp datafile', function (done) {
var dbFile = 'workspace/test2.db', theDb;
if (fs.existsSync(dbFile)) { fs.unlinkSync(dbFile); }
if (fs.existsSync(dbFile + '~')) { fs.unlinkSync(dbFile + '~'); }
if (fs.existsSync(dbFile + '~~')) { fs.unlinkSync(dbFile + '~~'); }
theDb = new Datastore({ filename: dbFile });
@ -750,7 +742,6 @@ describe('Persistence', function () {
assert.isNull(err);
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
if (contents != "") {
throw "Datafile contents not as expected";
}
@ -762,85 +753,79 @@ describe('Persistence', function () {
var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2;
async.waterfall([
async.apply(storage.ensureFileDoesntExist, dbFile)
async.apply(storage.ensureFileDoesntExist, dbFile)
, async.apply(storage.ensureFileDoesntExist, dbFile + '~')
, async.apply(storage.ensureFileDoesntExist, dbFile + '~~')
, function (cb) {
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(cb);
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(cb);
}
, function (cb) {
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(0);
return cb();
});
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(0);
return cb();
});
}
, function (cb) {
theDb.insert({ a: 'hello' }, function (err, _doc1) {
assert.isNull(err);
doc1 = _doc1;
theDb.insert({ a: 'world' }, function (err, _doc2) {
theDb.insert({ a: 'hello' }, function (err, _doc1) {
assert.isNull(err);
doc2 = _doc2;
return cb();
});
});
doc1 = _doc1;
theDb.insert({ a: 'world' }, function (err, _doc2) {
assert.isNull(err);
doc2 = _doc2;
return cb();
});
});
}
, function (cb) {
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
theDb.loadDatabase(cb);
theDb.loadDatabase(cb);
}
, function (cb) { // No change
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
return cb();
}
, function (cb) {
theDb2 = new Datastore({ filename: dbFile });
theDb2.loadDatabase(cb);
}
theDb2 = new Datastore({ filename: dbFile });
theDb2.loadDatabase(cb);
}
, function (cb) { // No change in second db
theDb2.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
theDb2.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(2);
_.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
_.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
return cb();
});
}
, function (cb) {
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
fs.existsSync(dbFile + '~~').should.equal(false);
return cb();
}
fs.existsSync(dbFile).should.equal(true);
fs.existsSync(dbFile + '~').should.equal(false);
return cb();
}
], done);
});
// This test is a bit complicated since it depends on the time I/O actions take to execute
// That depends on the machine and the load on the machine when the tests are run
// It is timed for my machine with nothing else running but may not work as expected on others (it will not fail but may not be a proof)
// Every new version of NeDB passes it on my machine before rtelease
// This test needs to be rewritten. The goal is to check what happens when the system crashes during a writeFile, so this test
// must rewrite a custom and buggy writeFile that will be used by the child process and crash in the midst of writing the file
it('If system crashes during a loadDatabase, the former version is not lost', function (done) {
var cp, N = 150000, toWrite = "", i;
@ -851,7 +836,7 @@ describe('Persistence', function () {
// Creating a db file with 150k records (a bit long to load)
for (i = 0; i < N; i += 1) {
toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n';
}
}
fs.writeFileSync('workspace/lac.db', toWrite, 'utf8');
// Loading it in a separate process that we will crash before finishing the loadDatabase

Write Preview
Loading…
Cancel
Save