TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use a more robust persistence scheme, thanks spolu and szawcz

Browse Source
pull/2/head
Louis Chatriot 11 years ago
parent 35b46c286b
commit 4776c988ec
4 changed files with 137 additions and 98 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      lib/customUtils.js
  2. 68
      lib/persistence.js
  3. 35
      test/customUtil.test.js
  4. 115
      test/persistence.test.js

17
lib/customUtils.js
Unescape View File

@ -1,4 +1,6 @@
var crypto = require('crypto');
var crypto = require('crypto')
, fs = require('fs')
;
/**
* Return a random alphanumerical string of length len
@ -16,4 +18,17 @@ function uid (len) {
}
/**
* Callback signature: err
*/
function ensureFileDoesntExist (file, callback) {
fs.exists(file, function (exists) {
if (!exists) { return callback(null); }
fs.unlink(file, function (err) { return callback(err); });
});
}
module.exports.uid = uid;
module.exports.ensureFileDoesntExist = ensureFileDoesntExist;

68
lib/persistence.js
Unescape View File

@ -10,6 +10,7 @@ var fs = require('fs')
, model = require('./model')
, async = require('async')
, mkdirp = require('mkdirp')
, customUtils = require('./customUtils')
;
@ -29,6 +30,7 @@ function Persistence (options) {
throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
} else {
this.tempFilename = this.filename + '~';
this.oldFilename = this.filename + '~~';
}
}
@ -103,36 +105,24 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
});
async.waterfall([
function (cb) {
fs.exists(self.tempFilename, function (exists) {
if (exists) { // Shouldn't happen since ensureDatafileIntegrity removes the temp datafile
fs.unlink(self.tempFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
async.apply(customUtils.ensureFileDoesntExist, self.tempFilename)
, async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
, function (cb) {
fs.exists(self.filename, function (exists) {
if (exists) {
fs.rename(self.filename, self.tempFilename, function (err) { return cb(err); });
fs.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
fs.writeFile(self.filename, toPersist, function (err) { return cb(err); });
fs.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
}
, function (cb) {
fs.exists(self.tempFilename, function (exists) {
if (exists) {
fs.unlink(self.tempFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
fs.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
}
, async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
};
@ -236,41 +226,19 @@ Persistence.treatRawData = function (rawData) {
*/
Persistence.prototype.ensureDatafileIntegrity = function (callback) {
var self = this ;
fs.exists(self.filename, function (filenameExists) {
fs.exists(self.tempFilename, function (tempFilenameExists) {
// Normal case
if (filenameExists && !tempFilenameExists) {
return callback(null);
// Write was successful
if (filenameExists) { return callback(null); }
fs.exists(self.oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}
// Process crashed right after renaming filename
if (!filenameExists && tempFilenameExists) {
return fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); });
}
// No file exists, create empty datafile
if (!filenameExists && !tempFilenameExists) {
return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}
// Process crashed after or during write to datafile
// If datafile is not empty, it means process crashed after the write so we use it
// If it is empty, we don't know whether the database was emptied or we had a crash during write. The safest option is to use the temp datafile
if (filenameExists && tempFilenameExists) {
return fs.stat(self.filename, function (err, stats) {
if (err) { return callback(err); }
if (stats.size > 0) {
fs.unlink(self.tempFilename, function (err) { return callback(err); });
} else {
fs.unlink(self.filename, function (err) {
if (err) { return callback(err); }
fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); });
});
}
});
}
// Write failed, use old version
fs.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
});
});
};

35
test/customUtil.test.js
Unescape View File

@ -0,0 +1,35 @@
var should = require('chai').should()
, assert = require('chai').assert
, customUtils = require('../lib/customUtils')
, fs = require('fs')
;
describe('customUtils', function () {
describe('ensureFileDoesntExist', function () {
it('Doesnt do anything if file already doesnt exist', function (done) {
customUtils.ensureFileDoesntExist('workspace/nonexisting', function (err) {
assert.isNull(err);
fs.existsSync('workspace/nonexisting').should.equal(false);
done();
});
});
it('Deletes file if it exists', function (done) {
fs.writeFileSync('workspace/existing', 'hello world', 'utf8');
fs.existsSync('workspace/existing').should.equal(true);
customUtils.ensureFileDoesntExist('workspace/existing', function (err) {
assert.isNull(err);
fs.existsSync('workspace/existing').should.equal(false);
done();
});
});
});
});

115
test/persistence.test.js
Unescape View File

@ -265,15 +265,16 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('');
@ -285,18 +286,18 @@ describe('Persistence', function () {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
@ -304,76 +305,60 @@ describe('Persistence', function () {
});
});
it('If only temp datafile exists, ensureDatafileIntegrity will use it', function (done) {
it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db~', 'something', 'utf8');
fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
done();
});
});
it('If both files exist and datafile is not empty, ensureDatafileIntegrity will use the datafile', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
fs.writeFileSync('workspace/it.db', 'something', 'utf8');
fs.writeFileSync('workspace/it.db~', 'other', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
p.ensureDatafileIntegrity(function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
done();
});
});
it('If both files exist and datafile is empty, ensureDatafileIntegrity will use the temp datafile', function (done) {
var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 1 of persistence failed', function (done) {
var theDb = new Datastore({ filename: 'workspace/it.db' });
if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
if (fs.existsSync('workspace/it.db~')) { fs.unlinkSync('workspace/it.db~'); }
if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
fs.writeFileSync('workspace/it.db', '', 'utf8');
fs.writeFileSync('workspace/it.db~', 'other', 'utf8');
fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
p.ensureDatafileIntegrity(function (err) {
theDb.persistence.ensureDatafileIntegrity(function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(true);
fs.readFileSync('workspace/it.db', 'utf8').should.equal('other');
fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');
done();
theDb.loadDatabase(function (err) {
assert.isNull(err);
theDb.find({}, function (err, docs) {
assert.isNull(err);
docs.length.should.equal(1);
docs[0].hello.should.equal("world");
done();
});
});
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) {
d.insert({ hello: 'world' }, function () {
@ -382,14 +367,17 @@ describe('Persistence', function () {
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
@ -399,21 +387,54 @@ describe('Persistence', function () {
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp datafile', function (done) {
it('After a persistCachedDatabase, there should be no temp or old filename', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
fs.existsSync(testDb).should.equal(false);
fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}
done();
});
});
});
});
it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
d.insert({ hello: 'world' }, function () {
d.find({}, function (err, docs) {
docs.length.should.equal(1);
if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
fs.existsSync(testDb).should.equal(false);
fs.existsSync(testDb + '~').should.equal(true);
fs.existsSync(testDb + '~~').should.equal(true);
d.persistence.persistCachedDatabase(function (err) {
var contents = fs.readFileSync(testDb, 'utf8');
assert.isNull(err);
fs.existsSync(testDb).should.equal(true);
fs.existsSync(testDb + '~').should.equal(false);
fs.existsSync(testDb + '~~').should.equal(false);
if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
throw "Datafile contents not as expected";
}

Write Preview
Loading…
Cancel
Save