TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Moved all crash safe operations to storage

Browse Source
pull/2/head
Louis Chatriot 9 years ago
parent 26441f1aae
commit d1e6d14b53
5 changed files with 343 additions and 312 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 65
      lib/persistence.js
  2. 90
      lib/storage.js
  3. 1
      package.json
  4. 29
      test/persistence.test.js

65
lib/persistence.js
Unescape View File

@ -28,13 +28,8 @@ function Persistence (options) {
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename) {
if (this.filename.charAt(this.filename.length - 1) === '~') {
throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
} else {
this.tempFilename = this.filename + '~';
this.oldFilename = this.filename + '~~';
}
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw "The datafile name can't end with a ~, which is reserved for crash safe backup files";
}
// After serialization and before deserialization hooks with some basic sanity checks
@ -65,8 +60,6 @@ function Persistence (options) {
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
this.tempFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.tempFilename);
this.oldFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.oldFilename);
}
};
@ -83,13 +76,6 @@ Persistence.ensureDirectoryExists = function (dir, cb) {
};
Persistence.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
/**
@ -148,26 +134,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
}
});
async.waterfall([
async.apply(Persistence.ensureFileDoesntExist, self.tempFilename)
, async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
, function (cb) {
storage.exists(self.filename, function (exists) {
if (exists) {
storage.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
}
, function (cb) {
storage.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
}
, async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
storage.crashSafeWriteFile(this.filename, toPersist, callback);
};
@ -279,30 +246,6 @@ Persistence.prototype.treatRawData = function (rawData) {
};
/**
* Ensure that this.filename contains the most up-to-date version of the data
* Even if a loadDatabase crashed before
*/
Persistence.prototype.ensureDatafileIntegrity = function (callback) {
var self = this ;
storage.exists(self.filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(self.oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
});
});
};
/**
* Load the database
* 1) Create all indexes
@ -326,7 +269,7 @@ Persistence.prototype.loadDatabase = function (cb) {
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
self.ensureDatafileIntegrity(function (exists) {
storage.ensureDatafileIntegrity(self.filename, function (exists) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }

90
lib/storage.js
Unescape View File

@ -4,12 +4,98 @@
* For a browser-side database it's localStorage when supported
*
* This version is the Node.js/Node Webkit version
* It's essentially fs, mkdirp and crash safe write and read functions
*/
var fs = require('fs')
, mkdirp = require('mkdirp')
, async = require('async')
, storage = {}
;
storage.exists = fs.exists;
storage.rename = fs.rename;
storage.writeFile = fs.writeFile;
storage.unlink = fs.unlink;
storage.appendFile = fs.appendFile;
storage.readFile = fs.readFile;
storage.mkdirp = mkdirp;
module.exports = fs;
module.exports.mkdirp = mkdirp;
/**
* Explicit name ...
*/
storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
/**
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
* @param {String} filename
* @param {String} data
* @param {Function} cb Optional callback, signature: err
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
async.waterfall([
async.apply(storage.ensureFileDoesntExist, tempFilename)
, async.apply(storage.ensureFileDoesntExist, oldFilename)
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.rename(filename, oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.ensureFileDoesntExist, oldFilename)
], function (err) { if (err) { return callback(err); } else { return callback(null); } })
};
/**
* Ensure the datafile contains all the data, even if there was a crash during a full file write
* @param {String} filename
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~'
, oldFilename = filename + '~~'
;
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(oldFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(oldFilename, filename, function (err) { return callback(err); });
});
});
};
// Interface
module.exports = storage;

1
package.json
Unescape View File

@ -22,6 +22,7 @@
"dependencies": {
"async": "0.2.10",
"binary-search-tree": "0.2.4",
"localforage": "^1.3.0",
"mkdirp": "~0.5.1",
"underscore": "~1.4.4"
},

29
test/persistence.test.js
Unescape View File

@ -9,6 +9,7 @@ var should = require('chai').should()
, customUtils = require('../lib/customUtils')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, storage = require('../lib/storage')
, child_process = require('child_process')
;
@ -305,7 +306,7 @@ describe('Persistence', function () {
it("Declaring only one hook will throw an exception to prevent data loss", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
fs.writeFileSync(hookTestFilename, "Some content", "utf8");
(function () {
@ -332,7 +333,7 @@ describe('Persistence', function () {
it("Declaring two hooks that are not reverse of one another will cause an exception to prevent data loss", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
fs.writeFileSync(hookTestFilename, "Some content", "utf8");
(function () {
@ -351,7 +352,7 @@ describe('Persistence', function () {
it("A serialization hook can be used to transform data before writing new state to disk", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
@ -430,7 +431,7 @@ describe('Persistence', function () {
it("Use serialization hook when persisting cached database or compacting", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
@ -492,7 +493,7 @@ describe('Persistence', function () {
it("Deserialization hook is correctly used when loading data", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
Persistence.ensureFileDoesntExist(hookTestFilename, function () {
storage.ensureFileDoesntExist(hookTestFilename, function () {
var d = new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
, beforeDeserialization: bd
@ -558,7 +559,7 @@ describe('Persistence', function () {
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
@ -581,7 +582,7 @@ describe('Persistence', function () {
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(false);
p.ensureDatafileIntegrity(function (err) {
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
@ -604,7 +605,7 @@ describe('Persistence', function () {
fs.existsSync('workspace/it.db').should.equal(false);
fs.existsSync('workspace/it.db~~').should.equal(true);
p.ensureDatafileIntegrity(function (err) {
storage.ensureDatafileIntegrity(p.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
@ -628,7 +629,7 @@ describe('Persistence', function () {
fs.existsSync('workspace/it.db').should.equal(true);
fs.existsSync('workspace/it.db~~').should.equal(true);
theDb.persistence.ensureDatafileIntegrity(function (err) {
storage.ensureDatafileIntegrity(theDb.persistence.filename, function (err) {
assert.isNull(err);
fs.existsSync('workspace/it.db').should.equal(true);
@ -761,9 +762,9 @@ describe('Persistence', function () {
var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2;
async.waterfall([
async.apply(Persistence.ensureFileDoesntExist, dbFile)
, async.apply(Persistence.ensureFileDoesntExist, dbFile + '~')
, async.apply(Persistence.ensureFileDoesntExist, dbFile + '~~')
async.apply(storage.ensureFileDoesntExist, dbFile)
, async.apply(storage.ensureFileDoesntExist, dbFile + '~')
, async.apply(storage.ensureFileDoesntExist, dbFile + '~~')
, function (cb) {
theDb = new Datastore({ filename: dbFile });
theDb.loadDatabase(cb);
@ -891,7 +892,7 @@ describe('Persistence', function () {
describe('ensureFileDoesntExist', function () {
it('Doesnt do anything if file already doesnt exist', function (done) {
Persistence.ensureFileDoesntExist('workspace/nonexisting', function (err) {
storage.ensureFileDoesntExist('workspace/nonexisting', function (err) {
assert.isNull(err);
fs.existsSync('workspace/nonexisting').should.equal(false);
done();
@ -902,7 +903,7 @@ describe('Persistence', function () {
fs.writeFileSync('workspace/existing', 'hello world', 'utf8');
fs.existsSync('workspace/existing').should.equal(true);
Persistence.ensureFileDoesntExist('workspace/existing', function (err) {
storage.ensureFileDoesntExist('workspace/existing', function (err) {
assert.isNull(err);
fs.existsSync('workspace/existing').should.equal(false);
done();

Write Preview
Loading…
Cancel
Save