Moved all crash safe operations to storage

browser-version/browser-specific/lib/storage.js

@@ -36,7 +36,7 @@ function rename (filename, newFilename, callback) {
 
 function writeFile (filename, contents, options, callback) {
   if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
-  
+
   // Options do not matter in browser setup
   if (typeof options === 'function') { callback = options; }
 
@@ -47,7 +47,7 @@ function writeFile (filename, contents, options, callback) {
 
 function appendFile (filename, toAppend, options, callback) {
   if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
-  
+
   // Options do not matter in browser setup
   if (typeof options === 'function') { callback = options; }
 
@@ -61,7 +61,7 @@ function appendFile (filename, toAppend, options, callback) {
 
 function readFile (filename, options, callback) {
   if (typeof localStorage === 'undefined') { console.log("WARNING - This browser doesn't support localStorage, no data will be saved in NeDB!"); return callback(); }
-  
+
   // Options do not matter in browser setup
   if (typeof options === 'function') { callback = options; }
 

lib/persistence.js

@@ -22,19 +22,14 @@ var storage = require('./storage')
  */
 function Persistence (options) {
   var i, j, randomString;
-  
+
   this.db = options.db;
   this.inMemoryOnly = this.db.inMemoryOnly;
   this.filename = this.db.filename;
   this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
-  
+
-  if (!this.inMemoryOnly && this.filename) {
+  if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
-    if (this.filename.charAt(this.filename.length - 1) === '~') {
+    throw "The datafile name can't end with a ~, which is reserved for crash safe backup files";
-      throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
-    } else {
-      this.tempFilename = this.filename + '~';
-      this.oldFilename = this.filename + '~~';
-    }
   }
 
   // After serialization and before deserialization hooks with some basic sanity checks
@@ -54,7 +49,7 @@ function Persistence (options) {
       }
     }
   }
-  
+
   // For NW apps, store data in the same directory where NW stores application data
   if (this.filename && options.nodeWebkitAppName) {
     console.log("==================================================================");
@@ -65,8 +60,6 @@ function Persistence (options) {
     console.log("See https://github.com/rogerwang/node-webkit/issues/500");
     console.log("==================================================================");
     this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
-    this.tempFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.tempFilename);
-    this.oldFilename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.oldFilename);
   }
 };
 
@@ -83,13 +76,6 @@ Persistence.ensureDirectoryExists = function (dir, cb) {
 };
 
 
-Persistence.ensureFileDoesntExist = function (file, callback) {
-  storage.exists(file, function (exists) {
-    if (!exists) { return callback(null); }
-    
-    storage.unlink(file, function (err) { return callback(err); });
-  });
-};
 
 
 /**
@@ -137,7 +123,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
     , self = this
     ;
 
-  if (this.inMemoryOnly) { return callback(null); } 
+  if (this.inMemoryOnly) { return callback(null); }
 
   this.db.getAllData().forEach(function (doc) {
     toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
@@ -148,26 +134,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
     }
   });
 
-  async.waterfall([
+  storage.crashSafeWriteFile(this.filename, toPersist, callback);
-    async.apply(Persistence.ensureFileDoesntExist, self.tempFilename)
-  , async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
-  , function (cb) {
-      storage.exists(self.filename, function (exists) {
-        if (exists) {
-          storage.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
-        } else {
-          return cb();
-        }
-      });  
-  }
-  , function (cb) {
-      storage.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
-    }
-  , function (cb) {
-      storage.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
-    }
-  , async.apply(Persistence.ensureFileDoesntExist, self.oldFilename)
-  ], function (err) { if (err) { return callback(err); } else { return callback(null); } })
 };
 
 
@@ -244,10 +211,10 @@ Persistence.prototype.treatRawData = function (rawData) {
     , indexes = {}
     , corruptItems = -1   // Last line of every data file is usually blank so not really corrupt
     ;
-    
+
   for (i = 0; i < data.length; i += 1) {
     var doc;
-    
+
     try {
       doc = model.deserialize(this.beforeDeserialization(data[i]));
       if (doc._id) {
@@ -265,7 +232,7 @@ Persistence.prototype.treatRawData = function (rawData) {
       corruptItems += 1;
     }
   }
-    
+
   // A bit lenient on corruption
   if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
     throw "More than 10% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss"
@@ -279,30 +246,6 @@ Persistence.prototype.treatRawData = function (rawData) {
 };
 
 
-/**
- * Ensure that this.filename contains the most up-to-date version of the data
- * Even if a loadDatabase crashed before
- */
-Persistence.prototype.ensureDatafileIntegrity = function (callback) {
-  var self = this  ;
-
-  storage.exists(self.filename, function (filenameExists) {
-    // Write was successful
-    if (filenameExists) { return callback(null); }
-  
-    storage.exists(self.oldFilename, function (oldFilenameExists) {
-      // New database
-      if (!oldFilenameExists) {
-        return storage.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });            
-      }
-    
-      // Write failed, use old version
-      storage.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
-    });
-  });
-};
-
-
 /**
  * Load the database
  * 1) Create all indexes
@@ -326,16 +269,16 @@ Persistence.prototype.loadDatabase = function (cb) {
   async.waterfall([
     function (cb) {
       Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
-        self.ensureDatafileIntegrity(function (exists) {
+        storage.ensureDatafileIntegrity(self.filename, function (exists) {
           storage.readFile(self.filename, 'utf8', function (err, rawData) {
             if (err) { return cb(err); }
-            
+
             try {
               var treatedData = self.treatRawData(rawData);
             } catch (e) {
               return cb(e);
             }
-            
+
             // Recreate all indexes in the datafile
             Object.keys(treatedData.indexes).forEach(function (key) {
               self.db.indexes[key] = new Index(treatedData.indexes[key]);

lib/storage.js

@@ -4,12 +4,98 @@
  * For a browser-side database it's localStorage when supported
  *
  * This version is the Node.js/Node Webkit version
+ * It's essentially fs, mkdirp and crash safe write and read functions
  */
 
 var fs = require('fs')
   , mkdirp = require('mkdirp')
+  , async = require('async')
+  , storage = {}
   ;
 
+storage.exists = fs.exists;
+storage.rename = fs.rename;
+storage.writeFile = fs.writeFile;
+storage.unlink = fs.unlink;
+storage.appendFile = fs.appendFile;
+storage.readFile = fs.readFile;
+storage.mkdirp = mkdirp;
 
-module.exports = fs;
+
-module.exports.mkdirp = mkdirp;
+/**
+ * Explicit name ...
+ */
+storage.ensureFileDoesntExist = function (file, callback) {
+  storage.exists(file, function (exists) {
+    if (!exists) { return callback(null); }
+
+    storage.unlink(file, function (err) { return callback(err); });
+  });
+};
+
+
+/**
+ * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
+ * @param {String} filename
+ * @param {String} data
+ * @param {Function} cb Optional callback, signature: err
+ */
+storage.crashSafeWriteFile = function (filename, data, cb) {
+  var callback = cb || function () {}
+    , tempFilename = filename + '~'
+    , oldFilename = filename + '~~'
+    ;
+
+  async.waterfall([
+    async.apply(storage.ensureFileDoesntExist, tempFilename)
+  , async.apply(storage.ensureFileDoesntExist, oldFilename)
+  , function (cb) {
+      storage.exists(filename, function (exists) {
+        if (exists) {
+          storage.rename(filename, oldFilename, function (err) { return cb(err); });
+        } else {
+          return cb();
+        }
+      });
+  }
+  , function (cb) {
+      storage.writeFile(tempFilename, data, function (err) { return cb(err); });
+    }
+  , function (cb) {
+      storage.rename(tempFilename, filename, function (err) { return cb(err); });
+    }
+  , async.apply(storage.ensureFileDoesntExist, oldFilename)
+  ], function (err) { if (err) { return callback(err); } else { return callback(null); } })
+};
+
+
+/**
+ * Ensure the datafile contains all the data, even if there was a crash during a full file write
+ * @param {String} filename
+ * @param {Function} callback signature: err
+ */
+storage.ensureDatafileIntegrity = function (filename, callback) {
+  var tempFilename = filename + '~'
+    , oldFilename = filename + '~~'
+    ;
+
+  storage.exists(filename, function (filenameExists) {
+    // Write was successful
+    if (filenameExists) { return callback(null); }
+
+    storage.exists(oldFilename, function (oldFilenameExists) {
+      // New database
+      if (!oldFilenameExists) {
+        return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
+      }
+
+      // Write failed, use old version
+      storage.rename(oldFilename, filename, function (err) { return callback(err); });
+    });
+  });
+};
+
+
+
+// Interface
+module.exports = storage;

package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "async": "0.2.10",
     "binary-search-tree": "0.2.4",
+    "localforage": "^1.3.0",
     "mkdirp": "~0.5.1",
     "underscore": "~1.4.4"
   },

test/persistence.test.js

@@ -9,8 +9,9 @@ var should = require('chai').should()
   , customUtils = require('../lib/customUtils')
   , Datastore = require('../lib/datastore')
   , Persistence = require('../lib/persistence')
+  , storage = require('../lib/storage')
   , child_process = require('child_process')
-  ;
+;
 
 
 describe('Persistence', function () {
@@ -32,22 +33,22 @@ describe('Persistence', function () {
         });
       }
     , function (cb) {
-        d.loadDatabase(function (err) {
+  d.loadDatabase(function (err) {
-          assert.isNull(err);
+    assert.isNull(err);
-          d.getAllData().length.should.equal(0);
+    d.getAllData().length.should.equal(0);
-          return cb();
+    return cb();
-        });
+  });
-      }
+    }
     ], done);
   });
 
   it('Every line represents a document', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
-                  model.serialize({ _id: "3", nested: { today: now } })
+    model.serialize({ _id: "3", nested: { today: now } })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(3);
@@ -59,10 +60,10 @@ describe('Persistence', function () {
   it('Badly formatted lines have no impact on the treated data', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  'garbage\n' +
+    'garbage\n' +
-                  model.serialize({ _id: "3", nested: { today: now } })
+    model.serialize({ _id: "3", nested: { today: now } })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
@@ -73,10 +74,10 @@ describe('Persistence', function () {
   it('Well formatted lines that have no _id are not included in the data', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
-                  model.serialize({ nested: { today: now } })
+    model.serialize({ nested: { today: now } })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
@@ -87,10 +88,10 @@ describe('Persistence', function () {
   it('If two lines concern the same doc (= same _id), the last one is the good version', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
-                  model.serialize({ _id: "1", nested: { today: now } })
+    model.serialize({ _id: "1", nested: { today: now } })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
@@ -101,11 +102,11 @@ describe('Persistence', function () {
   it('If a doc contains $$deleted: true, that means we need to remove it from the data', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
-                  model.serialize({ _id: "1", $$deleted: true }) + '\n' +
+    model.serialize({ _id: "1", $$deleted: true }) + '\n' +
-                  model.serialize({ _id: "3", today: now })
+    model.serialize({ _id: "3", today: now })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
@@ -116,29 +117,29 @@ describe('Persistence', function () {
   it('If a doc contains $$deleted: true, no error is thrown if the doc wasnt in the list before', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ _id: "2", $$deleted: true }) + '\n' +
+    model.serialize({ _id: "2", $$deleted: true }) + '\n' +
-                  model.serialize({ _id: "3", today: now })
+    model.serialize({ _id: "3", today: now })
       , treatedData = d.persistence.treatRawData(rawData).data
-      ;
+    ;
 
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
     _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
     _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true);
   });
-  
+
   it('If a doc contains $$indexCreated, no error is thrown during treatRawData and we can get the index options', function () {
     var now = new Date()
       , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
-                  model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' +
+    model.serialize({ $$indexCreated: { fieldName: "test", unique: true } }) + '\n' +
-                  model.serialize({ _id: "3", today: now })
+    model.serialize({ _id: "3", today: now })
       , treatedData = d.persistence.treatRawData(rawData).data
       , indexes = d.persistence.treatRawData(rawData).indexes
-      ;
+    ;
 
     Object.keys(indexes).length.should.equal(1);
     assert.deepEqual(indexes.test, { fieldName: "test", unique: true });
-      
+
     treatedData.sort(function (a, b) { return a._id - b._id; });
     treatedData.length.should.equal(2);
     _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
@@ -181,7 +182,7 @@ describe('Persistence', function () {
           var data = d.getAllData()
             , doc1 = _.find(data, function (doc) { return doc.a === 1; })
             , doc2 = _.find(data, function (doc) { return doc.a === 2; })
-            ;
+          ;
           assert.isNull(err);
           data.length.should.equal(2);
           doc1.a.should.equal(1);
@@ -191,7 +192,7 @@ describe('Persistence', function () {
             var data = d.getAllData()
               , doc1 = _.find(data, function (doc) { return doc.a === 1; })
               , doc2 = _.find(data, function (doc) { return doc.a === 2; })
-              ;
+            ;
             assert.isNull(err);
             data.length.should.equal(2);
             doc1.a.should.equal(1);
@@ -212,7 +213,7 @@ describe('Persistence', function () {
           var data = d.getAllData()
             , doc1 = _.find(data, function (doc) { return doc.a === 1; })
             , doc2 = _.find(data, function (doc) { return doc.a === 2; })
-            ;
+          ;
           assert.isNull(err);
           data.length.should.equal(2);
           doc1.a.should.equal(1);
@@ -240,7 +241,7 @@ describe('Persistence', function () {
           var data = d.getAllData()
             , doc1 = _.find(data, function (doc) { return doc.a === 1; })
             , doc2 = _.find(data, function (doc) { return doc.a === 2; })
-            ;
+          ;
           assert.isNull(err);
           data.length.should.equal(2);
           doc1.a.should.equal(1);
@@ -250,9 +251,9 @@ describe('Persistence', function () {
             assert.isNull(err);
             d.loadDatabase(function (err) {
               var data = d.getAllData()
-              , doc1 = _.find(data, function (doc) { return doc.a === 1; })
+                , doc1 = _.find(data, function (doc) { return doc.a === 1; })
-              , doc2 = _.find(data, function (doc) { return doc.a === 2; })
+                , doc2 = _.find(data, function (doc) { return doc.a === 2; })
-              , doc3 = _.find(data, function (doc) { return doc.a === 3; })
+                , doc3 = _.find(data, function (doc) { return doc.a === 3; })
               ;
               assert.isNull(err);
               data.length.should.equal(1);
@@ -272,7 +273,7 @@ describe('Persistence', function () {
     var corruptTestFilename = 'workspace/corruptTest.db'
       , fakeData = '{"_id":"one","hello":"world"}\n' + 'Some corrupt data\n' + '{"_id":"two","hello":"earth"}\n' + '{"_id":"three","hello":"you"}\n'
       , d
-      ;
+    ;
     fs.writeFileSync(corruptTestFilename, fakeData, "utf8");
 
     // Default corruptAlertThreshold
@@ -285,7 +286,7 @@ describe('Persistence', function () {
       d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 1 });
       d.loadDatabase(function (err) {
         assert.isNull(err);
-      
+
         fs.writeFileSync(corruptTestFilename, fakeData, "utf8");
         d = new Datastore({ filename: corruptTestFilename, corruptAlertThreshold: 0 });
         d.loadDatabase(function (err) {
@@ -295,32 +296,32 @@ describe('Persistence', function () {
           done();
         });
       });
-    });    
+    });
   });
-  
+
-  
+
   describe('Serialization hooks', function () {
     var as = function (s) { return "before_" + s + "_after"; }
-      , bd = function (s) { return s.substring(7, s.length - 6); } 
+      , bd = function (s) { return s.substring(7, s.length - 6); }
-  
+
     it("Declaring only one hook will throw an exception to prevent data loss", function (done) {
       var hookTestFilename = 'workspace/hookTest.db'
-      Persistence.ensureFileDoesntExist(hookTestFilename, function () {
+      storage.ensureFileDoesntExist(hookTestFilename, function () {
         fs.writeFileSync(hookTestFilename, "Some content", "utf8");
-      
+
         (function () {
           new Datastore({ filename: hookTestFilename, autoload: true
                         , afterSerialization: as
-                        });
+          });
         }).should.throw();
-        
+
         // Data file left untouched
         fs.readFileSync(hookTestFilename, "utf8").should.equal("Some content");
-        
+
         (function () {
           new Datastore({ filename: hookTestFilename, autoload: true
                         , beforeDeserialization: bd
-                        });
+          });
         }).should.throw();
 
         // Data file left untouched
@@ -329,17 +330,17 @@ describe('Persistence', function () {
         done();
       });
     });
-    
+
     it("Declaring two hooks that are not reverse of one another will cause an exception to prevent data loss", function (done) {
       var hookTestFilename = 'workspace/hookTest.db'
-      Persistence.ensureFileDoesntExist(hookTestFilename, function () {
+      storage.ensureFileDoesntExist(hookTestFilename, function () {
         fs.writeFileSync(hookTestFilename, "Some content", "utf8");
-      
+
         (function () {
           new Datastore({ filename: hookTestFilename, autoload: true
                         , afterSerialization: as
                         , beforeDeserialization: function (s) { return s; }
-                        });
+          });
         }).should.throw();
 
         // Data file left untouched
@@ -348,24 +349,24 @@ describe('Persistence', function () {
         done();
       });
     });
-    
+
     it("A serialization hook can be used to transform data before writing new state to disk", function (done) {
       var hookTestFilename = 'workspace/hookTest.db'
-      Persistence.ensureFileDoesntExist(hookTestFilename, function () {
+      storage.ensureFileDoesntExist(hookTestFilename, function () {
         var d = new Datastore({ filename: hookTestFilename, autoload: true
-                               , afterSerialization: as
+          , afterSerialization: as
-                               , beforeDeserialization: bd
+          , beforeDeserialization: bd
-                               })
+        })
-          ;
+        ;
-          
+
         d.insert({ hello: "world" }, function () {
           var _data = fs.readFileSync(hookTestFilename, 'utf8')
             , data = _data.split('\n')
             , doc0 = bd(data[0])
-            ;
+          ;
-          
+
           data.length.should.equal(2);
-          
+
           data[0].substring(0, 7).should.equal('before_');
           data[0].substring(data[0].length - 6).should.equal('_after');
 
@@ -378,10 +379,10 @@ describe('Persistence', function () {
               , data = _data.split('\n')
               , doc0 = bd(data[0])
               , doc1 = bd(data[1])
-              ;
+            ;
-            
+
             data.length.should.equal(3);
-            
+
             data[0].substring(0, 7).should.equal('before_');
             data[0].substring(data[0].length - 6).should.equal('_after');
             data[1].substring(0, 7).should.equal('before_');
@@ -401,10 +402,10 @@ describe('Persistence', function () {
                 , doc0 = bd(data[0])
                 , doc1 = bd(data[1])
                 , idx = bd(data[2])
-                ;
+              ;
-              
+
               data.length.should.equal(4);
-              
+
               data[0].substring(0, 7).should.equal('before_');
               data[0].substring(data[0].length - 6).should.equal('_after');
               data[1].substring(0, 7).should.equal('before_');
@@ -412,33 +413,33 @@ describe('Persistence', function () {
 
               doc0 = model.deserialize(doc0);
               Object.keys(doc0).length.should.equal(2);
-              doc0.hello.should.equal('world');        
+              doc0.hello.should.equal('world');
 
               doc1 = model.deserialize(doc1);
               Object.keys(doc1).length.should.equal(2);
               doc1.p.should.equal('Mars');
-            
+
               idx = model.deserialize(idx);
               assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } });
-              
+
               done();
-            });          
+            });
           });
         });
       });
     });
-    
+
     it("Use serialization hook when persisting cached database or compacting", function (done) {
       var hookTestFilename = 'workspace/hookTest.db'
-      Persistence.ensureFileDoesntExist(hookTestFilename, function () {
+      storage.ensureFileDoesntExist(hookTestFilename, function () {
         var d = new Datastore({ filename: hookTestFilename, autoload: true
-                               , afterSerialization: as
+          , afterSerialization: as
-                               , beforeDeserialization: bd
+          , beforeDeserialization: bd
-                               })
+        })
-          ;
+        ;
 
         d.insert({ hello: "world" }, function () {
-          d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () {      
+          d.update({ hello: "world" }, { $set: { hello: "earth" } }, {}, function () {
             d.ensureIndex({ fieldName: 'idefix' }, function () {
               var _data = fs.readFileSync(hookTestFilename, 'utf8')
                 , data = _data.split('\n')
@@ -446,21 +447,21 @@ describe('Persistence', function () {
                 , doc1 = bd(data[1])
                 , idx = bd(data[2])
                 , _id
-                ;
+              ;
-              
+
               data.length.should.equal(4);
-              
+
               doc0 = model.deserialize(doc0);
               Object.keys(doc0).length.should.equal(2);
-              doc0.hello.should.equal('world');        
+              doc0.hello.should.equal('world');
-              
+
               doc1 = model.deserialize(doc1);
               Object.keys(doc1).length.should.equal(2);
               doc1.hello.should.equal('earth');
 
               doc0._id.should.equal(doc1._id);
               _id = doc0._id;
-              
+
               idx = model.deserialize(idx);
               assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix' } });
 
@@ -469,19 +470,19 @@ describe('Persistence', function () {
                   , data = _data.split('\n')
                   , doc0 = bd(data[0])
                   , idx = bd(data[1])
-                  ;
+                ;
-                  
+
                 data.length.should.equal(3);
-                
+
                 doc0 = model.deserialize(doc0);
                 Object.keys(doc0).length.should.equal(2);
                 doc0.hello.should.equal('earth');
 
                 doc0._id.should.equal(_id);
-                
+
                 idx = model.deserialize(idx);
                 assert.deepEqual(idx, { '$$indexCreated': { fieldName: 'idefix', unique: false, sparse: false } });
-              
+
                 done();
               });
             });
@@ -489,15 +490,15 @@ describe('Persistence', function () {
         });
       });
     });
-    
+
     it("Deserialization hook is correctly used when loading data", function (done) {
       var hookTestFilename = 'workspace/hookTest.db'
-      Persistence.ensureFileDoesntExist(hookTestFilename, function () {
+      storage.ensureFileDoesntExist(hookTestFilename, function () {
         var d = new Datastore({ filename: hookTestFilename, autoload: true
-                               , afterSerialization: as
+          , afterSerialization: as
-                               , beforeDeserialization: bd
+          , beforeDeserialization: bd
-                               })
+        })
-          ;
+        ;
 
         d.insert({ hello: "world" }, function (err, doc) {
           var _id = doc._id;
@@ -507,22 +508,22 @@ describe('Persistence', function () {
                 d.ensureIndex({ fieldName: 'idefix' }, function () {
                   var _data = fs.readFileSync(hookTestFilename, 'utf8')
                     , data = _data.split('\n')
-                    ;
+                  ;
 
                   data.length.should.equal(6);
 
                   // Everything is deserialized correctly, including deletes and indexes
                   var d = new Datastore({ filename: hookTestFilename
-                                         , afterSerialization: as
+                    , afterSerialization: as
-                                         , beforeDeserialization: bd
+                    , beforeDeserialization: bd
-                                         })
+                  })
-                    ;                
+                  ;
                   d.loadDatabase(function () {
                     d.find({}, function (err, docs) {
                       docs.length.should.equal(1);
                       docs[0].hello.should.equal("earth");
                       docs[0]._id.should.equal(_id);
-                      
+
                       Object.keys(d.indexes).length.should.equal(2);
                       Object.keys(d.indexes).indexOf("idefix").should.not.equal(-1);
 
@@ -531,111 +532,111 @@ describe('Persistence', function () {
                   });
                 });
               });
-            });  
+            });
           });
         });
       });
     });
-    
+
   });   // ==== End of 'Serialization hooks' ==== //
-  
+
   describe('Prevent dataloss when persisting data', function () {
 
     it('Creating a datastore with in memory as true and a bad filename wont cause an error', function () {
       new Datastore({ filename: 'workspace/bad.db~', inMemoryOnly: true });
     })
-    
+
     it('Creating a persistent datastore with a bad filename will cause an error', function () {
       (function () { new Datastore({ filename: 'workspace/bad.db~' }); }).should.throw();
-    })  
+    })
-  
+
     it('If no file exists, ensureDatafileIntegrity creates an empty datafile', function (done) {
       var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
-    
+
       if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
       if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
-      
+
       fs.existsSync('workspace/it.db').should.equal(false);
-      fs.existsSync('workspace/it.db~~').should.equal(false);      
+      fs.existsSync('workspace/it.db~~').should.equal(false);
-      
+
-      p.ensureDatafileIntegrity(function (err) {
+      storage.ensureDatafileIntegrity(p.filename, function (err) {
         assert.isNull(err);
-        
+
         fs.existsSync('workspace/it.db').should.equal(true);
         fs.existsSync('workspace/it.db~~').should.equal(false);
-        
+
         fs.readFileSync('workspace/it.db', 'utf8').should.equal('');
-        
+
         done();
       });
     });
-  
+
     it('If only datafile exists, ensureDatafileIntegrity will use it', function (done) {
       var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
-    
+
       if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
       if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
-      
+
       fs.writeFileSync('workspace/it.db', 'something', 'utf8');
 
       fs.existsSync('workspace/it.db').should.equal(true);
-      fs.existsSync('workspace/it.db~~').should.equal(false);      
+      fs.existsSync('workspace/it.db~~').should.equal(false);
-      
+
-      p.ensureDatafileIntegrity(function (err) {
+      storage.ensureDatafileIntegrity(p.filename, function (err) {
         assert.isNull(err);
 
         fs.existsSync('workspace/it.db').should.equal(true);
         fs.existsSync('workspace/it.db~~').should.equal(false);
-        
+
         fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
-        
+
         done();
       });
     });
-    
+
     it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
       var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
-    
+
       if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
       if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
-      
+
       fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');
-      
+
       fs.existsSync('workspace/it.db').should.equal(false);
-      fs.existsSync('workspace/it.db~~').should.equal(true);      
+      fs.existsSync('workspace/it.db~~').should.equal(true);
-      
+
-      p.ensureDatafileIntegrity(function (err) {
+      storage.ensureDatafileIntegrity(p.filename, function (err) {
         assert.isNull(err);
-        
+
         fs.existsSync('workspace/it.db').should.equal(true);
         fs.existsSync('workspace/it.db~~').should.equal(false);
-        
+
         fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
-        
+
         done();
       });
     });
-    
+
     it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 4 of persistence failed', function (done) {
       var theDb = new Datastore({ filename: 'workspace/it.db' });
-    
+
       if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
       if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
-      
+
       fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
       fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');
-      
+
       fs.existsSync('workspace/it.db').should.equal(true);
-      fs.existsSync('workspace/it.db~~').should.equal(true);      
+      fs.existsSync('workspace/it.db~~').should.equal(true);
-      
+
-      theDb.persistence.ensureDatafileIntegrity(function (err) {
+      storage.ensureDatafileIntegrity(theDb.persistence.filename, function (err) {
         assert.isNull(err);
-        
+
         fs.existsSync('workspace/it.db').should.equal(true);
         fs.existsSync('workspace/it.db~~').should.equal(true);
-        
+
         fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');
-        
+
         theDb.loadDatabase(function (err) {
           assert.isNull(err);
           theDb.find({}, function (err, docs) {
@@ -647,22 +648,22 @@ describe('Persistence', function () {
         });
       });
     });
-  
+
     it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) {
       d.insert({ hello: 'world' }, function () {
         d.find({}, function (err, docs) {
           docs.length.should.equal(1);
-          
+
           if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
           if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
           if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
           fs.existsSync(testDb).should.equal(false);
-          
+
           fs.writeFileSync(testDb + '~', 'something', 'utf8');
           fs.writeFileSync(testDb + '~~', 'something else', 'utf8');
           fs.existsSync(testDb + '~').should.equal(true);
           fs.existsSync(testDb + '~~').should.equal(true);
-          
+
           d.persistence.persistCachedDatabase(function (err) {
             var contents = fs.readFileSync(testDb, 'utf8');
             assert.isNull(err);
@@ -677,22 +678,22 @@ describe('Persistence', function () {
         });
       });
     });
-    
+
     it('After a persistCachedDatabase, there should be no temp or old filename', function (done) {
       d.insert({ hello: 'world' }, function () {
         d.find({}, function (err, docs) {
           docs.length.should.equal(1);
-          
+
           if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
           if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
           if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
           fs.existsSync(testDb).should.equal(false);
-          
+
           fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
           fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
           fs.existsSync(testDb + '~').should.equal(true);
           fs.existsSync(testDb + '~~').should.equal(true);
-          
+
           d.persistence.persistCachedDatabase(function (err) {
             var contents = fs.readFileSync(testDb, 'utf8');
             assert.isNull(err);
@@ -707,19 +708,19 @@ describe('Persistence', function () {
         });
       });    
     });
-    
+
     it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
       d.insert({ hello: 'world' }, function () {
         d.find({}, function (err, docs) {
           docs.length.should.equal(1);
-          
+
           if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
           fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
           fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
           fs.existsSync(testDb).should.equal(false);
           fs.existsSync(testDb + '~').should.equal(true);
           fs.existsSync(testDb + '~~').should.equal(true);
-          
+
           d.persistence.persistCachedDatabase(function (err) {
             var contents = fs.readFileSync(testDb, 'utf8');
             assert.isNull(err);
@@ -734,22 +735,22 @@ describe('Persistence', function () {
         });
       });
     });
-    
+
     it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
       var dbFile = 'workspace/test2.db', theDb;
-    
+
       if (fs.existsSync(dbFile)) { fs.unlinkSync(dbFile); }
       if (fs.existsSync(dbFile + '~')) { fs.unlinkSync(dbFile + '~'); }
       if (fs.existsSync(dbFile + '~~')) { fs.unlinkSync(dbFile + '~~'); }
-      
+
       theDb = new Datastore({ filename: dbFile });
-      
+
       theDb.loadDatabase(function (err) {
         var contents = fs.readFileSync(dbFile, 'utf8');
         assert.isNull(err);
         fs.existsSync(dbFile).should.equal(true);
-        fs.existsSync(dbFile + '~').should.equal(false);            
+        fs.existsSync(dbFile + '~').should.equal(false);
-        fs.existsSync(dbFile + '~~').should.equal(false);            
+        fs.existsSync(dbFile + '~~').should.equal(false);
         if (contents != "") {
           throw "Datafile contents not as expected";
         }
@@ -759,90 +760,90 @@ describe('Persistence', function () {
 
     it('Persistence works as expected when everything goes fine', function (done) {
       var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2;
-      
+
       async.waterfall([
-        async.apply(Persistence.ensureFileDoesntExist, dbFile)
+        async.apply(storage.ensureFileDoesntExist, dbFile)
-      , async.apply(Persistence.ensureFileDoesntExist, dbFile + '~')
+        , async.apply(storage.ensureFileDoesntExist, dbFile + '~')
-      , async.apply(Persistence.ensureFileDoesntExist, dbFile + '~~')
+        , async.apply(storage.ensureFileDoesntExist, dbFile + '~~')
-      , function (cb) {
+        , function (cb) {
-        theDb = new Datastore({ filename: dbFile });
+      theDb = new Datastore({ filename: dbFile });
-        theDb.loadDatabase(cb);
+      theDb.loadDatabase(cb);
-      }
+        }
-      , function (cb) {
+        , function (cb) {
-        theDb.find({}, function (err, docs) {
+      theDb.find({}, function (err, docs) {
-          assert.isNull(err);
+        assert.isNull(err);
-          docs.length.should.equal(0);
+        docs.length.should.equal(0);
-          return cb();
+        return cb();
-        });
+      });
-      }
+        }
       , function (cb) {
-        theDb.insert({ a: 'hello' }, function (err, _doc1) {
+    theDb.insert({ a: 'hello' }, function (err, _doc1) {
-          assert.isNull(err);
+      assert.isNull(err);
-          doc1 = _doc1;
+      doc1 = _doc1;
-          theDb.insert({ a: 'world' }, function (err, _doc2) {
+      theDb.insert({ a: 'world' }, function (err, _doc2) {
-            assert.isNull(err);
+        assert.isNull(err);
-            doc2 = _doc2;
+        doc2 = _doc2;
-            return cb();
+        return cb();
-          });
+      });
-        });
+    });
       }
       , function (cb) {
-        theDb.find({}, function (err, docs) {
+    theDb.find({}, function (err, docs) {
-          assert.isNull(err);
+      assert.isNull(err);
-          docs.length.should.equal(2);
+      docs.length.should.equal(2);
-          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+      _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
-          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+      _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
-          return cb();
+      return cb();
-        });
+    });
       }
       , function (cb) {
-        theDb.loadDatabase(cb);
+    theDb.loadDatabase(cb);
       }
       , function (cb) {   // No change
-        theDb.find({}, function (err, docs) {
+    theDb.find({}, function (err, docs) {
-          assert.isNull(err);
+      assert.isNull(err);
-          docs.length.should.equal(2);
+      docs.length.should.equal(2);
-          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+      _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
-          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+      _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
-          return cb();
+      return cb();
-        });
+    });
       }
       , function (cb) {
-        fs.existsSync(dbFile).should.equal(true);
+    fs.existsSync(dbFile).should.equal(true);
-        fs.existsSync(dbFile + '~').should.equal(false);
+    fs.existsSync(dbFile + '~').should.equal(false);
-        fs.existsSync(dbFile + '~~').should.equal(false);
+    fs.existsSync(dbFile + '~~').should.equal(false);
-        return cb();
+    return cb();
       }
       , function (cb) {
-        theDb2 = new Datastore({ filename: dbFile });
+    theDb2 = new Datastore({ filename: dbFile });
-        theDb2.loadDatabase(cb);
+    theDb2.loadDatabase(cb);
       }  
       , function (cb) {   // No change in second db
-        theDb2.find({}, function (err, docs) {
+    theDb2.find({}, function (err, docs) {
-          assert.isNull(err);
+      assert.isNull(err);
-          docs.length.should.equal(2);
+      docs.length.should.equal(2);
-          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+      _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
-          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+      _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
-          return cb();
+      return cb();
-        });
+    });
       }
       , function (cb) {
-        fs.existsSync(dbFile).should.equal(true);
+    fs.existsSync(dbFile).should.equal(true);
-        fs.existsSync(dbFile + '~').should.equal(false);
+    fs.existsSync(dbFile + '~').should.equal(false);
-        fs.existsSync(dbFile + '~~').should.equal(false);
+    fs.existsSync(dbFile + '~~').should.equal(false);
-        return cb();
+    return cb();
       } 
       ], done);
     });
-    
+
-  
+
     // This test is a bit complicated since it depends on the time I/O actions take to execute
     // That depends on the machine and the load on the machine when the tests are run
     // It is timed for my machine with nothing else running but may not work as expected on others (it will not fail but may not be a proof)
     // Every new version of NeDB passes it on my machine before rtelease
     it('If system crashes during a loadDatabase, the former version is not lost', function (done) {
       var cp, N = 150000, toWrite = "", i;
-      
+
       // Ensuring the state is clean
       if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); }
       if (fs.existsSync('workspace/lac.db~')) { fs.unlinkSync('workspace/lac.db~'); }
@@ -852,46 +853,46 @@ describe('Persistence', function () {
         toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n';
       }        
       fs.writeFileSync('workspace/lac.db', toWrite, 'utf8');
-      
+
       // Loading it in a separate process that we will crash before finishing the loadDatabase
       cp = child_process.fork('test_lac/loadAndCrash.test')
-      
+
       // Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile)
       setTimeout(function() {
         cp.kill('SIGINT');
-        
+
         // If the timing is correct, only the temp datafile contains data
         // The datafile was in the middle of being written and is empty
-        
+
         // Let the process crash be finished then load database without a crash, and test we didn't lose data
         setTimeout(function () {
           var db = new Datastore({ filename: 'workspace/lac.db' });
           db.loadDatabase(function (err) {
             assert.isNull(err);
-            
+
             db.count({}, function (err, n) {
               // Data has not been lost
               assert.isNull(err);
               n.should.equal(150000);
-              
+
               // State is clean, the temp datafile has been erased and the datafile contains all the data
               fs.existsSync('workspace/lac.db').should.equal(true);
               fs.existsSync('workspace/lac.db~').should.equal(false);
-              
+
               done();
             });
           });
-        }, 100);        
+        }, 100);
       }, 2000);
     });
-  
+
   });   // ==== End of 'Prevent dataloss when persisting data' ====
 
 
   describe('ensureFileDoesntExist', function () {
-  
+
     it('Doesnt do anything if file already doesnt exist', function (done) {
-      Persistence.ensureFileDoesntExist('workspace/nonexisting', function (err) {
+      storage.ensureFileDoesntExist('workspace/nonexisting', function (err) {
         assert.isNull(err);
         fs.existsSync('workspace/nonexisting').should.equal(false);
         done();
@@ -901,14 +902,14 @@ describe('Persistence', function () {
     it('Deletes file if it exists', function (done) {
       fs.writeFileSync('workspace/existing', 'hello world', 'utf8');
       fs.existsSync('workspace/existing').should.equal(true);
-    
+
-      Persistence.ensureFileDoesntExist('workspace/existing', function (err) {
+      storage.ensureFileDoesntExist('workspace/existing', function (err) {
         assert.isNull(err);
         fs.existsSync('workspace/existing').should.equal(false);
         done();
       });
     });
-    
+
   });   // ==== End of 'ensureFileDoesntExist' ====