Correct and fast file write crash test

test/persistence.test.js

@@ -824,10 +824,10 @@ describe('Persistence', function () {
       ], done);
     });
 
-    // This test needs to be rewritten. The goal is to check what happens when the system crashes during a writeFile, so this test
+    // The child process will load the database with the given datafile, but the fs.writeFile function
-    // must rewrite a custom and buggy writeFile that will be used by the child process and crash in the midst of writing the file
+    // is rewritten to crash the process before it finished (after 5000 bytes), to ensure data was not lost
     it('If system crashes during a loadDatabase, the former version is not lost', function (done) {
-      var cp, N = 150000, toWrite = "", i;
+      var N = 500, toWrite = "", i, doc_i;
 
       // Ensuring the state is clean
       if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); }
@@ -835,40 +835,41 @@ describe('Persistence', function () {
 
       // Creating a db file with 150k records (a bit long to load)
       for (i = 0; i < N; i += 1) {
-        toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n';
+        toWrite += model.serialize({ _id: 'anid_' + i, hello: 'world' }) + '\n';
       }
       fs.writeFileSync('workspace/lac.db', toWrite, 'utf8');
 
-      // Loading it in a separate process that we will crash before finishing the loadDatabase
+      var datafileLength = fs.readFileSync('workspace/lac.db', 'utf8').length;
-      cp = child_process.fork('test_lac/loadAndCrash.test')
-
-      // Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile)
-      setTimeout(function() {
-        cp.kill('SIGINT');
 
-        // If the timing is correct, only the temp datafile contains data
+      // Loading it in a separate process that we will crash before finishing the loadDatabase
-        // The datafile was in the middle of being written and is empty
+      child_process.fork('test_lac/loadAndCrash.test').on('exit', function (code) {
+        code.should.equal(1);   // See test_lac/loadAndCrash.test.js
 
-        // Let the process crash be finished then load database without a crash, and test we didn't lose data
+        fs.existsSync('workspace/lac.db').should.equal(true);
-        setTimeout(function () {
+        fs.existsSync('workspace/lac.db~').should.equal(true);
-          var db = new Datastore({ filename: 'workspace/lac.db' });
+        fs.readFileSync('workspace/lac.db', 'utf8').length.should.equal(datafileLength);
-          db.loadDatabase(function (err) {
+        fs.readFileSync('workspace/lac.db~', 'utf8').length.should.equal(5000);
-            assert.isNull(err);
 
-            db.count({}, function (err, n) {
+        // Reload database without a crash, check that no data was lost and fs state is clean (no temp file)
-              // Data has not been lost
+        var db = new Datastore({ filename: 'workspace/lac.db' });
-              assert.isNull(err);
+        db.loadDatabase(function (err) {
-              n.should.equal(150000);
+          assert.isNull(err);
 
-              // State is clean, the temp datafile has been erased and the datafile contains all the data
+          fs.existsSync('workspace/lac.db').should.equal(true);
-              fs.existsSync('workspace/lac.db').should.equal(true);
+          fs.existsSync('workspace/lac.db~').should.equal(false);
-              fs.existsSync('workspace/lac.db~').should.equal(false);
+          fs.readFileSync('workspace/lac.db', 'utf8').length.should.equal(datafileLength);
 
-              done();
+          db.find({}, function (err, docs) {
-            });
+            docs.length.should.equal(N);
+            for (i = 0; i < N; i += 1) {
+              doc_i = _.find(docs, function (d) { return d._id === 'anid_' + i; });
+              assert.isDefined(doc_i);
+              assert.deepEqual({ hello: 'world', _id: 'anid_' + i }, doc_i);
+            }
+            return done();
           });
-        }, 100);
+        });
-      }, 2000);
+      });
     });
 
   });   // ==== End of 'Prevent dataloss when persisting data' ====

test_lac/loadAndCrash.test.js

@@ -1,3 +1,121 @@
+/**
+ * Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes
+ */
+var fs = require('fs');
+
+function rethrow() {
+  // Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
+  // is fairly slow to generate.
+  if (DEBUG) {
+    var backtrace = new Error();
+    return function(err) {
+      if (err) {
+        backtrace.stack = err.name + ': ' + err.message +
+                          backtrace.stack.substr(backtrace.name.length);
+        throw backtrace;
+      }
+    };
+  }
+
+  return function(err) {
+    if (err) {
+      throw err;  // Forgot a callback but don't know where? Use NODE_DEBUG=fs
+    }
+  };
+}
+
+function maybeCallback(cb) {
+  return typeof cb === 'function' ? cb : rethrow();
+}
+
+function isFd(path) {
+  return (path >>> 0) === path;
+}
+
+function assertEncoding(encoding) {
+  if (encoding && !Buffer.isEncoding(encoding)) {
+    throw new Error('Unknown encoding: ' + encoding);
+  }
+}
+
+var onePassDone = false;
+function writeAll(fd, isUserFd, buffer, offset, length, position, callback_) {
+  var callback = maybeCallback(arguments[arguments.length - 1]);
+
+  if (onePassDone) { process.exit(1); }   // Crash on purpose before rewrite done
+  var l = Math.min(5000, length);   // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
+
+  // write(fd, buffer, offset, length, position, callback)
+  fs.write(fd, buffer, offset, l, position, function(writeErr, written) {
+    if (writeErr) {
+      if (isUserFd) {
+        if (callback) callback(writeErr);
+      } else {
+        fs.close(fd, function() {
+          if (callback) callback(writeErr);
+        });
+      }
+    } else {
+      onePassDone = true;
+      if (written === length) {
+        if (isUserFd) {
+          if (callback) callback(null);
+        } else {
+          fs.close(fd, callback);
+        }
+      } else {
+        offset += written;
+        length -= written;
+        if (position !== null) {
+          position += written;
+        }
+        writeAll(fd, isUserFd, buffer, offset, length, position, callback);
+      }
+    }
+  });
+}
+
+fs.writeFile = function(path, data, options, callback_) {
+  var callback = maybeCallback(arguments[arguments.length - 1]);
+
+  if (!options || typeof options === 'function') {
+    options = { encoding: 'utf8', mode: 0o666, flag: 'w' };
+  } else if (typeof options === 'string') {
+    options = { encoding: options, mode: 0o666, flag: 'w' };
+  } else if (typeof options !== 'object') {
+    throwOptionsError(options);
+  }
+
+  assertEncoding(options.encoding);
+
+  var flag = options.flag || 'w';
+
+  if (isFd(path)) {
+    writeFd(path, true);
+    return;
+  }
+
+  fs.open(path, flag, options.mode, function(openErr, fd) {
+    if (openErr) {
+      if (callback) callback(openErr);
+    } else {
+      writeFd(fd, false);
+    }
+  });
+
+  function writeFd(fd, isUserFd) {
+    var buffer = (data instanceof Buffer) ? data : new Buffer('' + data,
+        options.encoding || 'utf8');
+    var position = /a/.test(flag) ? null : 0;
+
+    writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback);
+  }
+};
+
+
+
+
+// End of fs modification
 var Nedb = require('../lib/datastore.js')
   , db = new Nedb({ filename: 'workspace/lac.db' })
   ;