From f6b5910eb3145cdcaf79d4908dd24c177cf9be21 Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Mon, 23 Nov 2015 17:59:15 +0100 Subject: [PATCH] Correct and fast file write crash test --- test/persistence.test.js | 59 +++++++++-------- test_lac/loadAndCrash.test.js | 120 +++++++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 30 deletions(-) diff --git a/test/persistence.test.js b/test/persistence.test.js index 5c3c167..f2e58cb 100755 --- a/test/persistence.test.js +++ b/test/persistence.test.js @@ -390,7 +390,7 @@ describe('Persistence', function () { doc0 = model.deserialize(doc0); Object.keys(doc0).length.should.equal(2); - doc0.hello.should.equal('world'); + doc0.hello.should.equal('world'); doc1 = model.deserialize(doc1); Object.keys(doc1).length.should.equal(2); @@ -824,10 +824,10 @@ describe('Persistence', function () { ], done); }); - // This test needs to be rewritten. The goal is to check what happens when the system crashes during a writeFile, so this test - // must rewrite a custom and buggy writeFile that will be used by the child process and crash in the midst of writing the file + // The child process will load the database with the given datafile, but the fs.writeFile function + // is rewritten to crash the process before it finished (after 5000 bytes), to ensure data was not lost it('If system crashes during a loadDatabase, the former version is not lost', function (done) { - var cp, N = 150000, toWrite = "", i; + var N = 500, toWrite = "", i, doc_i; // Ensuring the state is clean if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); } @@ -835,40 +835,41 @@ describe('Persistence', function () { // Creating a db file with 150k records (a bit long to load) for (i = 0; i < N; i += 1) { - toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n'; + toWrite += model.serialize({ _id: 'anid_' + i, hello: 'world' }) + '\n'; } fs.writeFileSync('workspace/lac.db', toWrite, 'utf8'); - // Loading it in a separate process that we will crash before finishing the loadDatabase - cp = child_process.fork('test_lac/loadAndCrash.test') - - // Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile) - setTimeout(function() { - cp.kill('SIGINT'); + var datafileLength = fs.readFileSync('workspace/lac.db', 'utf8').length; - // If the timing is correct, only the temp datafile contains data - // The datafile was in the middle of being written and is empty + // Loading it in a separate process that we will crash before finishing the loadDatabase + child_process.fork('test_lac/loadAndCrash.test').on('exit', function (code) { + code.should.equal(1); // See test_lac/loadAndCrash.test.js - // Let the process crash be finished then load database without a crash, and test we didn't lose data - setTimeout(function () { - var db = new Datastore({ filename: 'workspace/lac.db' }); - db.loadDatabase(function (err) { - assert.isNull(err); + fs.existsSync('workspace/lac.db').should.equal(true); + fs.existsSync('workspace/lac.db~').should.equal(true); + fs.readFileSync('workspace/lac.db', 'utf8').length.should.equal(datafileLength); + fs.readFileSync('workspace/lac.db~', 'utf8').length.should.equal(5000); - db.count({}, function (err, n) { - // Data has not been lost - assert.isNull(err); - n.should.equal(150000); + // Reload database without a crash, check that no data was lost and fs state is clean (no temp file) + var db = new Datastore({ filename: 'workspace/lac.db' }); + db.loadDatabase(function (err) { + assert.isNull(err); - // State is clean, the temp datafile has been erased and the datafile contains all the data - fs.existsSync('workspace/lac.db').should.equal(true); - fs.existsSync('workspace/lac.db~').should.equal(false); + fs.existsSync('workspace/lac.db').should.equal(true); + fs.existsSync('workspace/lac.db~').should.equal(false); + fs.readFileSync('workspace/lac.db', 'utf8').length.should.equal(datafileLength); - done(); - }); + db.find({}, function (err, docs) { + docs.length.should.equal(N); + for (i = 0; i < N; i += 1) { + doc_i = _.find(docs, function (d) { return d._id === 'anid_' + i; }); + assert.isDefined(doc_i); + assert.deepEqual({ hello: 'world', _id: 'anid_' + i }, doc_i); + } + return done(); }); - }, 100); - }, 2000); + }); + }); }); }); // ==== End of 'Prevent dataloss when persisting data' ==== diff --git a/test_lac/loadAndCrash.test.js b/test_lac/loadAndCrash.test.js index 88eb08f..75de20b 100755 --- a/test_lac/loadAndCrash.test.js +++ b/test_lac/loadAndCrash.test.js @@ -1,5 +1,123 @@ +/** + * Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes + */ +var fs = require('fs'); + +function rethrow() { + // Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and + // is fairly slow to generate. + if (DEBUG) { + var backtrace = new Error(); + return function(err) { + if (err) { + backtrace.stack = err.name + ': ' + err.message + + backtrace.stack.substr(backtrace.name.length); + throw backtrace; + } + }; + } + + return function(err) { + if (err) { + throw err; // Forgot a callback but don't know where? Use NODE_DEBUG=fs + } + }; +} + +function maybeCallback(cb) { + return typeof cb === 'function' ? cb : rethrow(); +} + +function isFd(path) { + return (path >>> 0) === path; +} + +function assertEncoding(encoding) { + if (encoding && !Buffer.isEncoding(encoding)) { + throw new Error('Unknown encoding: ' + encoding); + } +} + +var onePassDone = false; +function writeAll(fd, isUserFd, buffer, offset, length, position, callback_) { + var callback = maybeCallback(arguments[arguments.length - 1]); + + if (onePassDone) { process.exit(1); } // Crash on purpose before rewrite done + var l = Math.min(5000, length); // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash + + // write(fd, buffer, offset, length, position, callback) + fs.write(fd, buffer, offset, l, position, function(writeErr, written) { + if (writeErr) { + if (isUserFd) { + if (callback) callback(writeErr); + } else { + fs.close(fd, function() { + if (callback) callback(writeErr); + }); + } + } else { + onePassDone = true; + if (written === length) { + if (isUserFd) { + if (callback) callback(null); + } else { + fs.close(fd, callback); + } + } else { + offset += written; + length -= written; + if (position !== null) { + position += written; + } + writeAll(fd, isUserFd, buffer, offset, length, position, callback); + } + } + }); +} + +fs.writeFile = function(path, data, options, callback_) { + var callback = maybeCallback(arguments[arguments.length - 1]); + + if (!options || typeof options === 'function') { + options = { encoding: 'utf8', mode: 0o666, flag: 'w' }; + } else if (typeof options === 'string') { + options = { encoding: options, mode: 0o666, flag: 'w' }; + } else if (typeof options !== 'object') { + throwOptionsError(options); + } + + assertEncoding(options.encoding); + + var flag = options.flag || 'w'; + + if (isFd(path)) { + writeFd(path, true); + return; + } + + fs.open(path, flag, options.mode, function(openErr, fd) { + if (openErr) { + if (callback) callback(openErr); + } else { + writeFd(fd, false); + } + }); + + function writeFd(fd, isUserFd) { + var buffer = (data instanceof Buffer) ? data : new Buffer('' + data, + options.encoding || 'utf8'); + var position = /a/.test(flag) ? null : 0; + + writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback); + } +}; + + + + +// End of fs modification var Nedb = require('../lib/datastore.js') , db = new Nedb({ filename: 'workspace/lac.db' }) ; -db.loadDatabase(); \ No newline at end of file +db.loadDatabase();