mirror of https://github.com/seald/nedb
Timothée Rebours
3 years ago
parent
8b776b5c45
commit
dbcc3647d1
@ -1,133 +1,59 @@ |
||||
/* eslint-env mocha */ |
||||
/* global DEBUG */ |
||||
/** |
||||
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes |
||||
*/ |
||||
const fs = require('fs') |
||||
const { Writable } = require('stream') |
||||
const { callbackify } = require('util') |
||||
|
||||
function rethrow () { |
||||
// Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
|
||||
// is fairly slow to generate.
|
||||
if (DEBUG) { |
||||
const backtrace = new Error() |
||||
return function (err) { |
||||
if (err) { |
||||
backtrace.stack = err.name + ': ' + err.message + |
||||
backtrace.stack.substr(backtrace.name.length) |
||||
throw backtrace |
||||
} |
||||
} |
||||
} |
||||
|
||||
return function (err) { |
||||
if (err) { |
||||
throw err // Forgot a callback but don't know where? Use NODE_DEBUG=fs
|
||||
} |
||||
} |
||||
} |
||||
|
||||
function maybeCallback (cb) { |
||||
return typeof cb === 'function' ? cb : rethrow() |
||||
} |
||||
|
||||
function isFd (path) { |
||||
return (path >>> 0) === path |
||||
} |
||||
|
||||
function assertEncoding (encoding) { |
||||
if (encoding && !Buffer.isEncoding(encoding)) { |
||||
throw new Error('Unknown encoding: ' + encoding) |
||||
} |
||||
} |
||||
|
||||
let onePassDone = false |
||||
fs.promises.writeFile = async function (path, data) { |
||||
let onePassDone = false |
||||
const options = { encoding: 'utf8', mode: 0o666, flag: 'w' } // we don't care about the actual options passed
|
||||
|
||||
function writeAll (fd, isUserFd, buffer, offset, length, position, callback_) { |
||||
const callback = maybeCallback(arguments[arguments.length - 1]) |
||||
const filehandle = await fs.promises.open(path, options.flag, options.mode) |
||||
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8') |
||||
let length = buffer.length |
||||
let offset = 0 |
||||
|
||||
try { |
||||
while (length > 0) { |
||||
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
|
||||
const l = Math.min(5000, length) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
|
||||
|
||||
// write(fd, buffer, offset, length, position, callback)
|
||||
fs.write(fd, buffer, offset, l, position, function (writeErr, written) { |
||||
if (writeErr) { |
||||
if (isUserFd) { |
||||
if (callback) callback(writeErr) |
||||
} else { |
||||
fs.close(fd, function () { |
||||
if (callback) callback(writeErr) |
||||
}) |
||||
} |
||||
} else { |
||||
const { bytesWritten } = await filehandle.write(buffer, offset, Math.min(5000, length)) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
|
||||
onePassDone = true |
||||
if (written === length) { |
||||
if (isUserFd) { |
||||
if (callback) callback(null) |
||||
} else { |
||||
fs.close(fd, callback) |
||||
} |
||||
} else { |
||||
offset += written |
||||
length -= written |
||||
if (position !== null) { |
||||
position += written |
||||
offset += bytesWritten |
||||
length -= bytesWritten |
||||
} |
||||
writeAll(fd, isUserFd, buffer, offset, length, position, callback) |
||||
} finally { |
||||
await filehandle.close() |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
|
||||
fs.writeFile = function (path, data, options, callback_) { |
||||
const callback = maybeCallback(arguments[arguments.length - 1]) |
||||
|
||||
if (!options || typeof options === 'function') { |
||||
options = { encoding: 'utf8', mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
|
||||
} else if (typeof options === 'string') { |
||||
options = { encoding: options, mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
|
||||
} else if (typeof options !== 'object') { |
||||
throw new Error(`throwOptionsError${options}`) |
||||
class FakeFsWriteStream extends Writable { |
||||
constructor (filename) { |
||||
super() |
||||
this.filename = filename |
||||
this._content = Buffer.alloc(0) |
||||
} |
||||
|
||||
assertEncoding(options.encoding) |
||||
|
||||
const flag = options.flag || 'w' |
||||
|
||||
if (isFd(path)) { |
||||
writeFd(path, true) |
||||
return |
||||
_write (chunk, encoding, callback) { |
||||
this._content = Buffer.concat([this._content, Buffer.from(chunk, encoding)]) |
||||
callback() |
||||
} |
||||
|
||||
fs.open(path, flag, options.mode, function (openErr, fd) { |
||||
if (openErr) { |
||||
if (callback) callback(openErr) |
||||
} else { |
||||
writeFd(fd, false) |
||||
_end (chunk, encoding, callback) { |
||||
this._content = Buffer.concat([this._content, Buffer.from(chunk, encoding)]) |
||||
callback() |
||||
} |
||||
}) |
||||
|
||||
function writeFd (fd, isUserFd) { |
||||
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8') |
||||
const position = /a/.test(flag) ? null : 0 |
||||
|
||||
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback) |
||||
} |
||||
} |
||||
|
||||
fs.createWriteStream = function (path) { |
||||
let content = '' |
||||
return { |
||||
write (data) { |
||||
content += data |
||||
}, |
||||
close (callback) { |
||||
fs.writeFile(path, content, callback) |
||||
} |
||||
callbackify(fs.promises.writeFile)(this.filename, this._content, 'utf8', callback) |
||||
} |
||||
} |
||||
|
||||
// End of fs modification
|
||||
fs.createWriteStream = path => new FakeFsWriteStream(path) |
||||
|
||||
// End of fs monkey patching
|
||||
const Nedb = require('../lib/datastore.js') |
||||
const db = new Nedb({ filename: 'workspace/lac.db' }) |
||||
|
||||
db.loadDatabase() |
||||
db.loadDatabaseAsync() // no need to await
|
||||
|
||||
@ -1,65 +1,61 @@ |
||||
const fs = require('fs') |
||||
const { waterfall, whilst } = require('../test/utils.test.js') |
||||
const fsPromises = fs.promises |
||||
const Nedb = require('../lib/datastore') |
||||
const { callbackify } = require('util') |
||||
const db = new Nedb({ filename: './workspace/openfds.db', autoload: true }) |
||||
const N = 64 |
||||
let i |
||||
let fds |
||||
|
||||
function multipleOpen (filename, N, callback) { |
||||
whilst(function () { return i < N } |
||||
, function (cb) { |
||||
fs.open(filename, 'r', function (err, fd) { |
||||
i += 1 |
||||
if (fd) { fds.push(fd) } |
||||
return cb(err) |
||||
}) |
||||
} |
||||
, callback) |
||||
} |
||||
// A console.error triggers an error of the parent test
|
||||
|
||||
waterfall([ |
||||
// Check that ulimit has been set to the correct value
|
||||
function (cb) { |
||||
i = 0 |
||||
fds = [] |
||||
multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) { |
||||
if (!err) { console.log('No error occured while opening a file too many times') } |
||||
fds.forEach(function (fd) { fs.closeSync(fd) }) |
||||
return cb() |
||||
}) |
||||
}, |
||||
function (cb) { |
||||
i = 0 |
||||
fds = [] |
||||
multipleOpen('./test_lac/openFdsTestFile2', N, function (err) { |
||||
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err) } |
||||
fds.forEach(function (fd) { fs.closeSync(fd) }) |
||||
return cb() |
||||
}) |
||||
}, |
||||
// Then actually test NeDB persistence
|
||||
function () { |
||||
db.remove({}, { multi: true }, function (err) { |
||||
if (err) { console.log(err) } |
||||
db.insert({ hello: 'world' }, function (err) { |
||||
if (err) { console.log(err) } |
||||
const test = async () => { |
||||
let filehandles = [] |
||||
try { |
||||
for (let i = 0; i < 2 * N + 1; i++) { |
||||
const filehandle = await fsPromises.open('./test_lac/openFdsTestFile', 'r') |
||||
filehandles.push(filehandle) |
||||
} |
||||
console.error('No error occurred while opening a file too many times') |
||||
process.exit(1) |
||||
} catch (error) { |
||||
if (error.code !== 'EMFILE') { |
||||
console.error(error) |
||||
process.exit(1) |
||||
} |
||||
} finally { |
||||
for (const filehandle of filehandles) { |
||||
await filehandle.close() |
||||
} |
||||
filehandles = [] |
||||
} |
||||
|
||||
i = 0 |
||||
whilst(function () { return i < 2 * N + 1 } |
||||
, function (cb) { |
||||
callbackify(() => db.persistence.persistCachedDatabaseAsync())(function (err) { |
||||
if (err) { return cb(err) } |
||||
i += 1 |
||||
return cb() |
||||
}) |
||||
try { |
||||
for (let i = 0; i < N; i++) { |
||||
const filehandle = await fsPromises.open('./test_lac/openFdsTestFile2', 'r') |
||||
filehandles.push(filehandle) |
||||
} |
||||
, function (err) { |
||||
if (err) { console.log('Got unexpected error during one peresistence operation: ' + err) } |
||||
} catch (error) { |
||||
console.error(`An unexpected error occurred when opening file not too many times at i: ${i} with error: ${error}`) |
||||
process.exit(1) |
||||
} finally { |
||||
for (const filehandle of filehandles) { |
||||
await filehandle.close() |
||||
} |
||||
) |
||||
}) |
||||
}) |
||||
} |
||||
], () => {}) |
||||
|
||||
try { |
||||
const db = new Nedb({ filename: './workspace/openfds.db' }) |
||||
await db.loadDatabaseAsync() |
||||
await db.removeAsync({}, { multi: true }) |
||||
await db.insertAsync({ hello: 'world' }) |
||||
|
||||
for (let i = 0; i < 2 * N + 1; i++) { |
||||
await db.persistence.persistCachedDatabaseAsync() |
||||
} |
||||
} catch (error) { |
||||
console.error(`Got unexpected error during one persistence operation at ${i}: with error: ${error}`) |
||||
} |
||||
} |
||||
try { |
||||
test() |
||||
} catch (error) { |
||||
console.error(error) |
||||
process.exit(1) |
||||
} |
||||
|
||||
Loading…
Reference in new issue