mirror of https://github.com/seald/nedb
Timothée Rebours
3 years ago
parent
8b776b5c45
commit
dbcc3647d1
@ -1,133 +1,59 @@ |
|||||||
/* eslint-env mocha */ |
/* eslint-env mocha */ |
||||||
/* global DEBUG */ |
|
||||||
/** |
/** |
||||||
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes |
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes |
||||||
*/ |
*/ |
||||||
const fs = require('fs') |
const fs = require('fs') |
||||||
|
const { Writable } = require('stream') |
||||||
|
const { callbackify } = require('util') |
||||||
|
|
||||||
function rethrow () { |
fs.promises.writeFile = async function (path, data) { |
||||||
// Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
|
let onePassDone = false |
||||||
// is fairly slow to generate.
|
const options = { encoding: 'utf8', mode: 0o666, flag: 'w' } // we don't care about the actual options passed
|
||||||
if (DEBUG) { |
|
||||||
const backtrace = new Error() |
|
||||||
return function (err) { |
|
||||||
if (err) { |
|
||||||
backtrace.stack = err.name + ': ' + err.message + |
|
||||||
backtrace.stack.substr(backtrace.name.length) |
|
||||||
throw backtrace |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return function (err) { |
|
||||||
if (err) { |
|
||||||
throw err // Forgot a callback but don't know where? Use NODE_DEBUG=fs
|
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
function maybeCallback (cb) { |
|
||||||
return typeof cb === 'function' ? cb : rethrow() |
|
||||||
} |
|
||||||
|
|
||||||
function isFd (path) { |
|
||||||
return (path >>> 0) === path |
|
||||||
} |
|
||||||
|
|
||||||
function assertEncoding (encoding) { |
|
||||||
if (encoding && !Buffer.isEncoding(encoding)) { |
|
||||||
throw new Error('Unknown encoding: ' + encoding) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
let onePassDone = false |
|
||||||
|
|
||||||
function writeAll (fd, isUserFd, buffer, offset, length, position, callback_) { |
const filehandle = await fs.promises.open(path, options.flag, options.mode) |
||||||
const callback = maybeCallback(arguments[arguments.length - 1]) |
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8') |
||||||
|
let length = buffer.length |
||||||
|
let offset = 0 |
||||||
|
|
||||||
|
try { |
||||||
|
while (length > 0) { |
||||||
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
|
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
|
||||||
const l = Math.min(5000, length) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
|
const { bytesWritten } = await filehandle.write(buffer, offset, Math.min(5000, length)) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
|
||||||
|
|
||||||
// write(fd, buffer, offset, length, position, callback)
|
|
||||||
fs.write(fd, buffer, offset, l, position, function (writeErr, written) { |
|
||||||
if (writeErr) { |
|
||||||
if (isUserFd) { |
|
||||||
if (callback) callback(writeErr) |
|
||||||
} else { |
|
||||||
fs.close(fd, function () { |
|
||||||
if (callback) callback(writeErr) |
|
||||||
}) |
|
||||||
} |
|
||||||
} else { |
|
||||||
onePassDone = true |
onePassDone = true |
||||||
if (written === length) { |
offset += bytesWritten |
||||||
if (isUserFd) { |
length -= bytesWritten |
||||||
if (callback) callback(null) |
|
||||||
} else { |
|
||||||
fs.close(fd, callback) |
|
||||||
} |
|
||||||
} else { |
|
||||||
offset += written |
|
||||||
length -= written |
|
||||||
if (position !== null) { |
|
||||||
position += written |
|
||||||
} |
} |
||||||
writeAll(fd, isUserFd, buffer, offset, length, position, callback) |
} finally { |
||||||
|
await filehandle.close() |
||||||
} |
} |
||||||
} |
|
||||||
}) |
|
||||||
} |
} |
||||||
|
|
||||||
fs.writeFile = function (path, data, options, callback_) { |
class FakeFsWriteStream extends Writable { |
||||||
const callback = maybeCallback(arguments[arguments.length - 1]) |
constructor (filename) { |
||||||
|
super() |
||||||
if (!options || typeof options === 'function') { |
this.filename = filename |
||||||
options = { encoding: 'utf8', mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
|
this._content = Buffer.alloc(0) |
||||||
} else if (typeof options === 'string') { |
|
||||||
options = { encoding: options, mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
|
|
||||||
} else if (typeof options !== 'object') { |
|
||||||
throw new Error(`throwOptionsError${options}`) |
|
||||||
} |
} |
||||||
|
|
||||||
assertEncoding(options.encoding) |
_write (chunk, encoding, callback) { |
||||||
|
this._content = Buffer.concat([this._content, Buffer.from(chunk, encoding)]) |
||||||
const flag = options.flag || 'w' |
callback() |
||||||
|
|
||||||
if (isFd(path)) { |
|
||||||
writeFd(path, true) |
|
||||||
return |
|
||||||
} |
} |
||||||
|
|
||||||
fs.open(path, flag, options.mode, function (openErr, fd) { |
_end (chunk, encoding, callback) { |
||||||
if (openErr) { |
this._content = Buffer.concat([this._content, Buffer.from(chunk, encoding)]) |
||||||
if (callback) callback(openErr) |
callback() |
||||||
} else { |
|
||||||
writeFd(fd, false) |
|
||||||
} |
} |
||||||
}) |
|
||||||
|
|
||||||
function writeFd (fd, isUserFd) { |
|
||||||
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8') |
|
||||||
const position = /a/.test(flag) ? null : 0 |
|
||||||
|
|
||||||
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
fs.createWriteStream = function (path) { |
|
||||||
let content = '' |
|
||||||
return { |
|
||||||
write (data) { |
|
||||||
content += data |
|
||||||
}, |
|
||||||
close (callback) { |
close (callback) { |
||||||
fs.writeFile(path, content, callback) |
callbackify(fs.promises.writeFile)(this.filename, this._content, 'utf8', callback) |
||||||
} |
|
||||||
} |
} |
||||||
} |
} |
||||||
|
|
||||||
// End of fs modification
|
fs.createWriteStream = path => new FakeFsWriteStream(path) |
||||||
|
|
||||||
|
// End of fs monkey patching
|
||||||
const Nedb = require('../lib/datastore.js') |
const Nedb = require('../lib/datastore.js') |
||||||
const db = new Nedb({ filename: 'workspace/lac.db' }) |
const db = new Nedb({ filename: 'workspace/lac.db' }) |
||||||
|
|
||||||
db.loadDatabase() |
db.loadDatabaseAsync() // no need to await
|
||||||
|
|||||||
@ -1,65 +1,61 @@ |
|||||||
const fs = require('fs') |
const fs = require('fs') |
||||||
const { waterfall, whilst } = require('../test/utils.test.js') |
const fsPromises = fs.promises |
||||||
const Nedb = require('../lib/datastore') |
const Nedb = require('../lib/datastore') |
||||||
const { callbackify } = require('util') |
|
||||||
const db = new Nedb({ filename: './workspace/openfds.db', autoload: true }) |
|
||||||
const N = 64 |
const N = 64 |
||||||
let i |
|
||||||
let fds |
|
||||||
|
|
||||||
function multipleOpen (filename, N, callback) { |
// A console.error triggers an error of the parent test
|
||||||
whilst(function () { return i < N } |
|
||||||
, function (cb) { |
|
||||||
fs.open(filename, 'r', function (err, fd) { |
|
||||||
i += 1 |
|
||||||
if (fd) { fds.push(fd) } |
|
||||||
return cb(err) |
|
||||||
}) |
|
||||||
} |
|
||||||
, callback) |
|
||||||
} |
|
||||||
|
|
||||||
waterfall([ |
const test = async () => { |
||||||
// Check that ulimit has been set to the correct value
|
let filehandles = [] |
||||||
function (cb) { |
try { |
||||||
i = 0 |
for (let i = 0; i < 2 * N + 1; i++) { |
||||||
fds = [] |
const filehandle = await fsPromises.open('./test_lac/openFdsTestFile', 'r') |
||||||
multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) { |
filehandles.push(filehandle) |
||||||
if (!err) { console.log('No error occured while opening a file too many times') } |
} |
||||||
fds.forEach(function (fd) { fs.closeSync(fd) }) |
console.error('No error occurred while opening a file too many times') |
||||||
return cb() |
process.exit(1) |
||||||
}) |
} catch (error) { |
||||||
}, |
if (error.code !== 'EMFILE') { |
||||||
function (cb) { |
console.error(error) |
||||||
i = 0 |
process.exit(1) |
||||||
fds = [] |
} |
||||||
multipleOpen('./test_lac/openFdsTestFile2', N, function (err) { |
} finally { |
||||||
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err) } |
for (const filehandle of filehandles) { |
||||||
fds.forEach(function (fd) { fs.closeSync(fd) }) |
await filehandle.close() |
||||||
return cb() |
} |
||||||
}) |
filehandles = [] |
||||||
}, |
} |
||||||
// Then actually test NeDB persistence
|
|
||||||
function () { |
|
||||||
db.remove({}, { multi: true }, function (err) { |
|
||||||
if (err) { console.log(err) } |
|
||||||
db.insert({ hello: 'world' }, function (err) { |
|
||||||
if (err) { console.log(err) } |
|
||||||
|
|
||||||
i = 0 |
try { |
||||||
whilst(function () { return i < 2 * N + 1 } |
for (let i = 0; i < N; i++) { |
||||||
, function (cb) { |
const filehandle = await fsPromises.open('./test_lac/openFdsTestFile2', 'r') |
||||||
callbackify(() => db.persistence.persistCachedDatabaseAsync())(function (err) { |
filehandles.push(filehandle) |
||||||
if (err) { return cb(err) } |
|
||||||
i += 1 |
|
||||||
return cb() |
|
||||||
}) |
|
||||||
} |
} |
||||||
, function (err) { |
} catch (error) { |
||||||
if (err) { console.log('Got unexpected error during one peresistence operation: ' + err) } |
console.error(`An unexpected error occurred when opening file not too many times at i: ${i} with error: ${error}`) |
||||||
|
process.exit(1) |
||||||
|
} finally { |
||||||
|
for (const filehandle of filehandles) { |
||||||
|
await filehandle.close() |
||||||
} |
} |
||||||
) |
|
||||||
}) |
|
||||||
}) |
|
||||||
} |
} |
||||||
], () => {}) |
|
||||||
|
try { |
||||||
|
const db = new Nedb({ filename: './workspace/openfds.db' }) |
||||||
|
await db.loadDatabaseAsync() |
||||||
|
await db.removeAsync({}, { multi: true }) |
||||||
|
await db.insertAsync({ hello: 'world' }) |
||||||
|
|
||||||
|
for (let i = 0; i < 2 * N + 1; i++) { |
||||||
|
await db.persistence.persistCachedDatabaseAsync() |
||||||
|
} |
||||||
|
} catch (error) { |
||||||
|
console.error(`Got unexpected error during one persistence operation at ${i}: with error: ${error}`) |
||||||
|
} |
||||||
|
} |
||||||
|
try { |
||||||
|
test() |
||||||
|
} catch (error) { |
||||||
|
console.error(error) |
||||||
|
process.exit(1) |
||||||
|
} |
||||||
|
|||||||
Loading…
Reference in new issue