standard everything except browser-version

pull/2/head
Timothée Rebours 4 years ago
parent aa3302e5dc
commit 185ae680b8
  1. 22
      LICENSE
  2. 19
      LICENSE.md
  3. 12
      README.md
  4. 328
      benchmarks/commonUtilities.js
  5. 65
      benchmarks/ensureIndex.js
  6. 46
      benchmarks/find.js
  7. 48
      benchmarks/findOne.js
  8. 46
      benchmarks/findWithIn.js
  9. 50
      benchmarks/insert.js
  10. 52
      benchmarks/loadDatabase.js
  11. 60
      benchmarks/remove.js
  12. 61
      benchmarks/update.js
  13. 4
      index.js
  14. 345
      lib/cursor.js
  15. 11
      lib/customUtils.js
  16. 1252
      lib/datastore.js
  17. 123
      lib/executor.js
  18. 461
      lib/indexes.js
  19. 628
      lib/model.js
  20. 548
      lib/persistence.js
  21. 127
      lib/storage.js
  22. 31
      package.json
  23. 1318
      test/cursor.test.js
  24. 32
      test/customUtil.test.js
  25. 4155
      test/db.test.js
  26. 268
      test/executor.test.js
  27. 1231
      test/indexes.test.js
  28. 2
      test/mocha.opts
  29. 2178
      test/model.test.js
  30. 1331
      test/persistence.test.js
  31. 118
      test_lac/loadAndCrash.test.js
  32. 99
      test_lac/openFds.test.js

@ -1,22 +0,0 @@
(The MIT License)
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,19 @@
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Copyright (c) 2021 Seald [contact@seald.io](mailto:contact@seald.io);
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including without
limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom
the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -34,7 +34,7 @@ It is a subset of MongoDB's API (the most used operations).
* <a href="#inserting-documents">Inserting documents</a> * <a href="#inserting-documents">Inserting documents</a>
* <a href="#finding-documents">Finding documents</a> * <a href="#finding-documents">Finding documents</a>
* <a href="#basic-querying">Basic Querying</a> * <a href="#basic-querying">Basic Querying</a>
* <a href="#operators-lt-lte-gt-gte-in-nin-ne-exists-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)</a> * <a href="#operators-lt-lte-gt-gte-in-nin-ne-stat-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)</a>
* <a href="#array-fields">Array fields</a> * <a href="#array-fields">Array fields</a>
* <a href="#logical-operators-or-and-not-where">Logical operators $or, $and, $not, $where</a> * <a href="#logical-operators-or-and-not-where">Logical operators $or, $and, $not, $where</a>
* <a href="#sorting-and-paginating">Sorting and paginating</a> * <a href="#sorting-and-paginating">Sorting and paginating</a>
@ -236,14 +236,14 @@ db.findOne({ _id: 'id1' }, function (err, doc) {
}); });
``` ```
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex) #### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)
The syntax is `{ field: { $op: value } }` where `$op` is any comparison operator: The syntax is `{ field: { $op: value } }` where `$op` is any comparison operator:
* `$lt`, `$lte`: less than, less than or equal * `$lt`, `$lte`: less than, less than or equal
* `$gt`, `$gte`: greater than, greater than or equal * `$gt`, `$gte`: greater than, greater than or equal
* `$in`: member of. `value` must be an array of values * `$in`: member of. `value` must be an array of values
* `$ne`, `$nin`: not equal, not a member of * `$ne`, `$nin`: not equal, not a member of
* `$exists`: checks whether the document posses the property `field`. `value` should be true or false * `$stat`: checks whether the document posses the property `field`. `value` should be true or false
* `$regex`: checks whether a string is matched by the regular expression. Contrary to MongoDB, the use of `$options` with `$regex` is not supported, because it doesn't give you more power than regex flags. Basic queries are more readable so only use the `$regex` operator when you need to use another operator with it (see example below) * `$regex`: checks whether a string is matched by the regular expression. Contrary to MongoDB, the use of `$options` with `$regex` is not supported, because it doesn't give you more power than regex flags. Basic queries are more readable so only use the `$regex` operator when you need to use another operator with it (see example below)
```javascript ```javascript
@ -262,8 +262,8 @@ db.find({ planet: { $in: ['Earth', 'Jupiter'] }}, function (err, docs) {
// docs contains Earth and Jupiter // docs contains Earth and Jupiter
}); });
// Using $exists // Using $stat
db.find({ satellites: { $exists: true } }, function (err, docs) { db.find({ satellites: { $stat: true } }, function (err, docs) {
// docs contains only Mars // docs contains only Mars
}); });
@ -724,4 +724,4 @@ You don't have time? You can support NeDB by sending bitcoins to this address: 1
## License ## License
See [License](LICENSE) See [License](LICENSE.md)

@ -1,65 +1,58 @@
/** /**
* Functions that are used in several benchmark tests * Functions that are used in several benchmark tests
*/ */
const fs = require('fs')
var customUtils = require('../lib/customUtils') const path = require('path')
, fs = require('fs') const Datastore = require('../lib/datastore')
, path = require('path') const Persistence = require('../lib/persistence')
, Datastore = require('../lib/datastore') let executeAsap
, Persistence = require('../lib/persistence')
, executeAsap // process.nextTick or setImmediate depending on your Node version
;
try { try {
executeAsap = setImmediate; executeAsap = setImmediate
} catch (e) { } catch (e) {
executeAsap = process.nextTick; executeAsap = process.nextTick
} }
/** /**
* Configure the benchmark * Configure the benchmark
*/ */
module.exports.getConfiguration = function (benchDb) { module.exports.getConfiguration = function (benchDb) {
var d, n const program = require('commander')
, program = require('commander')
;
program program
.option('-n --number [number]', 'Size of the collection to test on', parseInt) .option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Use an index') .option('-i --with-index', 'Use an index')
.option('-m --in-memory', 'Test with an in-memory only store') .option('-m --in-memory', 'Test with an in-memory only store')
.parse(process.argv); .parse(process.argv)
n = program.number || 10000; const n = program.number || 10000
console.log("----------------------------"); console.log('----------------------------')
console.log("Test with " + n + " documents"); console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? "Use an index" : "Don't use an index"); console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log(program.inMemory ? "Use an in-memory datastore" : "Use a persistent datastore"); console.log(program.inMemory ? 'Use an in-memory datastore' : 'Use a persistent datastore')
console.log("----------------------------"); console.log('----------------------------')
d = new Datastore({ filename: benchDb const d = new Datastore({
, inMemoryOnly: program.inMemory filename: benchDb,
}); inMemoryOnly: program.inMemory
})
return { n: n, d: d, program: program };
};
return { n: n, d: d, program: program }
}
/** /**
* Ensure the workspace exists and the db datafile is empty * Ensure the workspace stat and the db datafile is empty
*/ */
module.exports.prepareDb = function (filename, cb) { module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () { Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) { fs.access(filename, fs.constants.FS_OK, function (err) {
if (exists) { if (!err) {
fs.unlink(filename, cb); fs.unlink(filename, cb)
} else { return cb(); } } else { return cb() }
}); })
}); })
}; }
/** /**
* Return an array with the numbers from 0 to n-1, in a random order * Return an array with the numbers from 0 to n-1, in a random order
@ -67,242 +60,221 @@ module.exports.prepareDb = function (filename, cb) {
* Useful to get fair tests * Useful to get fair tests
*/ */
function getRandomArray (n) { function getRandomArray (n) {
var res = [] const res = []
, i, j, temp let i
; let j
let temp
for (i = 0; i < n; i += 1) { res[i] = i; } for (i = 0; i < n; i += 1) { res[i] = i }
for (i = n - 1; i >= 1; i -= 1) { for (i = n - 1; i >= 1; i -= 1) {
j = Math.floor((i + 1) * Math.random()); j = Math.floor((i + 1) * Math.random())
temp = res[i]; temp = res[i]
res[i] = res[j]; res[i] = res[j]
res[j] = temp; res[j] = temp
} }
return res; return res
}; };
module.exports.getRandomArray = getRandomArray; module.exports.getRandomArray = getRandomArray
/** /**
* Insert a certain number of documents for testing * Insert a certain number of documents for testing
*/ */
module.exports.insertDocs = function (d, n, profiler, cb) { module.exports.insertDocs = function (d, n, profiler, cb) {
var beg = new Date() const order = getRandomArray(n)
, order = getRandomArray(n)
; profiler.step('Begin inserting ' + n + ' docs')
profiler.step('Begin inserting ' + n + ' docs'); function runFrom (i) {
if (i === n) { // Finished
function runFrom(i) { const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
if (i === n) { // Finished console.log('===== RESULT (insert) ===== ' + opsPerSecond + ' ops/s')
var opsPerSecond = Math.floor(1000* n / profiler.elapsedSinceLastStep()); profiler.step('Finished inserting ' + n + ' docs')
console.log("===== RESULT (insert) ===== " + opsPerSecond + " ops/s"); profiler.insertOpsPerSecond = opsPerSecond
profiler.step('Finished inserting ' + n + ' docs'); return cb()
profiler.insertOpsPerSecond = opsPerSecond;
return cb();
} }
// eslint-disable-next-line node/handle-callback-err
d.insert({ docNumber: order[i] }, function (err) { d.insert({ docNumber: order[i] }, function (err) {
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Find documents with find * Find documents with find
*/ */
module.exports.findDocs = function (d, n, profiler, cb) { module.exports.findDocs = function (d, n, profiler, cb) {
var beg = new Date() const order = getRandomArray(n)
, order = getRandomArray(n)
;
profiler.step("Finding " + n + " documents"); profiler.step('Finding ' + n + ' documents')
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
console.log("===== RESULT (find) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s"); console.log('===== RESULT (find) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs'); profiler.step('Finished finding ' + n + ' docs')
return cb(); return cb()
} }
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: order[i] }, function (err, docs) { d.find({ docNumber: order[i] }, function (err, docs) {
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb('One find didnt work'); } if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Find documents with find and the $in operator * Find documents with find and the $in operator
*/ */
module.exports.findDocsWithIn = function (d, n, profiler, cb) { module.exports.findDocsWithIn = function (d, n, profiler, cb) {
var beg = new Date() const ins = []
, order = getRandomArray(n) const arraySize = Math.min(10, n)
, ins = [], i, j
, arraySize = Math.min(10, n) // The array for $in needs to be smaller than n (inclusive)
;
// Preparing all the $in arrays, will take some time // Preparing all the $in arrays, will take some time
for (i = 0; i < n; i += 1) { for (let i = 0; i < n; i += 1) {
ins[i] = []; ins[i] = []
for (j = 0; j < arraySize; j += 1) { for (let j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n); ins[i].push((i + j) % n)
} }
} }
profiler.step("Finding " + n + " documents WITH $IN OPERATOR"); profiler.step('Finding ' + n + ' documents WITH $IN OPERATOR')
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
console.log("===== RESULT (find with in selector) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s"); console.log('===== RESULT (find with in selector) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs'); profiler.step('Finished finding ' + n + ' docs')
return cb(); return cb()
} }
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: { $in: ins[i] } }, function (err, docs) { d.find({ docNumber: { $in: ins[i] } }, function (err, docs) {
if (docs.length !== arraySize) { return cb('One find didnt work'); } if (docs.length !== arraySize) { return cb(new Error('One find didnt work')) }
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Find documents with findOne * Find documents with findOne
*/ */
module.exports.findOneDocs = function (d, n, profiler, cb) { module.exports.findOneDocs = function (d, n, profiler, cb) {
var beg = new Date() const order = getRandomArray(n)
, order = getRandomArray(n)
;
profiler.step("FindingOne " + n + " documents"); profiler.step('FindingOne ' + n + ' documents')
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
console.log("===== RESULT (findOne) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s"); console.log('===== RESULT (findOne) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs'); profiler.step('Finished finding ' + n + ' docs')
return cb(); return cb()
} }
// eslint-disable-next-line node/handle-callback-err
d.findOne({ docNumber: order[i] }, function (err, doc) { d.findOne({ docNumber: order[i] }, function (err, doc) {
if (!doc || doc.docNumber !== order[i]) { return cb('One find didnt work'); } if (!doc || doc.docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Update documents * Update documents
* options is the same as the options object for update * options is the same as the options object for update
*/ */
module.exports.updateDocs = function (options, d, n, profiler, cb) { module.exports.updateDocs = function (options, d, n, profiler, cb) {
var beg = new Date() const order = getRandomArray(n)
, order = getRandomArray(n)
;
profiler.step("Updating " + n + " documents"); profiler.step('Updating ' + n + ' documents')
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
console.log("===== RESULT (update) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s"); console.log('===== RESULT (update) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished updating ' + n + ' docs'); profiler.step('Finished updating ' + n + ' docs')
return cb(); return cb()
} }
// Will not actually modify the document but will take the same time // Will not actually modify the document but will take the same time
d.update({ docNumber: order[i] }, { docNumber: order[i] }, options, function (err, nr) { d.update({ docNumber: order[i] }, { docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (nr !== 1) { return cb('One update didnt work'); } if (nr !== 1) { return cb(new Error('One update didnt work')) }
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Remove documents * Remove documents
* options is the same as the options object for update * options is the same as the options object for update
*/ */
module.exports.removeDocs = function (options, d, n, profiler, cb) { module.exports.removeDocs = function (options, d, n, profiler, cb) {
var beg = new Date() const order = getRandomArray(n)
, order = getRandomArray(n)
;
profiler.step("Removing " + n + " documents"); profiler.step('Removing ' + n + ' documents')
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
// opsPerSecond corresponds to 1 insert + 1 remove, needed to keep collection size at 10,000 // opsPerSecond corresponds to 1 insert + 1 remove, needed to keep collection size at 10,000
// We need to subtract the time taken by one insert to get the time actually taken by one remove // We need to subtract the time taken by one insert to get the time actually taken by one remove
var opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep()); const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
var removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond))) const removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log("===== RESULT (remove) ===== " + removeOpsPerSecond + " ops/s"); console.log('===== RESULT (remove) ===== ' + removeOpsPerSecond + ' ops/s')
profiler.step('Finished removing ' + n + ' docs'); profiler.step('Finished removing ' + n + ' docs')
return cb(); return cb()
} }
d.remove({ docNumber: order[i] }, options, function (err, nr) { d.remove({ docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (nr !== 1) { return cb('One remove didnt work'); } if (nr !== 1) { return cb(new Error('One remove didnt work')) }
d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size at n // eslint-disable-next-line node/handle-callback-err
// So actually we're calculating the average time taken by one insert + one remove d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size at n
// So actually we're calculating the average time taken by one insert + one remove
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }
/** /**
* Load database * Load database
*/ */
module.exports.loadDatabase = function (d, n, profiler, cb) { module.exports.loadDatabase = function (d, n, profiler, cb) {
var beg = new Date() profiler.step('Loading the database ' + n + ' times')
, order = getRandomArray(n)
;
profiler.step("Loading the database " + n + " times");
function runFrom(i) { function runFrom (i) {
if (i === n) { // Finished if (i === n) { // Finished
console.log("===== RESULT ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s"); console.log('===== RESULT ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished loading a database' + n + ' times'); profiler.step('Finished loading a database' + n + ' times')
return cb(); return cb()
} }
// eslint-disable-next-line node/handle-callback-err
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
executeAsap(function () { executeAsap(function () {
runFrom(i + 1); runFrom(i + 1)
}); })
}); })
} }
runFrom(0); runFrom(0)
}; }

@ -1,51 +1,48 @@
var Datastore = require('../lib/datastore') const Datastore = require('../lib/datastore')
, benchDb = 'workspace/insert.bench.db' const benchDb = 'workspace/insert.bench.db'
, async = require('async') const async = require('async')
, commonUtilities = require('./commonUtilities') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const ExecTime = require('exec-time')
, profiler = new execTime('INSERT BENCH') const profiler = new ExecTime('INSERT BENCH')
, d = new Datastore(benchDb) const d = new Datastore(benchDb)
, program = require('commander') const program = require('commander')
, n
;
program program
.option('-n --number [number]', 'Size of the collection to test on', parseInt) .option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index') .option('-i --with-index', 'Test with an index')
.parse(process.argv); .parse(process.argv)
n = program.number || 10000; const n = program.number || 10000
console.log("----------------------------"); console.log('----------------------------')
console.log("Test with " + n + " documents"); console.log('Test with ' + n + ' documents')
console.log("----------------------------"); console.log('----------------------------')
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, function (cb) { function (cb) {
var i; let i
profiler.step('Begin calling ensureIndex ' + n + ' times'); profiler.step('Begin calling ensureIndex ' + n + ' times')
for (i = 0; i < n; i += 1) { for (i = 0; i < n; i += 1) {
d.ensureIndex({ fieldName: 'docNumber' }); d.ensureIndex({ fieldName: 'docNumber' })
delete d.indexes.docNumber; delete d.indexes.docNumber
} }
console.log("Average time for one ensureIndex: " + (profiler.elapsedSinceLastStep() / n) + "ms"); console.log('Average time for one ensureIndex: ' + (profiler.elapsedSinceLastStep() / n) + 'ms')
profiler.step('Finished calling ensureIndex ' + n + ' times'); profiler.step('Finished calling ensureIndex ' + n + ' times')
} }
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/find.bench.db'
, benchDb = 'workspace/find.bench.db' const async = require('async')
, fs = require('fs') const ExecTime = require('exec-time')
, path = require('path') const profiler = new ExecTime('FIND BENCH')
, async = require('async') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const config = commonUtilities.getConfiguration(benchDb)
, profiler = new execTime('FIND BENCH') const d = config.d
, commonUtilities = require('./commonUtilities') const n = config.n
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); } if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, async.apply(commonUtilities.findDocs, d, n, profiler) async.apply(commonUtilities.findDocs, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,31 +1,27 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/findOne.bench.db'
, benchDb = 'workspace/findOne.bench.db' const async = require('async')
, fs = require('fs') const ExecTime = require('exec-time')
, path = require('path') const profiler = new ExecTime('FINDONE BENCH')
, async = require('async') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const config = commonUtilities.getConfiguration(benchDb)
, profiler = new execTime('FINDONE BENCH') const d = config.d
, commonUtilities = require('./commonUtilities') const n = config.n
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); } if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, function (cb) { setTimeout(function () {cb();}, 500); } function (cb) { setTimeout(function () { cb() }, 500) },
, async.apply(commonUtilities.findOneDocs, d, n, profiler) async.apply(commonUtilities.findOneDocs, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/find.bench.db'
, benchDb = 'workspace/find.bench.db' const async = require('async')
, fs = require('fs') const ExecTime = require('exec-time')
, path = require('path') const profiler = new ExecTime('FIND BENCH')
, async = require('async') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const config = commonUtilities.getConfiguration(benchDb)
, profiler = new execTime('FIND BENCH') const d = config.d
, commonUtilities = require('./commonUtilities') const n = config.n
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); } if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, async.apply(commonUtilities.findDocsWithIn, d, n, profiler) async.apply(commonUtilities.findDocsWithIn, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,33 +1,31 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/insert.bench.db'
, benchDb = 'workspace/insert.bench.db' const async = require('async')
, async = require('async') const ExecTime = require('exec-time')
, execTime = require('exec-time') const profiler = new ExecTime('INSERT BENCH')
, profiler = new execTime('INSERT BENCH') const commonUtilities = require('./commonUtilities')
, commonUtilities = require('./commonUtilities') const config = commonUtilities.getConfiguration(benchDb)
, config = commonUtilities.getConfiguration(benchDb) const d = config.d
, d = config.d let n = config.n
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { if (config.program.withIndex) {
d.ensureIndex({ fieldName: 'docNumber' }); d.ensureIndex({ fieldName: 'docNumber' })
n = 2 * n; // We will actually insert twice as many documents n = 2 * n // We will actually insert twice as many documents
// because the index is slower when the collection is already // because the index is slower when the collection is already
// big. So the result given by the algorithm will be a bit worse than // big. So the result given by the algorithm will be a bit worse than
// actual performance // actual performance
} }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore') const Datastore = require('../lib/datastore')
, benchDb = 'workspace/loaddb.bench.db' const benchDb = 'workspace/loaddb.bench.db'
, fs = require('fs') const async = require('async')
, path = require('path') const commonUtilities = require('./commonUtilities')
, async = require('async') const ExecTime = require('exec-time')
, commonUtilities = require('./commonUtilities') const profiler = new ExecTime('LOADDB BENCH')
, execTime = require('exec-time') const d = new Datastore(benchDb)
, profiler = new execTime('LOADDB BENCH') const program = require('commander')
, d = new Datastore(benchDb)
, program = require('commander')
, n
;
program program
.option('-n --number [number]', 'Size of the collection to test on', parseInt) .option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index') .option('-i --with-index', 'Test with an index')
.parse(process.argv); .parse(process.argv)
n = program.number || 10000; const n = program.number || 10000
console.log("----------------------------"); console.log('----------------------------')
console.log("Test with " + n + " documents"); console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? "Use an index" : "Don't use an index"); console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log("----------------------------"); console.log('----------------------------')
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(cb); d.loadDatabase(cb)
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, async.apply(commonUtilities.loadDatabase, d, n, profiler) async.apply(commonUtilities.loadDatabase, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/remove.bench.db'
, benchDb = 'workspace/remove.bench.db' const async = require('async')
, fs = require('fs') const ExecTime = require('exec-time')
, path = require('path') const profiler = new ExecTime('REMOVE BENCH')
, async = require('async') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const config = commonUtilities.getConfiguration(benchDb)
, profiler = new execTime('REMOVE BENCH') const d = config.d
, commonUtilities = require('./commonUtilities') const n = config.n
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); } if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with remove only one document // Test with remove only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); } function (cb) { profiler.step('MULTI: FALSE'); return cb() },
, async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler) async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler),
// Test with multiple documents // Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function () { return cb(); }); } function (cb) { d.remove({}, { multi: true }, function () { return cb() }) },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); } function (cb) { profiler.step('MULTI: TRUE'); return cb() },
, async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler) async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,39 +1,36 @@
var Datastore = require('../lib/datastore') const benchDb = 'workspace/update.bench.db'
, benchDb = 'workspace/update.bench.db' const async = require('async')
, fs = require('fs') const ExecTime = require('exec-time')
, path = require('path') const profiler = new ExecTime('UPDATE BENCH')
, async = require('async') const commonUtilities = require('./commonUtilities')
, execTime = require('exec-time') const config = commonUtilities.getConfiguration(benchDb)
, profiler = new execTime('UPDATE BENCH') const d = config.d
, commonUtilities = require('./commonUtilities') const n = config.n
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([ async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb) async.apply(commonUtilities.prepareDb, benchDb),
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); } if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb(); cb()
}); })
} },
, function (cb) { profiler.beginProfiling(); return cb(); } function (cb) { profiler.beginProfiling(); return cb() },
, async.apply(commonUtilities.insertDocs, d, n, profiler) async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with update only one document // Test with update only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); } function (cb) { profiler.step('MULTI: FALSE'); return cb() },
, async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler) async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler),
// Test with multiple documents // Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function (err) { return cb(); }); } // eslint-disable-next-line node/handle-callback-err
, async.apply(commonUtilities.insertDocs, d, n, profiler) function (cb) { d.remove({}, { multi: true }, function (err) { return cb() }) },
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); } async.apply(commonUtilities.insertDocs, d, n, profiler),
, async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler) function (cb) { profiler.step('MULTI: TRUE'); return cb() },
async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler)
], function (err) { ], function (err) {
profiler.step("Benchmark finished"); profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); } if (err) { return console.log('An error was encountered: ', err) }
}); })

@ -1,3 +1,3 @@
var Datastore = require('./lib/datastore'); const Datastore = require('./lib/datastore')
module.exports = Datastore; module.exports = Datastore

@ -1,204 +1,201 @@
/** /**
* Manage access to data, be it to find, update or remove it * Manage access to data, be it to find, update or remove it
*/ */
var model = require('./model') const model = require('./model')
, _ = require('underscore') const _ = require('underscore')
;
class Cursor {
/**
* Create a new cursor for this collection
/** * @param {Datastore} db - The datastore this cursor is bound to
* Create a new cursor for this collection * @param {Query} query - The query this cursor will operate on
* @param {Datastore} db - The datastore this cursor is bound to * @param {Function} execFn - Handler to be executed after cursor has found the results and before the callback passed to find/findOne/update/remove
* @param {Query} query - The query this cursor will operate on */
* @param {Function} execFn - Handler to be executed after cursor has found the results and before the callback passed to find/findOne/update/remove constructor (db, query, execFn) {
*/ this.db = db
function Cursor (db, query, execFn) { this.query = query || {}
this.db = db; if (execFn) { this.execFn = execFn }
this.query = query || {}; }
if (execFn) { this.execFn = execFn; }
}
/**
* Set a limit to the number of results
*/
Cursor.prototype.limit = function(limit) {
this._limit = limit;
return this;
};
/**
* Skip a the number of results
*/
Cursor.prototype.skip = function(skip) {
this._skip = skip;
return this;
};
/**
* Sort results of the query
* @param {SortQuery} sortQuery - SortQuery is { field: order }, field can use the dot-notation, order is 1 for ascending and -1 for descending
*/
Cursor.prototype.sort = function(sortQuery) {
this._sort = sortQuery;
return this;
};
/** /**
* Add the use of a projection * Set a limit to the number of results
* @param {Object} projection - MongoDB-style projection. {} means take all fields. Then it's { key1: 1, key2: 1 } to take only key1 and key2 */
* { key1: 0, key2: 0 } to omit only key1 and key2. Except _id, you can't mix takes and omits limit (limit) {
*/ this._limit = limit
Cursor.prototype.projection = function(projection) { return this
this._projection = projection; }
return this;
};
/**
* Skip a the number of results
*/
skip (skip) {
this._skip = skip
return this
}
/** /**
* Apply the projection * Sort results of the query
*/ * @param {SortQuery} sortQuery - SortQuery is { field: order }, field can use the dot-notation, order is 1 for ascending and -1 for descending
Cursor.prototype.project = function (candidates) { */
var res = [], self = this sort (sortQuery) {
, keepId, action, keys this._sort = sortQuery
; return this
}
if (this._projection === undefined || Object.keys(this._projection).length === 0) { /**
return candidates; * Add the use of a projection
* @param {Object} projection - MongoDB-style projection. {} means take all fields. Then it's { key1: 1, key2: 1 } to take only key1 and key2
* { key1: 0, key2: 0 } to omit only key1 and key2. Except _id, you can't mix takes and omits
*/
projection (projection) {
this._projection = projection
return this
} }
keepId = this._projection._id === 0 ? false : true; /**
this._projection = _.omit(this._projection, '_id'); * Apply the projection
*/
// Check for consistency project (candidates) {
keys = Object.keys(this._projection); const res = []
keys.forEach(function (k) { const self = this
if (action !== undefined && self._projection[k] !== action) { throw new Error("Can't both keep and omit fields except for _id"); } let action
action = self._projection[k];
});
// Do the actual projection
candidates.forEach(function (candidate) {
var toPush;
if (action === 1) { // pick-type projection
toPush = { $set: {} };
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
}
if (keepId) {
toPush._id = candidate._id;
} else {
delete toPush._id;
}
res.push(toPush);
});
return res; if (this._projection === undefined || Object.keys(this._projection).length === 0) {
}; return candidates
}
const keepId = this._projection._id !== 0
this._projection = _.omit(this._projection, '_id')
// Check for consistency
const keys = Object.keys(this._projection)
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error('Can\'t both keep and omit fields except for _id') }
action = self._projection[k]
})
// Do the actual projection
candidates.forEach(function (candidate) {
let toPush
if (action === 1) { // pick-type projection
toPush = { $set: {} }
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k)
if (toPush.$set[k] === undefined) { delete toPush.$set[k] }
})
toPush = model.modify({}, toPush)
} else { // omit-type projection
toPush = { $unset: {} }
keys.forEach(function (k) { toPush.$unset[k] = true })
toPush = model.modify(candidate, toPush)
}
if (keepId) {
toPush._id = candidate._id
} else {
delete toPush._id
}
res.push(toPush)
})
/** return res
* Get all matching elements
* Will return pointers to matched elements (shallow copies), returning full copies is the role of find or findOne
* This is an internal function, use exec which uses the executor
*
* @param {Function} callback - Signature: err, results
*/
Cursor.prototype._exec = function(_callback) {
var res = [], added = 0, skipped = 0, self = this
, error = null
, i, keys, key
;
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback);
} else {
return _callback(error, res);
}
} }
this.db.getCandidates(this.query, function (err, candidates) { /**
if (err) { return callback(err); } * Get all matching elements
* Will return pointers to matched elements (shallow copies), returning full copies is the role of find or findOne
* This is an internal function, use exec which uses the executor
*
* @param {Function} callback - Signature: err, results
*/
_exec (_callback) {
let res = []
let added = 0
let skipped = 0
const self = this
let error = null
let i
let keys
let key
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback)
} else {
return _callback(error, res)
}
}
try { this.db.getCandidates(this.query, function (err, candidates) {
for (i = 0; i < candidates.length; i += 1) { if (err) { return callback(err) }
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip try {
if (!self._sort) { for (i = 0; i < candidates.length; i += 1) {
if (self._skip && self._skip > skipped) { if (model.match(candidates[i], self.query)) {
skipped += 1; // If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1
} else {
res.push(candidates[i])
added += 1
if (self._limit && self._limit <= added) { break }
}
} else { } else {
res.push(candidates[i]); res.push(candidates[i])
added += 1;
if (self._limit && self._limit <= added) { break; }
} }
} else {
res.push(candidates[i]);
} }
} }
} catch (err) {
return callback(err)
} }
} catch (err) {
return callback(err);
}
// Apply all sorts // Apply all sorts
if (self._sort) { if (self._sort) {
keys = Object.keys(self._sort); keys = Object.keys(self._sort)
// Sorting // Sorting
var criteria = []; const criteria = []
for (i = 0; i < keys.length; i++) { for (i = 0; i < keys.length; i++) {
key = keys[i]; key = keys[i]
criteria.push({ key: key, direction: self._sort[key] }); criteria.push({ key: key, direction: self._sort[key] })
}
res.sort(function(a, b) {
var criterion, compare, i;
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i];
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings);
if (compare !== 0) {
return compare;
}
} }
return 0; res.sort(function (a, b) {
}); let criterion
let compare
// Applying limit and skip let i
var limit = self._limit || res.length for (i = 0; i < criteria.length; i++) {
, skip = self._skip || 0; criterion = criteria[i]
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings)
res = res.slice(skip, skip + limit); if (compare !== 0) {
} return compare
}
}
return 0
})
// Apply projection // Applying limit and skip
try { const limit = self._limit || res.length
res = self.project(res); const skip = self._skip || 0
} catch (e) {
error = e;
res = undefined;
}
return callback(error, res); res = res.slice(skip, skip + limit)
}); }
};
Cursor.prototype.exec = function () { // Apply projection
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments }); try {
}; res = self.project(res)
} catch (e) {
error = e
res = undefined
}
return callback(error, res)
})
}
exec () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments })
}
}
// Interface // Interface
module.exports = Cursor; module.exports = Cursor

@ -1,5 +1,4 @@
var crypto = require('crypto') const crypto = require('crypto')
;
/** /**
* Return a random alphanumerical string of length len * Return a random alphanumerical string of length len
@ -12,11 +11,9 @@ var crypto = require('crypto')
function uid (len) { function uid (len) {
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2))) return crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
.toString('base64') .toString('base64')
.replace(/[+\/]/g, '') .replace(/[+/]/g, '')
.slice(0, len); .slice(0, len)
} }
// Interface // Interface
module.exports.uid = uid; module.exports.uid = uid

File diff suppressed because it is too large Load Diff

@ -1,78 +1,73 @@
/** /**
* Responsible for sequentially executing actions on the database * Responsible for sequentially executing actions on the database
*/ */
const async = require('async')
var async = require('async') class Executor {
; constructor () {
this.buffer = []
this.ready = false
function Executor () { // This queue will execute all commands, one-by-one in order
this.buffer = []; this.queue = async.queue(function (task, cb) {
this.ready = false; const newArguments = []
// This queue will execute all commands, one-by-one in order // task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
this.queue = async.queue(function (task, cb) { for (let i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]) }
var newArguments = []; const lastArg = task.arguments[task.arguments.length - 1]
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array // Always tell the queue task is complete. Execute callback if any was given.
for (var i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]); } if (typeof lastArg === 'function') {
var lastArg = task.arguments[task.arguments.length - 1]; // Callback was supplied
newArguments[newArguments.length - 1] = function () {
// Always tell the queue task is complete. Execute callback if any was given. if (typeof setImmediate === 'function') {
if (typeof lastArg === 'function') { setImmediate(cb)
// Callback was supplied } else {
newArguments[newArguments.length - 1] = function () { process.nextTick(cb)
if (typeof setImmediate === 'function') { }
setImmediate(cb); lastArg.apply(null, arguments)
} else {
process.nextTick(cb);
} }
lastArg.apply(null, arguments); } else if (!lastArg && task.arguments.length !== 0) {
}; // false/undefined/null supplied as callback
} else if (!lastArg && task.arguments.length !== 0) { newArguments[newArguments.length - 1] = function () { cb() }
// false/undefined/null supplied as callbback } else {
newArguments[newArguments.length - 1] = function () { cb(); }; // Nothing supplied as callback
} else { newArguments.push(function () { cb() })
// Nothing supplied as callback }
newArguments.push(function () { cb(); });
}
task.fn.apply(task.this, newArguments)
task.fn.apply(task.this, newArguments); }, 1)
}, 1);
}
/**
* If executor is ready, queue task (and process it immediately if executor was idle)
* If not, buffer task for later processing
* @param {Object} task
* task.this - Object to use as this
* task.fn - Function to execute
* task.arguments - Array of arguments, IMPORTANT: only the last argument may be a function (the callback)
* and the last argument cannot be false/undefined/null
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
Executor.prototype.push = function (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task);
} else {
this.buffer.push(task);
} }
};
/**
* Queue all tasks in buffer (in the same order they came in)
* Automatically sets executor as ready
*/
Executor.prototype.processBuffer = function () {
var i;
this.ready = true;
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]); }
this.buffer = [];
};
/**
* If executor is ready, queue task (and process it immediately if executor was idle)
* If not, buffer task for later processing
* @param {Object} task
* task.this - Object to use as this
* task.fn - Function to execute
* task.arguments - Array of arguments, IMPORTANT: only the last argument may be a function (the callback)
* and the last argument cannot be false/undefined/null
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
push (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task)
} else {
this.buffer.push(task)
}
}
/**
* Queue all tasks in buffer (in the same order they came in)
* Automatically sets executor as ready
*/
processBuffer () {
let i
this.ready = true
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]) }
this.buffer = []
}
}
// Interface // Interface
module.exports = Executor; module.exports = Executor

@ -1,294 +1,295 @@
var BinarySearchTree = require('binary-search-tree').AVLTree const BinarySearchTree = require('@seald-io/binary-search-tree').BinarySearchTree
, model = require('./model') const model = require('./model')
, _ = require('underscore') const _ = require('underscore')
, util = require('util')
;
/** /**
* Two indexed pointers are equal iif they point to the same place * Two indexed pointers are equal iif they point to the same place
*/ */
function checkValueEquality (a, b) { function checkValueEquality (a, b) {
return a === b; return a === b
} }
/** /**
* Type-aware projection * Type-aware projection
*/ */
function projectForUnique (elt) { function projectForUnique (elt) {
if (elt === null) { return '$null'; } if (elt === null) { return '$null' }
if (typeof elt === 'string') { return '$string' + elt; } if (typeof elt === 'string') { return '$string' + elt }
if (typeof elt === 'boolean') { return '$boolean' + elt; } if (typeof elt === 'boolean') { return '$boolean' + elt }
if (typeof elt === 'number') { return '$number' + elt; } if (typeof elt === 'number') { return '$number' + elt }
if (util.isArray(elt)) { return '$date' + elt.getTime(); } if (Array.isArray(elt)) { return '$date' + elt.getTime() }
return elt; // Arrays and objects, will check for pointer equality return elt // Arrays and objects, will check for pointer equality
} }
class Index {
/**
* Create a new index
* All methods on an index guarantee that either the whole operation was successful and the index changed
* or the operation was unsuccessful and an error is thrown while the index is unchanged
* @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields)
* @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
* @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
*/
constructor (options) {
this.fieldName = options.fieldName
this.unique = options.unique || false
this.sparse = options.sparse || false
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality }
this.reset() // No data in the beginning
}
/** /**
* Create a new index * Reset an index
* All methods on an index guarantee that either the whole operation was successful and the index changed * @param {Document or Array of documents} newData Optional, data to initialize the index with
* or the operation was unsuccessful and an error is thrown while the index is unchanged * If an error is thrown during insertion, the index is not modified
* @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields) */
* @param {Boolean} options.unique Optional, enforce a unique constraint (default: false) reset (newData) {
* @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false) this.tree = new BinarySearchTree(this.treeOptions)
*/
function Index (options) {
this.fieldName = options.fieldName;
this.unique = options.unique || false;
this.sparse = options.sparse || false;
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
this.reset(); // No data in the beginning
}
/**
* Reset an index
* @param {Document or Array of documents} newData Optional, data to initialize the index with
* If an error is thrown during insertion, the index is not modified
*/
Index.prototype.reset = function (newData) {
this.tree = new BinarySearchTree(this.treeOptions);
if (newData) { this.insert(newData); }
};
/** if (newData) { this.insert(newData) }
* Insert a new document in the index }
* If an array is passed, we insert all its elements (if one insertion fails the index is not modified)
* O(log(n))
*/
Index.prototype.insert = function (doc) {
var key, self = this
, keys, i, failingI, error
;
if (util.isArray(doc)) { this.insertMultipleDocs(doc); return; } /**
* Insert a new document in the index
* If an array is passed, we insert all its elements (if one insertion fails the index is not modified)
* O(log(n))
*/
insert (doc) {
let keys
let i
let failingI
let error
if (Array.isArray(doc)) {
this.insertMultipleDocs(doc)
return
}
key = model.getDotValue(doc, this.fieldName); const key = model.getDotValue(doc, this.fieldName)
// We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return }
if (!Array.isArray(key)) {
this.tree.insert(key, doc)
} else {
// If an insert fails due to a unique constraint, roll back all inserts before it
keys = _.uniq(key, projectForUnique)
for (i = 0; i < keys.length; i += 1) {
try {
this.tree.insert(keys[i], doc)
} catch (e) {
error = e
failingI = i
break
}
}
// We don't index documents that don't contain the field if the index is sparse if (error) {
if (key === undefined && this.sparse) { return; } for (i = 0; i < failingI; i += 1) {
this.tree.delete(keys[i], doc)
}
if (!util.isArray(key)) { throw error
this.tree.insert(key, doc); }
} else { }
// If an insert fails due to a unique constraint, roll back all inserts before it }
keys = _.uniq(key, projectForUnique);
for (i = 0; i < keys.length; i += 1) { /**
* Insert an array of documents in the index
* If a constraint is violated, the changes should be rolled back and an error thrown
*
* @API private
*/
insertMultipleDocs (docs) {
let i
let error
let failingI
for (i = 0; i < docs.length; i += 1) {
try { try {
this.tree.insert(keys[i], doc); this.insert(docs[i])
} catch (e) { } catch (e) {
error = e; error = e
failingI = i; failingI = i
break; break
} }
} }
if (error) { if (error) {
for (i = 0; i < failingI; i += 1) { for (i = 0; i < failingI; i += 1) {
this.tree.delete(keys[i], doc); this.remove(docs[i])
} }
throw error; throw error
} }
} }
};
/**
/** * Remove a document from the index
* Insert an array of documents in the index * If an array is passed, we remove all its elements
* If a constraint is violated, the changes should be rolled back and an error thrown * The remove operation is safe with regards to the 'unique' constraint
* * O(log(n))
* @API private */
*/ remove (doc) {
Index.prototype.insertMultipleDocs = function (docs) { const self = this
var i, error, failingI;
if (Array.isArray(doc)) {
for (i = 0; i < docs.length; i += 1) { doc.forEach(function (d) { self.remove(d) })
try { return
this.insert(docs[i]);
} catch (e) {
error = e;
failingI = i;
break;
} }
}
if (error) { const key = model.getDotValue(doc, this.fieldName)
for (i = 0; i < failingI; i += 1) {
this.remove(docs[i]);
}
throw error;
}
};
/**
* Remove a document from the index
* If an array is passed, we remove all its elements
* The remove operation is safe with regards to the 'unique' constraint
* O(log(n))
*/
Index.prototype.remove = function (doc) {
var key, self = this;
if (util.isArray(doc)) { doc.forEach(function (d) { self.remove(d); }); return; }
key = model.getDotValue(doc, this.fieldName);
if (key === undefined && this.sparse) { return; }
if (!util.isArray(key)) {
this.tree.delete(key, doc);
} else {
_.uniq(key, projectForUnique).forEach(function (_key) {
self.tree.delete(_key, doc);
});
}
};
/**
* Update a document in the index
* If a constraint is violated, changes are rolled back and an error thrown
* Naive implementation, still in O(log(n))
*/
Index.prototype.update = function (oldDoc, newDoc) {
if (util.isArray(oldDoc)) { this.updateMultipleDocs(oldDoc); return; }
this.remove(oldDoc); if (key === undefined && this.sparse) { return }
try { if (!Array.isArray(key)) {
this.insert(newDoc); this.tree.delete(key, doc)
} catch (e) { } else {
this.insert(oldDoc); _.uniq(key, projectForUnique).forEach(function (_key) {
throw e; self.tree.delete(_key, doc)
})
}
} }
};
/** /**
* Update multiple documents in the index * Update a document in the index
* If a constraint is violated, the changes need to be rolled back * If a constraint is violated, changes are rolled back and an error thrown
* and an error thrown * Naive implementation, still in O(log(n))
* @param {Array of oldDoc, newDoc pairs} pairs */
* update (oldDoc, newDoc) {
* @API private if (Array.isArray(oldDoc)) {
*/ this.updateMultipleDocs(oldDoc)
Index.prototype.updateMultipleDocs = function (pairs) { return
var i, failingI, error; }
for (i = 0; i < pairs.length; i += 1) { this.remove(oldDoc)
this.remove(pairs[i].oldDoc);
}
for (i = 0; i < pairs.length; i += 1) {
try { try {
this.insert(pairs[i].newDoc); this.insert(newDoc)
} catch (e) { } catch (e) {
error = e; this.insert(oldDoc)
failingI = i; throw e
break;
} }
} }
// If an error was raised, roll back changes in the inverse order /**
if (error) { * Update multiple documents in the index
for (i = 0; i < failingI; i += 1) { * If a constraint is violated, the changes need to be rolled back
this.remove(pairs[i].newDoc); * and an error thrown
* @param {Array of oldDoc, newDoc pairs} pairs
*
* @API private
*/
updateMultipleDocs (pairs) {
let i
let failingI
let error
for (i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc)
} }
for (i = 0; i < pairs.length; i += 1) { for (i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc); try {
this.insert(pairs[i].newDoc)
} catch (e) {
error = e
failingI = i
break
}
} }
throw error; // If an error was raised, roll back changes in the inverse order
} if (error) {
}; for (i = 0; i < failingI; i += 1) {
this.remove(pairs[i].newDoc)
}
for (i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc)
}
/** throw error
* Revert an update }
*/
Index.prototype.revertUpdate = function (oldDoc, newDoc) {
var revert = [];
if (!util.isArray(oldDoc)) {
this.update(newDoc, oldDoc);
} else {
oldDoc.forEach(function (pair) {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc });
});
this.update(revert);
} }
};
/**
* Get all documents in index whose key match value (if it is a Thing) or one of the elements of value (if it is an array of Things)
* @param {Thing} value Value to match the key against
* @return {Array of documents}
*/
Index.prototype.getMatching = function (value) {
var self = this;
if (!util.isArray(value)) {
return self.tree.search(value);
} else {
var _res = {}, res = [];
value.forEach(function (v) { /**
self.getMatching(v).forEach(function (doc) { * Revert an update
_res[doc._id] = doc; */
}); revertUpdate (oldDoc, newDoc) {
}); const revert = []
Object.keys(_res).forEach(function (_id) { if (!Array.isArray(oldDoc)) {
res.push(_res[_id]); this.update(newDoc, oldDoc)
}); } else {
oldDoc.forEach(function (pair) {
return res; revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc })
})
this.update(revert)
}
} }
};
/**
* Get all documents in index whose key is between bounds are they are defined by query
* Documents are sorted by key
* @param {Query} query
* @return {Array of documents}
*/
Index.prototype.getBetweenBounds = function (query) {
return this.tree.betweenBounds(query);
};
/** /**
* Get all elements in the index * Get all documents in index whose key match value (if it is a Thing) or one of the elements of value (if it is an array of Things)
* @return {Array of documents} * @param {Thing} value Value to match the key against
*/ * @return {Array of documents}
Index.prototype.getAll = function () { */
var res = []; getMatching (value) {
const self = this
this.tree.executeOnEveryNode(function (node) {
var i; if (!Array.isArray(value)) {
return self.tree.search(value)
for (i = 0; i < node.data.length; i += 1) { } else {
res.push(node.data[i]); const _res = {}
const res = []
value.forEach(function (v) {
self.getMatching(v).forEach(function (doc) {
_res[doc._id] = doc
})
})
Object.keys(_res).forEach(function (_id) {
res.push(_res[_id])
})
return res
} }
}); }
return res; /**
}; * Get all documents in index whose key is between bounds are they are defined by query
* Documents are sorted by key
* @param {Query} query
* @return {Array of documents}
*/
getBetweenBounds (query) {
return this.tree.betweenBounds(query)
}
/**
* Get all elements in the index
* @return {Array of documents}
*/
getAll () {
const res = []
this.tree.executeOnEveryNode(function (node) {
let i
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i])
}
})
return res
}
}
// Interface // Interface
module.exports = Index; module.exports = Index

File diff suppressed because it is too large Load Diff

@ -4,311 +4,303 @@
* * Persistence.loadDatabase(callback) and callback has signature err * * Persistence.loadDatabase(callback) and callback has signature err
* * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err * * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
*/ */
const storage = require('./storage')
const path = require('path')
const model = require('./model')
const async = require('async')
const customUtils = require('./customUtils')
const Index = require('./indexes')
class Persistence {
/**
* Create a new Persistence object for database options.db
* @param {Datastore} options.db
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/
constructor (options) {
let i
let j
let randomString
this.db = options.db
this.inMemoryOnly = this.db.inMemoryOnly
this.filename = this.db.filename
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
}
var storage = require('./storage') // After serialization and before deserialization hooks with some basic sanity checks
, path = require('path') if (options.afterSerialization && !options.beforeDeserialization) {
, model = require('./model') throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
, async = require('async') }
, customUtils = require('./customUtils') if (!options.afterSerialization && options.beforeDeserialization) {
, Index = require('./indexes') throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
; }
this.afterSerialization = options.afterSerialization || function (s) { return s }
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s }
/** for (i = 1; i < 30; i += 1) {
* Create a new Persistence object for database options.db for (j = 0; j < 10; j += 1) {
* @param {Datastore} options.db randomString = customUtils.uid(i)
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion) throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss')
*/ }
function Persistence (options) {
var i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss");
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss");
}
this.afterSerialization = options.afterSerialization || function (s) { return s; };
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i);
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss");
} }
} }
}
// For NW apps, store data in the same directory where NW stores application data // For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) { if (this.filename && options.nodeWebkitAppName) {
console.log("=================================================================="); console.log('==================================================================')
console.log("WARNING: The nodeWebkitAppName option is deprecated"); console.log('WARNING: The nodeWebkitAppName option is deprecated')
console.log("To get the path to the directory where Node Webkit stores the data"); console.log('To get the path to the directory where Node Webkit stores the data')
console.log("for your app, use the internal nw.gui module like this"); console.log('for your app, use the internal nw.gui module like this')
console.log("require('nw.gui').App.dataPath"); console.log('require(\'nw.gui\').App.dataPath')
console.log("See https://github.com/rogerwang/node-webkit/issues/500"); console.log('See https://github.com/rogerwang/node-webkit/issues/500')
console.log("=================================================================="); console.log('==================================================================')
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename); this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename)
}
} }
};
/** /**
* Check if a directory exists and create it on the fly if it is not the case * Persist cached database
* cb is optional, signature: err * This serves as a compaction function since the cache always contains only the number of documents in the collection
*/ * while the data file is append-only so it may grow larger
Persistence.ensureDirectoryExists = function (dir, cb) { * @param {Function} cb Optional callback, signature: err
var callback = cb || function () {} */
; persistCachedDatabase (cb) {
const callback = cb || function () {}
let toPersist = ''
const self = this
if (this.inMemoryOnly) { return callback(null) }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({
$$indexCreated: {
fieldName: fieldName,
unique: self.db.indexes[fieldName].unique,
sparse: self.db.indexes[fieldName].sparse
}
})) + '\n'
}
})
storage.mkdirp(dir, function (err) { return callback(err); }); storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
}; if (err) { return callback(err) }
self.db.emit('compaction.done')
return callback(null)
})
}
/**
* Queue a rewrite of the datafile
*/
compactDatafile () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] })
}
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
setAutocompactionInterval (interval) {
const self = this
const minInterval = 5000
const realInterval = Math.max(interval || 0, minInterval)
this.stopAutocompaction()
/** this.autocompactionIntervalId = setInterval(function () {
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores self.compactDatafile()
* data for this application. Probably the best place to store data }, realInterval)
*/
Persistence.getNWAppFilename = function (appName, relativeFilename) {
var home;
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA;
if (!home) { throw new Error("Couldn't find the base application data folder"); }
home = path.join(home, appName);
break;
case 'darwin':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, 'Library', 'Application Support', appName);
break;
case 'linux':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, '.config', appName);
break;
default:
throw new Error("Can't use the Node Webkit relative path for platform " + process.platform);
break;
} }
return path.join(home, 'nedb-data', relativeFilename); /**
} * Stop autocompaction (do nothing if autocompaction was not running)
*/
stopAutocompaction () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId) }
}
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
*/
persistNewState (newDocs, cb) {
const self = this
let toPersist = ''
const callback = cb || function () {}
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null) }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
if (toPersist.length === 0) { return callback(null) }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err)
})
}
/** /**
* Persist cached database * From a database's raw data, return the corresponding
* This serves as a compaction function since the cache always contains only the number of documents in the collection * machine understandable collection
* while the data file is append-only so it may grow larger */
* @param {Function} cb Optional callback, signature: err treatRawData (rawData) {
*/ const data = rawData.split('\n')
Persistence.prototype.persistCachedDatabase = function (cb) { const dataById = {}
var callback = cb || function () {} const tdata = []
, toPersist = '' let i
, self = this const indexes = {}
; let corruptItems = -1
if (this.inMemoryOnly) { return callback(null); } for (i = 0; i < data.length; i += 1) {
let doc
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'; try {
}); doc = model.deserialize(this.beforeDeserialization(data[i]))
Object.keys(this.db.indexes).forEach(function (fieldName) { if (doc._id) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted if (doc.$$deleted === true) {
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n'; delete dataById[doc._id]
} else {
dataById[doc._id] = doc
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
} else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved]
}
} catch (e) {
corruptItems += 1
}
} }
});
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err); }
self.db.emit('compaction.done');
return callback(null);
});
};
/** // A bit lenient on corruption
* Queue a rewrite of the datafile if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
*/ throw new Error('More than ' + Math.floor(100 * this.corruptAlertThreshold) + '% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss')
Persistence.prototype.compactDatafile = function () { }
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
};
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
Persistence.prototype.setAutocompactionInterval = function (interval) {
var self = this
, minInterval = 5000
, realInterval = Math.max(interval || 0, minInterval)
;
this.stopAutocompaction();
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile();
}, realInterval);
};
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
Persistence.prototype.stopAutocompaction = function () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
};
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
*/
Persistence.prototype.persistNewState = function (newDocs, cb) {
var self = this
, toPersist = ''
, callback = cb || function () {}
;
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
if (toPersist.length === 0) { return callback(null); }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) { Object.keys(dataById).forEach(function (k) {
return callback(err); tdata.push(dataById[k])
}); })
};
return { data: tdata, indexes: indexes }
}
/** /**
* From a database's raw data, return the corresponding * Load the database
* machine understandable collection * 1) Create all indexes
*/ * 2) Insert all data
Persistence.prototype.treatRawData = function (rawData) { * 3) Compact the database
var data = rawData.split('\n') * This means pulling data out of the data file or creating it if it doesn't exist
, dataById = {} * Also, all data is persisted right away, which has the effect of compacting the database file
, tdata = [] * This operation is very quick at startup for a big collection (60ms for ~10k docs)
, i * @param {Function} cb Optional callback, signature: err
, indexes = {} */
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt loadDatabase (cb) {
; const callback = cb || function () {}
const self = this
for (i = 0; i < data.length; i += 1) {
var doc; self.db.resetIndexes()
try { // In-memory only datastore
doc = model.deserialize(this.beforeDeserialization(data[i])); if (self.inMemoryOnly) { return callback(null) }
if (doc._id) {
if (doc.$$deleted === true) { async.waterfall([
delete dataById[doc._id]; function (cb) {
} else { // eslint-disable-next-line node/handle-callback-err
dataById[doc._id] = doc; Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
} // TODO: handle error
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) { // eslint-disable-next-line node/handle-callback-err
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated; storage.ensureDatafileIntegrity(self.filename, function (err) {
} else if (typeof doc.$$indexRemoved === "string") { // TODO: handle error
delete indexes[doc.$$indexRemoved]; storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err) }
let treatedData
try {
treatedData = self.treatRawData(rawData)
} catch (e) {
return cb(e)
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key])
})
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data)
} catch (e) {
self.db.resetIndexes() // Rollback any index which didn't fail
return cb(e)
}
self.db.persistence.persistCachedDatabase(cb)
})
})
})
} }
} catch (e) { ], function (err) {
corruptItems += 1; if (err) { return callback(err) }
}
}
// A bit lenient on corruption self.db.executor.processBuffer()
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) { return callback(null)
throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss"); })
} }
Object.keys(dataById).forEach(function (k) { /**
tdata.push(dataById[k]); * Check if a directory stat and create it on the fly if it is not the case
}); * cb is optional, signature: err
*/
return { data: tdata, indexes: indexes }; static ensureDirectoryExists (dir, cb) {
}; const callback = cb || function () {}
storage.mkdir(dir, { recursive: true }, err => { callback(err) })
}
/** /**
* Load the database * Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
* 1) Create all indexes * data for this application. Probably the best place to store data
* 2) Insert all data */
* 3) Compact the database static getNWAppFilename (appName, relativeFilename) {
* This means pulling data out of the data file or creating it if it doesn't exist let home
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs) switch (process.platform) {
* @param {Function} cb Optional callback, signature: err case 'win32':
*/ case 'win64':
Persistence.prototype.loadDatabase = function (cb) { home = process.env.LOCALAPPDATA || process.env.APPDATA
var callback = cb || function () {} if (!home) { throw new Error('Couldn\'t find the base application data folder') }
, self = this home = path.join(home, appName)
; break
case 'darwin':
self.db.resetIndexes(); home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
// In-memory only datastore home = path.join(home, 'Library', 'Application Support', appName)
if (self.inMemoryOnly) { return callback(null); } break
case 'linux':
async.waterfall([ home = process.env.HOME
function (cb) { if (!home) { throw new Error('Couldn\'t find the base application data directory') }
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) { home = path.join(home, '.config', appName)
storage.ensureDatafileIntegrity(self.filename, function (err) { break
storage.readFile(self.filename, 'utf8', function (err, rawData) { default:
if (err) { return cb(err); } throw new Error('Can\'t use the Node Webkit relative path for platform ' + process.platform)
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);
});
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
return cb(e);
}
self.db.persistence.persistCachedDatabase(cb);
});
});
});
} }
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
};
return path.join(home, 'nedb-data', relativeFilename)
}
}
// Interface // Interface
module.exports = Persistence; module.exports = Persistence

@ -6,34 +6,30 @@
* This version is the Node.js/Node Webkit version * This version is the Node.js/Node Webkit version
* It's essentially fs, mkdirp and crash safe write and read functions * It's essentially fs, mkdirp and crash safe write and read functions
*/ */
const fs = require('fs')
var fs = require('fs') const async = require('async')
, mkdirp = require('mkdirp') const path = require('path')
, async = require('async') const storage = {}
, path = require('path')
, storage = {} // eslint-disable-next-line node/no-callback-literal
; storage.exists = (path, cb) => fs.access(path, fs.constants.F_OK, (err) => { cb(!err) })
storage.rename = fs.rename
storage.exists = fs.exists; storage.writeFile = fs.writeFile
storage.rename = fs.rename; storage.unlink = fs.unlink
storage.writeFile = fs.writeFile; storage.appendFile = fs.appendFile
storage.unlink = fs.unlink; storage.readFile = fs.readFile
storage.appendFile = fs.appendFile; storage.mkdir = fs.mkdir
storage.readFile = fs.readFile;
storage.mkdirp = mkdirp;
/** /**
* Explicit name ... * Explicit name ...
*/ */
storage.ensureFileDoesntExist = function (file, callback) { storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) { storage.exists(file, function (exists) {
if (!exists) { return callback(null); } if (!exists) { return callback(null) }
storage.unlink(file, function (err) { return callback(err); });
});
};
storage.unlink(file, function (err) { return callback(err) })
})
}
/** /**
* Flush data in OS buffer to storage if corresponding option is set * Flush data in OS buffer to storage if corresponding option is set
@ -42,36 +38,36 @@ storage.ensureFileDoesntExist = function (file, callback) {
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested * If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/ */
storage.flushToStorage = function (options, callback) { storage.flushToStorage = function (options, callback) {
var filename, flags; let filename
let flags
if (typeof options === 'string') { if (typeof options === 'string') {
filename = options; filename = options
flags = 'r+'; flags = 'r+'
} else { } else {
filename = options.filename; filename = options.filename
flags = options.isDir ? 'r' : 'r+'; flags = options.isDir ? 'r' : 'r+'
} }
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss // Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens // except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null); } if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null) }
fs.open(filename, flags, function (err, fd) { fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err); } if (err) { return callback(err) }
fs.fsync(fd, function (errFS) { fs.fsync(fd, function (errFS) {
fs.close(fd, function (errC) { fs.close(fd, function (errC) {
if (errFS || errC) { if (errFS || errC) {
var e = new Error('Failed to flush to storage'); const e = new Error('Failed to flush to storage')
e.errorOnFsync = errFS; e.errorOnFsync = errFS
e.errorOnClose = errC; e.errorOnClose = errC
return callback(e); return callback(e)
} else { } else {
return callback(null); return callback(null)
} }
}); })
}); })
}); })
}; }
/** /**
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost) * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
@ -80,31 +76,30 @@ storage.flushToStorage = function (options, callback) {
* @param {Function} cb Optional callback, signature: err * @param {Function} cb Optional callback, signature: err
*/ */
storage.crashSafeWriteFile = function (filename, data, cb) { storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {} const callback = cb || function () {}
, tempFilename = filename + '~'; const tempFilename = filename + '~'
async.waterfall([ async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }) async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }),
, function (cb) { function (cb) {
storage.exists(filename, function (exists) { storage.exists(filename, function (exists) {
if (exists) { if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err); }); storage.flushToStorage(filename, function (err) { return cb(err) })
} else { } else {
return cb(); return cb()
} }
}); })
} },
, function (cb) { function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); }); storage.writeFile(tempFilename, data, function (err) { return cb(err) })
} },
, async.apply(storage.flushToStorage, tempFilename) async.apply(storage.flushToStorage, tempFilename),
, function (cb) { function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); }); storage.rename(tempFilename, filename, function (err) { return cb(err) })
} },
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }) async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err); }) ], function (err) { return callback(err) })
}; }
/** /**
* Ensure the datafile contains all the data, even if there was a crash during a full file write * Ensure the datafile contains all the data, even if there was a crash during a full file write
@ -112,25 +107,23 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {Function} callback signature: err * @param {Function} callback signature: err
*/ */
storage.ensureDatafileIntegrity = function (filename, callback) { storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~'; const tempFilename = filename + '~'
storage.exists(filename, function (filenameExists) { storage.exists(filename, function (filenameExists) {
// Write was successful // Write was successful
if (filenameExists) { return callback(null); } if (filenameExists) { return callback(null) }
storage.exists(tempFilename, function (oldFilenameExists) { storage.exists(tempFilename, function (oldFilenameExists) {
// New database // New database
if (!oldFilenameExists) { if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); }); return storage.writeFile(filename, '', 'utf8', function (err) { callback(err) })
} }
// Write failed, use old version // Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err); }); storage.rename(tempFilename, filename, function (err) { return callback(err) })
}); })
}); })
}; }
// Interface // Interface
module.exports = storage; module.exports = storage

@ -20,27 +20,36 @@
"url": "git@github.com:louischatriot/nedb.git" "url": "git@github.com:louischatriot/nedb.git"
}, },
"dependencies": { "dependencies": {
"@seald-io/binary-search-tree": "^1.0.0",
"async": "0.2.10", "async": "0.2.10",
"binary-search-tree": "0.2.5", "localforage": "^1.9.0",
"localforage": "^1.3.0", "underscore": "^1.13.1"
"mkdirp": "~0.5.1",
"underscore": "~1.4.4"
}, },
"devDependencies": { "devDependencies": {
"chai": "^3.2.0", "chai": "^4.3.4",
"mocha": "1.4.x", "commander": "1.1.1",
"exec-time": "0.0.2",
"mocha": "^8.4.0",
"request": "2.9.x", "request": "2.9.x",
"semver": "^7.3.5",
"sinon": "1.3.x", "sinon": "1.3.x",
"exec-time": "0.0.2", "standard": "^16.0.3"
"commander": "1.1.1"
}, },
"scripts": { "scripts": {
"test": "./node_modules/.bin/mocha --reporter spec --timeout 10000" "test": "mocha --reporter spec --timeout 10000"
}, },
"main": "index", "main": "index.js",
"browser": { "browser": {
"./lib/customUtils.js": "./browser-version/browser-specific/lib/customUtils.js", "./lib/customUtils.js": "./browser-version/browser-specific/lib/customUtils.js",
"./lib/storage.js": "./browser-version/browser-specific/lib/storage.js" "./lib/storage.js": "./browser-version/browser-specific/lib/storage.js"
}, },
"license": "SEE LICENSE IN LICENSE" "license": "MIT",
"publishConfig": {
"access": "public"
},
"standard": {
"ignore": [
"browser-version"
]
}
} }

File diff suppressed because it is too large Load Diff

@ -1,26 +1,20 @@
var should = require('chai').should() /* eslint-env mocha */
, assert = require('chai').assert const chai = require('chai')
, customUtils = require('../lib/customUtils') const customUtils = require('../lib/customUtils')
, fs = require('fs')
;
chai.should()
describe('customUtils', function () { describe('customUtils', function () {
describe('uid', function () { describe('uid', function () {
it('Generates a string of the expected length', function () { it('Generates a string of the expected length', function () {
customUtils.uid(3).length.should.equal(3); customUtils.uid(3).length.should.equal(3)
customUtils.uid(16).length.should.equal(16); customUtils.uid(16).length.should.equal(16)
customUtils.uid(42).length.should.equal(42); customUtils.uid(42).length.should.equal(42)
customUtils.uid(1000).length.should.equal(1000); customUtils.uid(1000).length.should.equal(1000)
}); })
// Very small probability of conflict // Very small probability of conflict
it('Generated uids should not be the same', function () { it('Generated uids should not be the same', function () {
customUtils.uid(56).should.not.equal(customUtils.uid(56)); customUtils.uid(56).should.not.equal(customUtils.uid(56))
}); })
})
}); })
});

File diff suppressed because it is too large Load Diff

@ -1,213 +1,215 @@
var should = require('chai').should() /* eslint-env mocha */
, assert = require('chai').assert const chai = require('chai')
, testDb = 'workspace/test.db' const testDb = 'workspace/test.db'
, fs = require('fs') const fs = require('fs')
, path = require('path') const path = require('path')
, _ = require('underscore') const async = require('async')
, async = require('async') const Datastore = require('../lib/datastore')
, model = require('../lib/model') const Persistence = require('../lib/persistence')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence') const { assert } = chai
; chai.should()
// Test that even if a callback throws an exception, the next DB operations will still be executed // Test that even if a callback throws an exception, the next DB operations will still be executed
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends // We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testThrowInCallback (d, done) { function testThrowInCallback (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException'); const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException'); process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) { process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track // Do nothing with the error which is only there to test we stay on track
}); })
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err) { d.find({}, function (err) {
process.nextTick(function () { process.nextTick(function () {
// eslint-disable-next-line node/handle-callback-err
d.insert({ bar: 1 }, function (err) { d.insert({ bar: 1 }, function (err) {
process.removeAllListeners('uncaughtException'); process.removeAllListeners('uncaughtException')
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) { for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]); process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
} }
done(); done()
}); })
}); })
throw new Error('Some error'); throw new Error('Some error')
}); })
} }
// Test that if the callback is falsy, the next DB operations will still be executed // Test that if the callback is falsy, the next DB operations will still be executed
function testFalsyCallback (d, done) { function testFalsyCallback (d, done) {
d.insert({ a: 1 }, null); d.insert({ a: 1 }, null)
process.nextTick(function () { process.nextTick(function () {
d.update({ a: 1 }, { a: 2 }, {}, null); d.update({ a: 1 }, { a: 2 }, {}, null)
process.nextTick(function () { process.nextTick(function () {
d.update({ a: 2 }, { a: 1 }, null); d.update({ a: 2 }, { a: 1 }, null)
process.nextTick(function () { process.nextTick(function () {
d.remove({ a: 2 }, {}, null); d.remove({ a: 2 }, {}, null)
process.nextTick(function () { process.nextTick(function () {
d.remove({ a: 2 }, null); d.remove({ a: 2 }, null)
process.nextTick(function () { process.nextTick(function () {
d.find({}, done); d.find({}, done)
}); })
}); })
}); })
}); })
}); })
} }
// Test that operations are executed in the right order // Test that operations are executed in the right order
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends // We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testRightOrder (d, done) { function testRightOrder (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException'); const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException'); process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) { process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track // Do nothing with the error which is only there to test we stay on track
}); })
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) { d.find({}, function (err, docs) {
docs.length.should.equal(0); docs.length.should.equal(0)
d.insert({ a: 1 }, function () { d.insert({ a: 1 }, function () {
d.update({ a: 1 }, { a: 2 }, {}, function () { d.update({ a: 1 }, { a: 2 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) { d.find({}, function (err, docs) {
docs[0].a.should.equal(2); docs[0].a.should.equal(2)
process.nextTick(function () { process.nextTick(function () {
d.update({ a: 2 }, { a: 3 }, {}, function () { d.update({ a: 2 }, { a: 3 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) { d.find({}, function (err, docs) {
docs[0].a.should.equal(3); docs[0].a.should.equal(3)
process.removeAllListeners('uncaughtException'); process.removeAllListeners('uncaughtException')
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) { for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]); process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
} }
done(); done()
}); })
}); })
}); })
throw new Error('Some error'); throw new Error('Some error')
}); })
}); })
}); })
}); })
} }
// Note: The following test does not have any assertion because it // Note: The following test does not have any assertion because it
// is meant to address the deprecation warning: // is meant to address the deprecation warning:
// (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral. // (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
// see // see
var testEventLoopStarvation = function(d, done){ const testEventLoopStarvation = function (d, done) {
var times = 1001; const times = 1001
var i = 0; let i = 0
while ( i <times) { while (i < times) {
i++; i++
d.find({"bogus": "search"}, function (err, docs) { // eslint-disable-next-line node/handle-callback-err
}); d.find({ bogus: 'search' }, function (err, docs) {
} })
done(); }
}; done()
}
// Test that operations are executed in the right order even with no callback // Test that operations are executed in the right order even with no callback
function testExecutorWorksWithoutCallback (d, done) { function testExecutorWorksWithoutCallback (d, done) {
d.insert({ a: 1 }); d.insert({ a: 1 })
d.insert({ a: 2 }, false); d.insert({ a: 2 }, false)
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) { d.find({}, function (err, docs) {
docs.length.should.equal(2); docs.length.should.equal(2)
done(); done()
}); })
} }
describe('Executor', function () { describe('Executor', function () {
describe('With persistent database', function () { describe('With persistent database', function () {
var d; let d
beforeEach(function (done) { beforeEach(function (done) {
d = new Datastore({ filename: testDb }); d = new Datastore({ filename: testDb })
d.filename.should.equal(testDb); d.filename.should.equal(testDb)
d.inMemoryOnly.should.equal(false); d.inMemoryOnly.should.equal(false)
async.waterfall([ async.waterfall([
function (cb) { function (cb) {
Persistence.ensureDirectoryExists(path.dirname(testDb), function () { Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
fs.exists(testDb, function (exists) { fs.access(testDb, fs.constants.F_OK, function (err) {
if (exists) { if (!err) {
fs.unlink(testDb, cb); fs.unlink(testDb, cb)
} else { return cb(); } } else { return cb() }
}); })
}); })
} },
, function (cb) { function (cb) {
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
assert.isNull(err); assert.isNull(err)
d.getAllData().length.should.equal(0); d.getAllData().length.should.equal(0)
return cb(); return cb()
}); })
} }
], done); ], done)
}); })
it('A throw in a callback doesnt prevent execution of next operations', function(done) {
testThrowInCallback(d, done);
});
it('A falsy callback doesnt prevent execution of next operations', function(done) { it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done); testThrowInCallback(d, done)
}); })
it('Operations are executed in the right order', function(done) { it('A falsy callback doesnt prevent execution of next operations', function (done) {
testRightOrder(d, done); testFalsyCallback(d, done)
}); })
it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function(done){ it('Operations are executed in the right order', function (done) {
testEventLoopStarvation(d, done); testRightOrder(d, done)
}); })
it('Works in the right order even with no supplied callback', function(done){ it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function (done) {
testExecutorWorksWithoutCallback(d, done); testEventLoopStarvation(d, done)
}); })
}); // ==== End of 'With persistent database' ====
it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done)
})
}) // ==== End of 'With persistent database' ====
describe('With non persistent database', function () { describe('With non persistent database', function () {
var d; let d
beforeEach(function (done) { beforeEach(function (done) {
d = new Datastore({ inMemoryOnly: true }); d = new Datastore({ inMemoryOnly: true })
d.inMemoryOnly.should.equal(true); d.inMemoryOnly.should.equal(true)
d.loadDatabase(function (err) { d.loadDatabase(function (err) {
assert.isNull(err); assert.isNull(err)
d.getAllData().length.should.equal(0); d.getAllData().length.should.equal(0)
return done(); return done()
}); })
}); })
it('A throw in a callback doesnt prevent execution of next operations', function(done) { it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testThrowInCallback(d, done); testThrowInCallback(d, done)
}); })
it('A falsy callback doesnt prevent execution of next operations', function(done) { it('A falsy callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done); testFalsyCallback(d, done)
}); })
it('Operations are executed in the right order', function(done) { it('Operations are executed in the right order', function (done) {
testRightOrder(d, done); testRightOrder(d, done)
}); })
it('Works in the right order even with no supplied callback', function(done){ it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done); testExecutorWorksWithoutCallback(d, done)
}); })
}) // ==== End of 'With non persistent database' ====
}); // ==== End of 'With non persistent database' ==== })
});

File diff suppressed because it is too large Load Diff

@ -1,2 +0,0 @@
--reporter spec
--timeout 30000

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,123 +1,121 @@
/* eslint-env mocha */
/* global DEBUG */
/** /**
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes * Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes
*/ */
var fs = require('fs'); const fs = require('fs')
function rethrow() { function rethrow () {
// Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and // Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
// is fairly slow to generate. // is fairly slow to generate.
if (DEBUG) { if (DEBUG) {
var backtrace = new Error(); const backtrace = new Error()
return function(err) { return function (err) {
if (err) { if (err) {
backtrace.stack = err.name + ': ' + err.message + backtrace.stack = err.name + ': ' + err.message +
backtrace.stack.substr(backtrace.name.length); backtrace.stack.substr(backtrace.name.length)
throw backtrace; throw backtrace
} }
}; }
} }
return function(err) { return function (err) {
if (err) { if (err) {
throw err; // Forgot a callback but don't know where? Use NODE_DEBUG=fs throw err // Forgot a callback but don't know where? Use NODE_DEBUG=fs
} }
}; }
} }
function maybeCallback(cb) { function maybeCallback (cb) {
return typeof cb === 'function' ? cb : rethrow(); return typeof cb === 'function' ? cb : rethrow()
} }
function isFd(path) { function isFd (path) {
return (path >>> 0) === path; return (path >>> 0) === path
} }
function assertEncoding(encoding) { function assertEncoding (encoding) {
if (encoding && !Buffer.isEncoding(encoding)) { if (encoding && !Buffer.isEncoding(encoding)) {
throw new Error('Unknown encoding: ' + encoding); throw new Error('Unknown encoding: ' + encoding)
} }
} }
var onePassDone = false; let onePassDone = false
function writeAll(fd, isUserFd, buffer, offset, length, position, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
if (onePassDone) { process.exit(1); } // Crash on purpose before rewrite done function writeAll (fd, isUserFd, buffer, offset, length, position, callback_) {
var l = Math.min(5000, length); // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash const callback = maybeCallback(arguments[arguments.length - 1])
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
const l = Math.min(5000, length) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
// write(fd, buffer, offset, length, position, callback) // write(fd, buffer, offset, length, position, callback)
fs.write(fd, buffer, offset, l, position, function(writeErr, written) { fs.write(fd, buffer, offset, l, position, function (writeErr, written) {
if (writeErr) { if (writeErr) {
if (isUserFd) { if (isUserFd) {
if (callback) callback(writeErr); if (callback) callback(writeErr)
} else { } else {
fs.close(fd, function() { fs.close(fd, function () {
if (callback) callback(writeErr); if (callback) callback(writeErr)
}); })
} }
} else { } else {
onePassDone = true; onePassDone = true
if (written === length) { if (written === length) {
if (isUserFd) { if (isUserFd) {
if (callback) callback(null); if (callback) callback(null)
} else { } else {
fs.close(fd, callback); fs.close(fd, callback)
} }
} else { } else {
offset += written; offset += written
length -= written; length -= written
if (position !== null) { if (position !== null) {
position += written; position += written
} }
writeAll(fd, isUserFd, buffer, offset, length, position, callback); writeAll(fd, isUserFd, buffer, offset, length, position, callback)
} }
} }
}); })
} }
fs.writeFile = function(path, data, options, callback_) { fs.writeFile = function (path, data, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]); const callback = maybeCallback(arguments[arguments.length - 1])
if (!options || typeof options === 'function') { if (!options || typeof options === 'function') {
options = { encoding: 'utf8', mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases) options = { encoding: 'utf8', mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options === 'string') { } else if (typeof options === 'string') {
options = { encoding: options, mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases) options = { encoding: options, mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options !== 'object') { } else if (typeof options !== 'object') {
throwOptionsError(options); throw new Error(`throwOptionsError${options}`)
} }
assertEncoding(options.encoding); assertEncoding(options.encoding)
var flag = options.flag || 'w'; const flag = options.flag || 'w'
if (isFd(path)) { if (isFd(path)) {
writeFd(path, true); writeFd(path, true)
return; return
} }
fs.open(path, flag, options.mode, function(openErr, fd) { fs.open(path, flag, options.mode, function (openErr, fd) {
if (openErr) { if (openErr) {
if (callback) callback(openErr); if (callback) callback(openErr)
} else { } else {
writeFd(fd, false); writeFd(fd, false)
} }
}); })
function writeFd(fd, isUserFd) { function writeFd (fd, isUserFd) {
var buffer = (data instanceof Buffer) ? data : new Buffer('' + data, const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8')
options.encoding || 'utf8'); const position = /a/.test(flag) ? null : 0
var position = /a/.test(flag) ? null : 0;
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback); writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback)
} }
}; }
// End of fs modification // End of fs modification
var Nedb = require('../lib/datastore.js') const Nedb = require('../lib/datastore.js')
, db = new Nedb({ filename: 'workspace/lac.db' }) const db = new Nedb({ filename: 'workspace/lac.db' })
;
db.loadDatabase(); db.loadDatabase()

@ -1,67 +1,64 @@
var fs = require('fs') const fs = require('fs')
, child_process = require('child_process') const async = require('async')
, async = require('async') const Nedb = require('../lib/datastore')
, Nedb = require('../lib/datastore') const db = new Nedb({ filename: './workspace/openfds.db', autoload: true })
, db = new Nedb({ filename: './workspace/openfds.db', autoload: true }) const N = 64
, N = 64 // Half the allowed file descriptors let i
, i, fds let fds
;
function multipleOpen (filename, N, callback) { function multipleOpen (filename, N, callback) {
async.whilst( function () { return i < N; } async.whilst(function () { return i < N }
, function (cb) { , function (cb) {
fs.open(filename, 'r', function (err, fd) { fs.open(filename, 'r', function (err, fd) {
i += 1; i += 1
if (fd) { fds.push(fd); } if (fd) { fds.push(fd) }
return cb(err); return cb(err)
}); })
} }
, callback); , callback)
} }
async.waterfall([ async.waterfall([
// Check that ulimit has been set to the correct value // Check that ulimit has been set to the correct value
function (cb) { function (cb) {
i = 0; i = 0
fds = []; fds = []
multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) { multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) {
if (!err) { console.log("No error occured while opening a file too many times"); } if (!err) { console.log('No error occured while opening a file too many times') }
fds.forEach(function (fd) { fs.closeSync(fd); }); fds.forEach(function (fd) { fs.closeSync(fd) })
return cb(); return cb()
}) })
} },
, function (cb) { function (cb) {
i = 0; i = 0
fds = []; fds = []
multipleOpen('./test_lac/openFdsTestFile2', N, function (err) { multipleOpen('./test_lac/openFdsTestFile2', N, function (err) {
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err); } if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err) }
fds.forEach(function (fd) { fs.closeSync(fd); }); fds.forEach(function (fd) { fs.closeSync(fd) })
return cb(); return cb()
}) })
} },
// Then actually test NeDB persistence // Then actually test NeDB persistence
, function () { function () {
db.remove({}, { multi: true }, function (err) { db.remove({}, { multi: true }, function (err) {
if (err) { console.log(err); } if (err) { console.log(err) }
db.insert({ hello: 'world' }, function (err) { db.insert({ hello: 'world' }, function (err) {
if (err) { console.log(err); } if (err) { console.log(err) }
i = 0; i = 0
async.whilst( function () { return i < 2 * N + 1; } async.whilst(function () { return i < 2 * N + 1 }
, function (cb) { , function (cb) {
db.persistence.persistCachedDatabase(function (err) { db.persistence.persistCachedDatabase(function (err) {
if (err) { return cb(err); } if (err) { return cb(err) }
i += 1; i += 1
return cb(); return cb()
}); })
} }
, function (err) { , function (err) {
if (err) { console.log("Got unexpected error during one peresistence operation: " + err); } if (err) { console.log('Got unexpected error during one peresistence operation: ' + err) }
} }
); )
})
}); })
});
} }
]); ])

Loading…
Cancel
Save