standard everything except browser-version

pull/2/head
Timothée Rebours 4 years ago
parent aa3302e5dc
commit 185ae680b8
  1. 22
      LICENSE
  2. 19
      LICENSE.md
  3. 12
      README.md
  4. 306
      benchmarks/commonUtilities.js
  5. 65
      benchmarks/ensureIndex.js
  6. 46
      benchmarks/find.js
  7. 48
      benchmarks/findOne.js
  8. 46
      benchmarks/findWithIn.js
  9. 44
      benchmarks/insert.js
  10. 52
      benchmarks/loadDatabase.js
  11. 60
      benchmarks/remove.js
  12. 61
      benchmarks/update.js
  13. 4
      index.js
  14. 207
      lib/cursor.js
  15. 11
      lib/customUtils.js
  16. 682
      lib/datastore.js
  17. 69
      lib/executor.js
  18. 269
      lib/indexes.js
  19. 614
      lib/model.js
  20. 358
      lib/persistence.js
  21. 127
      lib/storage.js
  22. 31
      package.json
  23. 1270
      test/cursor.test.js
  24. 32
      test/customUtil.test.js
  25. 3981
      test/db.test.js
  26. 266
      test/executor.test.js
  27. 1231
      test/indexes.test.js
  28. 2
      test/mocha.opts
  29. 2150
      test/model.test.js
  30. 1311
      test/persistence.test.js
  31. 118
      test_lac/loadAndCrash.test.js
  32. 83
      test_lac/openFds.test.js

@ -1,22 +0,0 @@
(The MIT License)
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,19 @@
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Copyright (c) 2021 Seald [contact@seald.io](mailto:contact@seald.io);
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including without
limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom
the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -34,7 +34,7 @@ It is a subset of MongoDB's API (the most used operations).
* <a href="#inserting-documents">Inserting documents</a>
* <a href="#finding-documents">Finding documents</a>
* <a href="#basic-querying">Basic Querying</a>
* <a href="#operators-lt-lte-gt-gte-in-nin-ne-exists-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)</a>
* <a href="#operators-lt-lte-gt-gte-in-nin-ne-stat-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)</a>
* <a href="#array-fields">Array fields</a>
* <a href="#logical-operators-or-and-not-where">Logical operators $or, $and, $not, $where</a>
* <a href="#sorting-and-paginating">Sorting and paginating</a>
@ -236,14 +236,14 @@ db.findOne({ _id: 'id1' }, function (err, doc) {
});
```
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)
The syntax is `{ field: { $op: value } }` where `$op` is any comparison operator:
* `$lt`, `$lte`: less than, less than or equal
* `$gt`, `$gte`: greater than, greater than or equal
* `$in`: member of. `value` must be an array of values
* `$ne`, `$nin`: not equal, not a member of
* `$exists`: checks whether the document posses the property `field`. `value` should be true or false
* `$stat`: checks whether the document posses the property `field`. `value` should be true or false
* `$regex`: checks whether a string is matched by the regular expression. Contrary to MongoDB, the use of `$options` with `$regex` is not supported, because it doesn't give you more power than regex flags. Basic queries are more readable so only use the `$regex` operator when you need to use another operator with it (see example below)
```javascript
@ -262,8 +262,8 @@ db.find({ planet: { $in: ['Earth', 'Jupiter'] }}, function (err, docs) {
// docs contains Earth and Jupiter
});
// Using $exists
db.find({ satellites: { $exists: true } }, function (err, docs) {
// Using $stat
db.find({ satellites: { $stat: true } }, function (err, docs) {
// docs contains only Mars
});
@ -724,4 +724,4 @@ You don't have time? You can support NeDB by sending bitcoins to this address: 1
## License
See [License](LICENSE)
See [License](LICENSE.md)

@ -1,65 +1,58 @@
/**
* Functions that are used in several benchmark tests
*/
var customUtils = require('../lib/customUtils')
, fs = require('fs')
, path = require('path')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, executeAsap // process.nextTick or setImmediate depending on your Node version
;
const fs = require('fs')
const path = require('path')
const Datastore = require('../lib/datastore')
const Persistence = require('../lib/persistence')
let executeAsap
try {
executeAsap = setImmediate;
executeAsap = setImmediate
} catch (e) {
executeAsap = process.nextTick;
executeAsap = process.nextTick
}
/**
* Configure the benchmark
*/
module.exports.getConfiguration = function (benchDb) {
var d, n
, program = require('commander')
;
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Use an index')
.option('-m --in-memory', 'Test with an in-memory only store')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log(program.withIndex ? "Use an index" : "Don't use an index");
console.log(program.inMemory ? "Use an in-memory datastore" : "Use a persistent datastore");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log(program.inMemory ? 'Use an in-memory datastore' : 'Use a persistent datastore')
console.log('----------------------------')
d = new Datastore({ filename: benchDb
, inMemoryOnly: program.inMemory
});
return { n: n, d: d, program: program };
};
const d = new Datastore({
filename: benchDb,
inMemoryOnly: program.inMemory
})
return { n: n, d: d, program: program }
}
/**
* Ensure the workspace exists and the db datafile is empty
* Ensure the workspace stat and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
};
fs.access(filename, fs.constants.FS_OK, function (err) {
if (!err) {
fs.unlink(filename, cb)
} else { return cb() }
})
})
}
/**
* Return an array with the numbers from 0 to n-1, in a random order
@ -67,242 +60,221 @@ module.exports.prepareDb = function (filename, cb) {
* Useful to get fair tests
*/
function getRandomArray (n) {
var res = []
, i, j, temp
;
const res = []
let i
let j
let temp
for (i = 0; i < n; i += 1) { res[i] = i; }
for (i = 0; i < n; i += 1) { res[i] = i }
for (i = n - 1; i >= 1; i -= 1) {
j = Math.floor((i + 1) * Math.random());
temp = res[i];
res[i] = res[j];
res[j] = temp;
j = Math.floor((i + 1) * Math.random())
temp = res[i]
res[i] = res[j]
res[j] = temp
}
return res;
return res
};
module.exports.getRandomArray = getRandomArray;
module.exports.getRandomArray = getRandomArray
/**
* Insert a certain number of documents for testing
*/
module.exports.insertDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step('Begin inserting ' + n + ' docs');
profiler.step('Begin inserting ' + n + ' docs')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
var opsPerSecond = Math.floor(1000* n / profiler.elapsedSinceLastStep());
console.log("===== RESULT (insert) ===== " + opsPerSecond + " ops/s");
profiler.step('Finished inserting ' + n + ' docs');
profiler.insertOpsPerSecond = opsPerSecond;
return cb();
const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
console.log('===== RESULT (insert) ===== ' + opsPerSecond + ' ops/s')
profiler.step('Finished inserting ' + n + ' docs')
profiler.insertOpsPerSecond = opsPerSecond
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.insert({ docNumber: order[i] }, function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with find
*/
module.exports.findDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Finding " + n + " documents");
profiler.step('Finding ' + n + ' documents')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
console.log("===== RESULT (find) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
console.log('===== RESULT (find) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: order[i] }, function (err, docs) {
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb('One find didnt work'); }
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with find and the $in operator
*/
module.exports.findDocsWithIn = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
, ins = [], i, j
, arraySize = Math.min(10, n) // The array for $in needs to be smaller than n (inclusive)
;
const ins = []
const arraySize = Math.min(10, n)
// Preparing all the $in arrays, will take some time
for (i = 0; i < n; i += 1) {
ins[i] = [];
for (let i = 0; i < n; i += 1) {
ins[i] = []
for (j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n);
for (let j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n)
}
}
profiler.step("Finding " + n + " documents WITH $IN OPERATOR");
profiler.step('Finding ' + n + ' documents WITH $IN OPERATOR')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
console.log("===== RESULT (find with in selector) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
console.log('===== RESULT (find with in selector) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: { $in: ins[i] } }, function (err, docs) {
if (docs.length !== arraySize) { return cb('One find didnt work'); }
if (docs.length !== arraySize) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with findOne
*/
module.exports.findOneDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("FindingOne " + n + " documents");
profiler.step('FindingOne ' + n + ' documents')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
console.log("===== RESULT (findOne) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
console.log('===== RESULT (findOne) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.findOne({ docNumber: order[i] }, function (err, doc) {
if (!doc || doc.docNumber !== order[i]) { return cb('One find didnt work'); }
if (!doc || doc.docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Update documents
* options is the same as the options object for update
*/
module.exports.updateDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Updating " + n + " documents");
profiler.step('Updating ' + n + ' documents')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
console.log("===== RESULT (update) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished updating ' + n + ' docs');
return cb();
console.log('===== RESULT (update) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished updating ' + n + ' docs')
return cb()
}
// Will not actually modify the document but will take the same time
d.update({ docNumber: order[i] }, { docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One update didnt work'); }
if (err) { return cb(err) }
if (nr !== 1) { return cb(new Error('One update didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Remove documents
* options is the same as the options object for update
*/
module.exports.removeDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Removing " + n + " documents");
profiler.step('Removing ' + n + ' documents')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
// opsPerSecond corresponds to 1 insert + 1 remove, needed to keep collection size at 10,000
// We need to subtract the time taken by one insert to get the time actually taken by one remove
var opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep());
var removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log("===== RESULT (remove) ===== " + removeOpsPerSecond + " ops/s");
profiler.step('Finished removing ' + n + ' docs');
return cb();
const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
const removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log('===== RESULT (remove) ===== ' + removeOpsPerSecond + ' ops/s')
profiler.step('Finished removing ' + n + ' docs')
return cb()
}
d.remove({ docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One remove didnt work'); }
if (err) { return cb(err) }
if (nr !== 1) { return cb(new Error('One remove didnt work')) }
// eslint-disable-next-line node/handle-callback-err
d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size at n
// So actually we're calculating the average time taken by one insert + one remove
executeAsap(function () {
runFrom(i + 1);
});
});
});
runFrom(i + 1)
})
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Load database
*/
module.exports.loadDatabase = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Loading the database " + n + " times");
profiler.step('Loading the database ' + n + ' times')
function runFrom(i) {
function runFrom (i) {
if (i === n) { // Finished
console.log("===== RESULT ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished loading a database' + n + ' times');
return cb();
console.log('===== RESULT ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished loading a database' + n + ' times')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.loadDatabase(function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}

@ -1,51 +1,48 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/insert.bench.db'
, async = require('async')
, commonUtilities = require('./commonUtilities')
, execTime = require('exec-time')
, profiler = new execTime('INSERT BENCH')
, d = new Datastore(benchDb)
, program = require('commander')
, n
;
const Datastore = require('../lib/datastore')
const benchDb = 'workspace/insert.bench.db'
const async = require('async')
const commonUtilities = require('./commonUtilities')
const ExecTime = require('exec-time')
const profiler = new ExecTime('INSERT BENCH')
const d = new Datastore(benchDb)
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log('----------------------------')
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) {
var i;
if (err) { return cb(err) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) {
let i
profiler.step('Begin calling ensureIndex ' + n + ' times');
profiler.step('Begin calling ensureIndex ' + n + ' times')
for (i = 0; i < n; i += 1) {
d.ensureIndex({ fieldName: 'docNumber' });
delete d.indexes.docNumber;
d.ensureIndex({ fieldName: 'docNumber' })
delete d.indexes.docNumber
}
console.log("Average time for one ensureIndex: " + (profiler.elapsedSinceLastStep() / n) + "ms");
profiler.step('Finished calling ensureIndex ' + n + ' times');
console.log('Average time for one ensureIndex: ' + (profiler.elapsedSinceLastStep() / n) + 'ms')
profiler.step('Finished calling ensureIndex ' + n + ' times')
}
], function (err) {
profiler.step("Benchmark finished");
if (err) { return console.log("An error was encountered: ", err); }
});
profiler.step('Benchmark finished')
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/find.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FIND BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/find.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FIND BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.findDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.findDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,31 +1,27 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/findOne.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FINDONE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/findOne.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FINDONE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { setTimeout(function () {cb();}, 500); }
, async.apply(commonUtilities.findOneDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { setTimeout(function () { cb() }, 500) },
async.apply(commonUtilities.findOneDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/find.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FIND BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/find.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FIND BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.findDocsWithIn, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.findDocsWithIn, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,33 +1,31 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/insert.bench.db'
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('INSERT BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/insert.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('INSERT BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
let n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (err) { return cb(err) }
if (config.program.withIndex) {
d.ensureIndex({ fieldName: 'docNumber' });
n = 2 * n; // We will actually insert twice as many documents
d.ensureIndex({ fieldName: 'docNumber' })
n = 2 * n // We will actually insert twice as many documents
// because the index is slower when the collection is already
// big. So the result given by the algorithm will be a bit worse than
// actual performance
}
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/loaddb.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, commonUtilities = require('./commonUtilities')
, execTime = require('exec-time')
, profiler = new execTime('LOADDB BENCH')
, d = new Datastore(benchDb)
, program = require('commander')
, n
;
const Datastore = require('../lib/datastore')
const benchDb = 'workspace/loaddb.bench.db'
const async = require('async')
const commonUtilities = require('./commonUtilities')
const ExecTime = require('exec-time')
const profiler = new ExecTime('LOADDB BENCH')
const d = new Datastore(benchDb)
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log(program.withIndex ? "Use an index" : "Don't use an index");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log('----------------------------')
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
d.loadDatabase(cb);
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.loadDatabase, d, n, profiler)
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(cb)
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.loadDatabase, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/remove.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('REMOVE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/remove.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('REMOVE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with remove only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); }
, async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler)
// Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function () { return cb(); }); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); }
, async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler)
// Test with remove only one document
function (cb) { profiler.step('MULTI: FALSE'); return cb() },
async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler),
// Test with multiple documents
function (cb) { d.remove({}, { multi: true }, function () { return cb() }) },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { profiler.step('MULTI: TRUE'); return cb() },
async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,39 +1,36 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/update.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('UPDATE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/update.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('UPDATE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with update only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); }
, async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler)
// Test with update only one document
function (cb) { profiler.step('MULTI: FALSE'); return cb() },
async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler),
// Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function (err) { return cb(); }); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); }
, async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler)
// Test with multiple documents
// eslint-disable-next-line node/handle-callback-err
function (cb) { d.remove({}, { multi: true }, function (err) { return cb() }) },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { profiler.step('MULTI: TRUE'); return cb() },
async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

@ -1,3 +1,3 @@
var Datastore = require('./lib/datastore');
const Datastore = require('./lib/datastore')
module.exports = Datastore;
module.exports = Datastore

@ -1,136 +1,132 @@
/**
* Manage access to data, be it to find, update or remove it
*/
var model = require('./model')
, _ = require('underscore')
;
const model = require('./model')
const _ = require('underscore')
/**
class Cursor {
/**
* Create a new cursor for this collection
* @param {Datastore} db - The datastore this cursor is bound to
* @param {Query} query - The query this cursor will operate on
* @param {Function} execFn - Handler to be executed after cursor has found the results and before the callback passed to find/findOne/update/remove
*/
function Cursor (db, query, execFn) {
this.db = db;
this.query = query || {};
if (execFn) { this.execFn = execFn; }
}
constructor (db, query, execFn) {
this.db = db
this.query = query || {}
if (execFn) { this.execFn = execFn }
}
/**
/**
* Set a limit to the number of results
*/
Cursor.prototype.limit = function(limit) {
this._limit = limit;
return this;
};
limit (limit) {
this._limit = limit
return this
}
/**
/**
* Skip a the number of results
*/
Cursor.prototype.skip = function(skip) {
this._skip = skip;
return this;
};
skip (skip) {
this._skip = skip
return this
}
/**
/**
* Sort results of the query
* @param {SortQuery} sortQuery - SortQuery is { field: order }, field can use the dot-notation, order is 1 for ascending and -1 for descending
*/
Cursor.prototype.sort = function(sortQuery) {
this._sort = sortQuery;
return this;
};
sort (sortQuery) {
this._sort = sortQuery
return this
}
/**
/**
* Add the use of a projection
* @param {Object} projection - MongoDB-style projection. {} means take all fields. Then it's { key1: 1, key2: 1 } to take only key1 and key2
* { key1: 0, key2: 0 } to omit only key1 and key2. Except _id, you can't mix takes and omits
*/
Cursor.prototype.projection = function(projection) {
this._projection = projection;
return this;
};
projection (projection) {
this._projection = projection
return this
}
/**
/**
* Apply the projection
*/
Cursor.prototype.project = function (candidates) {
var res = [], self = this
, keepId, action, keys
;
project (candidates) {
const res = []
const self = this
let action
if (this._projection === undefined || Object.keys(this._projection).length === 0) {
return candidates;
return candidates
}
keepId = this._projection._id === 0 ? false : true;
this._projection = _.omit(this._projection, '_id');
const keepId = this._projection._id !== 0
this._projection = _.omit(this._projection, '_id')
// Check for consistency
keys = Object.keys(this._projection);
const keys = Object.keys(this._projection)
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error("Can't both keep and omit fields except for _id"); }
action = self._projection[k];
});
if (action !== undefined && self._projection[k] !== action) { throw new Error('Can\'t both keep and omit fields except for _id') }
action = self._projection[k]
})
// Do the actual projection
candidates.forEach(function (candidate) {
var toPush;
let toPush
if (action === 1) { // pick-type projection
toPush = { $set: {} };
toPush = { $set: {} }
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
toPush.$set[k] = model.getDotValue(candidate, k)
if (toPush.$set[k] === undefined) { delete toPush.$set[k] }
})
toPush = model.modify({}, toPush)
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
toPush = { $unset: {} }
keys.forEach(function (k) { toPush.$unset[k] = true })
toPush = model.modify(candidate, toPush)
}
if (keepId) {
toPush._id = candidate._id;
toPush._id = candidate._id
} else {
delete toPush._id;
delete toPush._id
}
res.push(toPush);
});
return res;
};
res.push(toPush)
})
return res
}
/**
/**
* Get all matching elements
* Will return pointers to matched elements (shallow copies), returning full copies is the role of find or findOne
* This is an internal function, use exec which uses the executor
*
* @param {Function} callback - Signature: err, results
*/
Cursor.prototype._exec = function(_callback) {
var res = [], added = 0, skipped = 0, self = this
, error = null
, i, keys, key
;
_exec (_callback) {
let res = []
let added = 0
let skipped = 0
const self = this
let error = null
let i
let keys
let key
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback);
return self.execFn(error, res, _callback)
} else {
return _callback(error, res);
return _callback(error, res)
}
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err); }
if (err) { return callback(err) }
try {
for (i = 0; i < candidates.length; i += 1) {
@ -138,67 +134,68 @@ Cursor.prototype._exec = function(_callback) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1;
skipped += 1
} else {
res.push(candidates[i]);
added += 1;
if (self._limit && self._limit <= added) { break; }
res.push(candidates[i])
added += 1
if (self._limit && self._limit <= added) { break }
}
} else {
res.push(candidates[i]);
res.push(candidates[i])
}
}
}
} catch (err) {
return callback(err);
return callback(err)
}
// Apply all sorts
if (self._sort) {
keys = Object.keys(self._sort);
keys = Object.keys(self._sort)
// Sorting
var criteria = [];
const criteria = []
for (i = 0; i < keys.length; i++) {
key = keys[i];
criteria.push({ key: key, direction: self._sort[key] });
key = keys[i]
criteria.push({ key: key, direction: self._sort[key] })
}
res.sort(function(a, b) {
var criterion, compare, i;
res.sort(function (a, b) {
let criterion
let compare
let i
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i];
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings);
criterion = criteria[i]
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings)
if (compare !== 0) {
return compare;
return compare
}
}
return 0;
});
return 0
})
// Applying limit and skip
var limit = self._limit || res.length
, skip = self._skip || 0;
const limit = self._limit || res.length
const skip = self._skip || 0
res = res.slice(skip, skip + limit);
res = res.slice(skip, skip + limit)
}
// Apply projection
try {
res = self.project(res);
res = self.project(res)
} catch (e) {
error = e;
res = undefined;
error = e
res = undefined
}
return callback(error, res);
});
};
Cursor.prototype.exec = function () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments });
};
return callback(error, res)
})
}
exec () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments })
}
}
// Interface
module.exports = Cursor;
module.exports = Cursor

@ -1,5 +1,4 @@
var crypto = require('crypto')
;
const crypto = require('crypto')
/**
* Return a random alphanumerical string of length len
@ -12,11 +11,9 @@ var crypto = require('crypto')
function uid (len) {
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
.toString('base64')
.replace(/[+\/]/g, '')
.slice(0, len);
.replace(/[+/]/g, '')
.slice(0, len)
}
// Interface
module.exports.uid = uid;
module.exports.uid = uid

File diff suppressed because it is too large Load Diff

@ -1,48 +1,45 @@
/**
* Responsible for sequentially executing actions on the database
*/
const async = require('async')
var async = require('async')
;
function Executor () {
this.buffer = [];
this.ready = false;
class Executor {
constructor () {
this.buffer = []
this.ready = false
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
var newArguments = [];
const newArguments = []
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (var i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]); }
var lastArg = task.arguments[task.arguments.length - 1];
for (let i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]) }
const lastArg = task.arguments[task.arguments.length - 1]
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
// Callback was supplied
newArguments[newArguments.length - 1] = function () {
if (typeof setImmediate === 'function') {
setImmediate(cb);
setImmediate(cb)
} else {
process.nextTick(cb);
process.nextTick(cb)
}
lastArg.apply(null, arguments)
}
lastArg.apply(null, arguments);
};
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callbback
newArguments[newArguments.length - 1] = function () { cb(); };
// false/undefined/null supplied as callback
newArguments[newArguments.length - 1] = function () { cb() }
} else {
// Nothing supplied as callback
newArguments.push(function () { cb(); });
newArguments.push(function () { cb() })
}
task.fn.apply(task.this, newArguments)
}, 1)
}
task.fn.apply(task.this, newArguments);
}, 1);
}
/**
/**
* If executor is ready, queue task (and process it immediately if executor was idle)
* If not, buffer task for later processing
* @param {Object} task
@ -52,27 +49,25 @@ function Executor () {
* and the last argument cannot be false/undefined/null
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
Executor.prototype.push = function (task, forceQueuing) {
push (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task);
this.queue.push(task)
} else {
this.buffer.push(task);
this.buffer.push(task)
}
}
};
/**
/**
* Queue all tasks in buffer (in the same order they came in)
* Automatically sets executor as ready
*/
Executor.prototype.processBuffer = function () {
var i;
this.ready = true;
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]); }
this.buffer = [];
};
processBuffer () {
let i
this.ready = true
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]) }
this.buffer = []
}
}
// Interface
module.exports = Executor;
module.exports = Executor

@ -1,31 +1,29 @@
var BinarySearchTree = require('binary-search-tree').AVLTree
, model = require('./model')
, _ = require('underscore')
, util = require('util')
;
const BinarySearchTree = require('@seald-io/binary-search-tree').BinarySearchTree
const model = require('./model')
const _ = require('underscore')
/**
* Two indexed pointers are equal iif they point to the same place
*/
function checkValueEquality (a, b) {
return a === b;
return a === b
}
/**
* Type-aware projection
*/
function projectForUnique (elt) {
if (elt === null) { return '$null'; }
if (typeof elt === 'string') { return '$string' + elt; }
if (typeof elt === 'boolean') { return '$boolean' + elt; }
if (typeof elt === 'number') { return '$number' + elt; }
if (util.isArray(elt)) { return '$date' + elt.getTime(); }
if (elt === null) { return '$null' }
if (typeof elt === 'string') { return '$string' + elt }
if (typeof elt === 'boolean') { return '$boolean' + elt }
if (typeof elt === 'number') { return '$number' + elt }
if (Array.isArray(elt)) { return '$date' + elt.getTime() }
return elt; // Arrays and objects, will check for pointer equality
return elt // Arrays and objects, will check for pointer equality
}
/**
class Index {
/**
* Create a new index
* All methods on an index guarantee that either the whole operation was successful and the index changed
* or the operation was unsuccessful and an error is thrown while the index is unchanged
@ -33,147 +31,153 @@ function projectForUnique (elt) {
* @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
* @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
*/
function Index (options) {
this.fieldName = options.fieldName;
this.unique = options.unique || false;
this.sparse = options.sparse || false;
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
constructor (options) {
this.fieldName = options.fieldName
this.unique = options.unique || false
this.sparse = options.sparse || false
this.reset(); // No data in the beginning
}
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality }
this.reset() // No data in the beginning
}
/**
/**
* Reset an index
* @param {Document or Array of documents} newData Optional, data to initialize the index with
* If an error is thrown during insertion, the index is not modified
*/
Index.prototype.reset = function (newData) {
this.tree = new BinarySearchTree(this.treeOptions);
if (newData) { this.insert(newData); }
};
reset (newData) {
this.tree = new BinarySearchTree(this.treeOptions)
if (newData) { this.insert(newData) }
}
/**
/**
* Insert a new document in the index
* If an array is passed, we insert all its elements (if one insertion fails the index is not modified)
* O(log(n))
*/
Index.prototype.insert = function (doc) {
var key, self = this
, keys, i, failingI, error
;
if (util.isArray(doc)) { this.insertMultipleDocs(doc); return; }
insert (doc) {
let keys
let i
let failingI
let error
if (Array.isArray(doc)) {
this.insertMultipleDocs(doc)
return
}
key = model.getDotValue(doc, this.fieldName);
const key = model.getDotValue(doc, this.fieldName)
// We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return; }
if (key === undefined && this.sparse) { return }
if (!util.isArray(key)) {
this.tree.insert(key, doc);
if (!Array.isArray(key)) {
this.tree.insert(key, doc)
} else {
// If an insert fails due to a unique constraint, roll back all inserts before it
keys = _.uniq(key, projectForUnique);
keys = _.uniq(key, projectForUnique)
for (i = 0; i < keys.length; i += 1) {
try {
this.tree.insert(keys[i], doc);
this.tree.insert(keys[i], doc)
} catch (e) {
error = e;
failingI = i;
break;
error = e
failingI = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.tree.delete(keys[i], doc);
this.tree.delete(keys[i], doc)
}
throw error;
throw error
}
}
}
};
/**
/**
* Insert an array of documents in the index
* If a constraint is violated, the changes should be rolled back and an error thrown
*
* @API private
*/
Index.prototype.insertMultipleDocs = function (docs) {
var i, error, failingI;
insertMultipleDocs (docs) {
let i
let error
let failingI
for (i = 0; i < docs.length; i += 1) {
try {
this.insert(docs[i]);
this.insert(docs[i])
} catch (e) {
error = e;
failingI = i;
break;
error = e
failingI = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.remove(docs[i]);
this.remove(docs[i])
}
throw error;
throw error
}
}
};
/**
/**
* Remove a document from the index
* If an array is passed, we remove all its elements
* The remove operation is safe with regards to the 'unique' constraint
* O(log(n))
*/
Index.prototype.remove = function (doc) {
var key, self = this;
remove (doc) {
const self = this
if (util.isArray(doc)) { doc.forEach(function (d) { self.remove(d); }); return; }
if (Array.isArray(doc)) {
doc.forEach(function (d) { self.remove(d) })
return
}
key = model.getDotValue(doc, this.fieldName);
const key = model.getDotValue(doc, this.fieldName)
if (key === undefined && this.sparse) { return; }
if (key === undefined && this.sparse) { return }
if (!util.isArray(key)) {
this.tree.delete(key, doc);
if (!Array.isArray(key)) {
this.tree.delete(key, doc)
} else {
_.uniq(key, projectForUnique).forEach(function (_key) {
self.tree.delete(_key, doc);
});
self.tree.delete(_key, doc)
})
}
}
};
/**
/**
* Update a document in the index
* If a constraint is violated, changes are rolled back and an error thrown
* Naive implementation, still in O(log(n))
*/
Index.prototype.update = function (oldDoc, newDoc) {
if (util.isArray(oldDoc)) { this.updateMultipleDocs(oldDoc); return; }
update (oldDoc, newDoc) {
if (Array.isArray(oldDoc)) {
this.updateMultipleDocs(oldDoc)
return
}
this.remove(oldDoc);
this.remove(oldDoc)
try {
this.insert(newDoc);
this.insert(newDoc)
} catch (e) {
this.insert(oldDoc);
throw e;
this.insert(oldDoc)
throw e
}
}
};
/**
/**
* Update multiple documents in the index
* If a constraint is violated, the changes need to be rolled back
* and an error thrown
@ -181,114 +185,111 @@ Index.prototype.update = function (oldDoc, newDoc) {
*
* @API private
*/
Index.prototype.updateMultipleDocs = function (pairs) {
var i, failingI, error;
updateMultipleDocs (pairs) {
let i
let failingI
let error
for (i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc);
this.remove(pairs[i].oldDoc)
}
for (i = 0; i < pairs.length; i += 1) {
try {
this.insert(pairs[i].newDoc);
this.insert(pairs[i].newDoc)
} catch (e) {
error = e;
failingI = i;
break;
error = e
failingI = i
break
}
}
// If an error was raised, roll back changes in the inverse order
if (error) {
for (i = 0; i < failingI; i += 1) {
this.remove(pairs[i].newDoc);
this.remove(pairs[i].newDoc)
}
for (i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc);
this.insert(pairs[i].oldDoc)
}
throw error;
throw error
}
}
};
/**
/**
* Revert an update
*/
Index.prototype.revertUpdate = function (oldDoc, newDoc) {
var revert = [];
revertUpdate (oldDoc, newDoc) {
const revert = []
if (!util.isArray(oldDoc)) {
this.update(newDoc, oldDoc);
if (!Array.isArray(oldDoc)) {
this.update(newDoc, oldDoc)
} else {
oldDoc.forEach(function (pair) {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc });
});
this.update(revert);
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc })
})
this.update(revert)
}
}
};
/**
/**
* Get all documents in index whose key match value (if it is a Thing) or one of the elements of value (if it is an array of Things)
* @param {Thing} value Value to match the key against
* @return {Array of documents}
*/
Index.prototype.getMatching = function (value) {
var self = this;
getMatching (value) {
const self = this
if (!util.isArray(value)) {
return self.tree.search(value);
if (!Array.isArray(value)) {
return self.tree.search(value)
} else {
var _res = {}, res = [];
const _res = {}
const res = []
value.forEach(function (v) {
self.getMatching(v).forEach(function (doc) {
_res[doc._id] = doc;
});
});
_res[doc._id] = doc
})
})
Object.keys(_res).forEach(function (_id) {
res.push(_res[_id]);
});
res.push(_res[_id])
})
return res;
return res
}
}
};
/**
/**
* Get all documents in index whose key is between bounds are they are defined by query
* Documents are sorted by key
* @param {Query} query
* @return {Array of documents}
*/
Index.prototype.getBetweenBounds = function (query) {
return this.tree.betweenBounds(query);
};
getBetweenBounds (query) {
return this.tree.betweenBounds(query)
}
/**
/**
* Get all elements in the index
* @return {Array of documents}
*/
Index.prototype.getAll = function () {
var res = [];
getAll () {
const res = []
this.tree.executeOnEveryNode(function (node) {
var i;
let i
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i]);
res.push(node.data[i])
}
});
return res;
};
})
return res
}
}
// Interface
module.exports = Index;
module.exports = Index

File diff suppressed because it is too large Load Diff

@ -4,253 +4,201 @@
* * Persistence.loadDatabase(callback) and callback has signature err
* * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
*/
var storage = require('./storage')
, path = require('path')
, model = require('./model')
, async = require('async')
, customUtils = require('./customUtils')
, Index = require('./indexes')
;
/**
const storage = require('./storage')
const path = require('path')
const model = require('./model')
const async = require('async')
const customUtils = require('./customUtils')
const Index = require('./indexes')
class Persistence {
/**
* Create a new Persistence object for database options.db
* @param {Datastore} options.db
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/
function Persistence (options) {
var i, j, randomString;
constructor (options) {
let i
let j
let randomString
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
this.db = options.db
this.inMemoryOnly = this.db.inMemoryOnly
this.filename = this.db.filename
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss");
throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss");
throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
}
this.afterSerialization = options.afterSerialization || function (s) { return s; };
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
this.afterSerialization = options.afterSerialization || function (s) { return s }
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s }
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i);
randomString = customUtils.uid(i)
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss");
throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss')
}
}
}
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log("==================================================================");
console.log("WARNING: The nodeWebkitAppName option is deprecated");
console.log("To get the path to the directory where Node Webkit stores the data");
console.log("for your app, use the internal nw.gui module like this");
console.log("require('nw.gui').App.dataPath");
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
console.log('==================================================================')
console.log('WARNING: The nodeWebkitAppName option is deprecated')
console.log('To get the path to the directory where Node Webkit stores the data')
console.log('for your app, use the internal nw.gui module like this')
console.log('require(\'nw.gui\').App.dataPath')
console.log('See https://github.com/rogerwang/node-webkit/issues/500')
console.log('==================================================================')
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename)
}
};
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
Persistence.ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
storage.mkdirp(dir, function (err) { return callback(err); });
};
/**
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
* data for this application. Probably the best place to store data
*/
Persistence.getNWAppFilename = function (appName, relativeFilename) {
var home;
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA;
if (!home) { throw new Error("Couldn't find the base application data folder"); }
home = path.join(home, appName);
break;
case 'darwin':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, 'Library', 'Application Support', appName);
break;
case 'linux':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, '.config', appName);
break;
default:
throw new Error("Can't use the Node Webkit relative path for platform " + process.platform);
break;
}
return path.join(home, 'nedb-data', relativeFilename);
}
/**
/**
* Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection
* while the data file is append-only so it may grow larger
* @param {Function} cb Optional callback, signature: err
*/
Persistence.prototype.persistCachedDatabase = function (cb) {
var callback = cb || function () {}
, toPersist = ''
, self = this
;
persistCachedDatabase (cb) {
const callback = cb || function () {}
let toPersist = ''
const self = this
if (this.inMemoryOnly) { return callback(null); }
if (this.inMemoryOnly) { return callback(null) }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({
$$indexCreated: {
fieldName: fieldName,
unique: self.db.indexes[fieldName].unique,
sparse: self.db.indexes[fieldName].sparse
}
});
})) + '\n'
}
})
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err); }
self.db.emit('compaction.done');
return callback(null);
});
};
if (err) { return callback(err) }
self.db.emit('compaction.done')
return callback(null)
})
}
/**
/**
* Queue a rewrite of the datafile
*/
Persistence.prototype.compactDatafile = function () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
};
compactDatafile () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] })
}
/**
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
Persistence.prototype.setAutocompactionInterval = function (interval) {
var self = this
, minInterval = 5000
, realInterval = Math.max(interval || 0, minInterval)
;
setAutocompactionInterval (interval) {
const self = this
const minInterval = 5000
const realInterval = Math.max(interval || 0, minInterval)
this.stopAutocompaction();
this.stopAutocompaction()
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile();
}, realInterval);
};
self.compactDatafile()
}, realInterval)
}
/**
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
Persistence.prototype.stopAutocompaction = function () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
};
stopAutocompaction () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId) }
}
/**
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
*/
Persistence.prototype.persistNewState = function (newDocs, cb) {
var self = this
, toPersist = ''
, callback = cb || function () {}
;
persistNewState (newDocs, cb) {
const self = this
let toPersist = ''
const callback = cb || function () {}
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
if (self.inMemoryOnly) { return callback(null) }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
if (toPersist.length === 0) { return callback(null); }
if (toPersist.length === 0) { return callback(null) }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err);
});
};
return callback(err)
})
}
/**
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Persistence.prototype.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, tdata = []
, i
, indexes = {}
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
;
treatRawData (rawData) {
const data = rawData.split('\n')
const dataById = {}
const tdata = []
let i
const indexes = {}
let corruptItems = -1
for (i = 0; i < data.length; i += 1) {
var doc;
let doc
try {
doc = model.deserialize(this.beforeDeserialization(data[i]));
doc = model.deserialize(this.beforeDeserialization(data[i]))
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
delete dataById[doc._id]
} else {
dataById[doc._id] = doc;
dataById[doc._id] = doc
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
} else if (typeof doc.$$indexRemoved === "string") {
delete indexes[doc.$$indexRemoved];
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
} else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved]
}
} catch (e) {
corruptItems += 1;
corruptItems += 1
}
}
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss");
throw new Error('More than ' + Math.floor(100 * this.corruptAlertThreshold) + '% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss')
}
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k]);
});
return { data: tdata, indexes: indexes };
};
tdata.push(dataById[k])
})
return { data: tdata, indexes: indexes }
}
/**
/**
* Load the database
* 1) Create all indexes
* 2) Insert all data
@ -260,55 +208,99 @@ Persistence.prototype.treatRawData = function (rawData) {
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*/
Persistence.prototype.loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
loadDatabase (cb) {
const callback = cb || function () {}
const self = this
self.db.resetIndexes();
self.db.resetIndexes()
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
if (self.inMemoryOnly) { return callback(null) }
async.waterfall([
function (cb) {
// eslint-disable-next-line node/handle-callback-err
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
// TODO: handle error
// eslint-disable-next-line node/handle-callback-err
storage.ensureDatafileIntegrity(self.filename, function (err) {
// TODO: handle error
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
if (err) { return cb(err) }
let treatedData
try {
var treatedData = self.treatRawData(rawData);
treatedData = self.treatRawData(rawData)
} catch (e) {
return cb(e);
return cb(e)
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);
});
self.db.indexes[key] = new Index(treatedData.indexes[key])
})
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data);
self.db.resetIndexes(treatedData.data)
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
return cb(e);
self.db.resetIndexes() // Rollback any index which didn't fail
return cb(e)
}
self.db.persistence.persistCachedDatabase(cb);
});
});
});
self.db.persistence.persistCachedDatabase(cb)
})
})
})
}
], function (err) {
if (err) { return callback(err); }
if (err) { return callback(err) }
self.db.executor.processBuffer()
return callback(null)
})
}
/**
* Check if a directory stat and create it on the fly if it is not the case
* cb is optional, signature: err
*/
static ensureDirectoryExists (dir, cb) {
const callback = cb || function () {}
self.db.executor.processBuffer();
return callback(null);
});
};
storage.mkdir(dir, { recursive: true }, err => { callback(err) })
}
/**
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
* data for this application. Probably the best place to store data
*/
static getNWAppFilename (appName, relativeFilename) {
let home
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA
if (!home) { throw new Error('Couldn\'t find the base application data folder') }
home = path.join(home, appName)
break
case 'darwin':
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
home = path.join(home, 'Library', 'Application Support', appName)
break
case 'linux':
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
home = path.join(home, '.config', appName)
break
default:
throw new Error('Can\'t use the Node Webkit relative path for platform ' + process.platform)
}
return path.join(home, 'nedb-data', relativeFilename)
}
}
// Interface
module.exports = Persistence;
module.exports = Persistence

@ -6,34 +6,30 @@
* This version is the Node.js/Node Webkit version
* It's essentially fs, mkdirp and crash safe write and read functions
*/
var fs = require('fs')
, mkdirp = require('mkdirp')
, async = require('async')
, path = require('path')
, storage = {}
;
storage.exists = fs.exists;
storage.rename = fs.rename;
storage.writeFile = fs.writeFile;
storage.unlink = fs.unlink;
storage.appendFile = fs.appendFile;
storage.readFile = fs.readFile;
storage.mkdirp = mkdirp;
const fs = require('fs')
const async = require('async')
const path = require('path')
const storage = {}
// eslint-disable-next-line node/no-callback-literal
storage.exists = (path, cb) => fs.access(path, fs.constants.F_OK, (err) => { cb(!err) })
storage.rename = fs.rename
storage.writeFile = fs.writeFile
storage.unlink = fs.unlink
storage.appendFile = fs.appendFile
storage.readFile = fs.readFile
storage.mkdir = fs.mkdir
/**
* Explicit name ...
*/
storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
if (!exists) { return callback(null) }
storage.unlink(file, function (err) { return callback(err) })
})
}
/**
* Flush data in OS buffer to storage if corresponding option is set
@ -42,36 +38,36 @@ storage.ensureFileDoesntExist = function (file, callback) {
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/
storage.flushToStorage = function (options, callback) {
var filename, flags;
let filename
let flags
if (typeof options === 'string') {
filename = options;
flags = 'r+';
filename = options
flags = 'r+'
} else {
filename = options.filename;
flags = options.isDir ? 'r' : 'r+';
filename = options.filename
flags = options.isDir ? 'r' : 'r+'
}
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null); }
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null) }
fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err); }
if (err) { return callback(err) }
fs.fsync(fd, function (errFS) {
fs.close(fd, function (errC) {
if (errFS || errC) {
var e = new Error('Failed to flush to storage');
e.errorOnFsync = errFS;
e.errorOnClose = errC;
return callback(e);
const e = new Error('Failed to flush to storage')
e.errorOnFsync = errFS
e.errorOnClose = errC
return callback(e)
} else {
return callback(null);
return callback(null)
}
});
});
});
};
})
})
})
}
/**
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
@ -80,31 +76,30 @@ storage.flushToStorage = function (options, callback) {
* @param {Function} cb Optional callback, signature: err
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~';
const callback = cb || function () {}
const tempFilename = filename + '~'
async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, function (cb) {
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }),
function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err); });
storage.flushToStorage(filename, function (err) { return cb(err) })
} else {
return cb();
}
});
return cb()
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err); })
};
})
},
function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err) })
},
async.apply(storage.flushToStorage, tempFilename),
function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err) })
},
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err) })
}
/**
* Ensure the datafile contains all the data, even if there was a crash during a full file write
@ -112,25 +107,23 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~';
const tempFilename = filename + '~'
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
if (filenameExists) { return callback(null) }
storage.exists(tempFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err) })
}
// Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err); });
});
});
};
storage.rename(tempFilename, filename, function (err) { return callback(err) })
})
})
}
// Interface
module.exports = storage;
module.exports = storage

@ -20,27 +20,36 @@
"url": "git@github.com:louischatriot/nedb.git"
},
"dependencies": {
"@seald-io/binary-search-tree": "^1.0.0",
"async": "0.2.10",
"binary-search-tree": "0.2.5",
"localforage": "^1.3.0",
"mkdirp": "~0.5.1",
"underscore": "~1.4.4"
"localforage": "^1.9.0",
"underscore": "^1.13.1"
},
"devDependencies": {
"chai": "^3.2.0",
"mocha": "1.4.x",
"chai": "^4.3.4",
"commander": "1.1.1",
"exec-time": "0.0.2",
"mocha": "^8.4.0",
"request": "2.9.x",
"semver": "^7.3.5",
"sinon": "1.3.x",
"exec-time": "0.0.2",
"commander": "1.1.1"
"standard": "^16.0.3"
},
"scripts": {
"test": "./node_modules/.bin/mocha --reporter spec --timeout 10000"
"test": "mocha --reporter spec --timeout 10000"
},
"main": "index",
"main": "index.js",
"browser": {
"./lib/customUtils.js": "./browser-version/browser-specific/lib/customUtils.js",
"./lib/storage.js": "./browser-version/browser-specific/lib/storage.js"
},
"license": "SEE LICENSE IN LICENSE"
"license": "MIT",
"publishConfig": {
"access": "public"
},
"standard": {
"ignore": [
"browser-version"
]
}
}

File diff suppressed because it is too large Load Diff

@ -1,26 +1,20 @@
var should = require('chai').should()
, assert = require('chai').assert
, customUtils = require('../lib/customUtils')
, fs = require('fs')
;
/* eslint-env mocha */
const chai = require('chai')
const customUtils = require('../lib/customUtils')
chai.should()
describe('customUtils', function () {
describe('uid', function () {
it('Generates a string of the expected length', function () {
customUtils.uid(3).length.should.equal(3);
customUtils.uid(16).length.should.equal(16);
customUtils.uid(42).length.should.equal(42);
customUtils.uid(1000).length.should.equal(1000);
});
customUtils.uid(3).length.should.equal(3)
customUtils.uid(16).length.should.equal(16)
customUtils.uid(42).length.should.equal(42)
customUtils.uid(1000).length.should.equal(1000)
})
// Very small probability of conflict
it('Generated uids should not be the same', function () {
customUtils.uid(56).should.not.equal(customUtils.uid(56));
});
});
});
customUtils.uid(56).should.not.equal(customUtils.uid(56))
})
})
})

File diff suppressed because it is too large Load Diff

@ -1,213 +1,215 @@
var should = require('chai').should()
, assert = require('chai').assert
, testDb = 'workspace/test.db'
, fs = require('fs')
, path = require('path')
, _ = require('underscore')
, async = require('async')
, model = require('../lib/model')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
;
/* eslint-env mocha */
const chai = require('chai')
const testDb = 'workspace/test.db'
const fs = require('fs')
const path = require('path')
const async = require('async')
const Datastore = require('../lib/datastore')
const Persistence = require('../lib/persistence')
const { assert } = chai
chai.should()
// Test that even if a callback throws an exception, the next DB operations will still be executed
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testThrowInCallback (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException');
const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException');
process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track
});
})
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err) {
process.nextTick(function () {
// eslint-disable-next-line node/handle-callback-err
d.insert({ bar: 1 }, function (err) {
process.removeAllListeners('uncaughtException');
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]);
process.removeAllListeners('uncaughtException')
for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
}
done();
});
});
done()
})
})
throw new Error('Some error');
});
throw new Error('Some error')
})
}
// Test that if the callback is falsy, the next DB operations will still be executed
function testFalsyCallback (d, done) {
d.insert({ a: 1 }, null);
d.insert({ a: 1 }, null)
process.nextTick(function () {
d.update({ a: 1 }, { a: 2 }, {}, null);
d.update({ a: 1 }, { a: 2 }, {}, null)
process.nextTick(function () {
d.update({ a: 2 }, { a: 1 }, null);
d.update({ a: 2 }, { a: 1 }, null)
process.nextTick(function () {
d.remove({ a: 2 }, {}, null);
d.remove({ a: 2 }, {}, null)
process.nextTick(function () {
d.remove({ a: 2 }, null);
d.remove({ a: 2 }, null)
process.nextTick(function () {
d.find({}, done);
});
});
});
});
});
d.find({}, done)
})
})
})
})
})
}
// Test that operations are executed in the right order
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testRightOrder (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException');
const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException');
process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track
});
})
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs.length.should.equal(0);
docs.length.should.equal(0)
d.insert({ a: 1 }, function () {
d.update({ a: 1 }, { a: 2 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs[0].a.should.equal(2);
docs[0].a.should.equal(2)
process.nextTick(function () {
d.update({ a: 2 }, { a: 3 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs[0].a.should.equal(3);
docs[0].a.should.equal(3)
process.removeAllListeners('uncaughtException');
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]);
process.removeAllListeners('uncaughtException')
for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
}
done();
});
});
});
done()
})
})
})
throw new Error('Some error');
});
});
});
});
throw new Error('Some error')
})
})
})
})
}
// Note: The following test does not have any assertion because it
// is meant to address the deprecation warning:
// (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
// see
var testEventLoopStarvation = function(d, done){
var times = 1001;
var i = 0;
while ( i <times) {
i++;
d.find({"bogus": "search"}, function (err, docs) {
});
const testEventLoopStarvation = function (d, done) {
const times = 1001
let i = 0
while (i < times) {
i++
// eslint-disable-next-line node/handle-callback-err
d.find({ bogus: 'search' }, function (err, docs) {
})
}
done();
};
done()
}
// Test that operations are executed in the right order even with no callback
function testExecutorWorksWithoutCallback (d, done) {
d.insert({ a: 1 });
d.insert({ a: 2 }, false);
d.insert({ a: 1 })
d.insert({ a: 2 }, false)
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs.length.should.equal(2);
done();
});
docs.length.should.equal(2)
done()
})
}
describe('Executor', function () {
describe('With persistent database', function () {
var d;
let d
beforeEach(function (done) {
d = new Datastore({ filename: testDb });
d.filename.should.equal(testDb);
d.inMemoryOnly.should.equal(false);
d = new Datastore({ filename: testDb })
d.filename.should.equal(testDb)
d.inMemoryOnly.should.equal(false)
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
fs.exists(testDb, function (exists) {
if (exists) {
fs.unlink(testDb, cb);
} else { return cb(); }
});
});
}
, function (cb) {
fs.access(testDb, fs.constants.F_OK, function (err) {
if (!err) {
fs.unlink(testDb, cb)
} else { return cb() }
})
})
},
function (cb) {
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return cb();
});
assert.isNull(err)
d.getAllData().length.should.equal(0)
return cb()
})
}
], done);
});
it('A throw in a callback doesnt prevent execution of next operations', function(done) {
testThrowInCallback(d, done);
});
], done)
})
it('A falsy callback doesnt prevent execution of next operations', function(done) {
testFalsyCallback(d, done);
});
it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testThrowInCallback(d, done)
})
it('Operations are executed in the right order', function(done) {
testRightOrder(d, done);
});
it('A falsy callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done)
})
it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function(done){
testEventLoopStarvation(d, done);
});
it('Operations are executed in the right order', function (done) {
testRightOrder(d, done)
})
it('Works in the right order even with no supplied callback', function(done){
testExecutorWorksWithoutCallback(d, done);
});
}); // ==== End of 'With persistent database' ====
it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function (done) {
testEventLoopStarvation(d, done)
})
it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done)
})
}) // ==== End of 'With persistent database' ====
describe('With non persistent database', function () {
var d;
let d
beforeEach(function (done) {
d = new Datastore({ inMemoryOnly: true });
d.inMemoryOnly.should.equal(true);
d = new Datastore({ inMemoryOnly: true })
d.inMemoryOnly.should.equal(true)
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return done();
});
});
it('A throw in a callback doesnt prevent execution of next operations', function(done) {
testThrowInCallback(d, done);
});
it('A falsy callback doesnt prevent execution of next operations', function(done) {
testFalsyCallback(d, done);
});
it('Operations are executed in the right order', function(done) {
testRightOrder(d, done);
});
it('Works in the right order even with no supplied callback', function(done){
testExecutorWorksWithoutCallback(d, done);
});
}); // ==== End of 'With non persistent database' ====
});
assert.isNull(err)
d.getAllData().length.should.equal(0)
return done()
})
})
it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testThrowInCallback(d, done)
})
it('A falsy callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done)
})
it('Operations are executed in the right order', function (done) {
testRightOrder(d, done)
})
it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done)
})
}) // ==== End of 'With non persistent database' ====
})

File diff suppressed because it is too large Load Diff

@ -1,2 +0,0 @@
--reporter spec
--timeout 30000

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,123 +1,121 @@
/* eslint-env mocha */
/* global DEBUG */
/**
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes
*/
var fs = require('fs');
const fs = require('fs')
function rethrow() {
function rethrow () {
// Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
// is fairly slow to generate.
if (DEBUG) {
var backtrace = new Error();
return function(err) {
const backtrace = new Error()
return function (err) {
if (err) {
backtrace.stack = err.name + ': ' + err.message +
backtrace.stack.substr(backtrace.name.length);
throw backtrace;
backtrace.stack.substr(backtrace.name.length)
throw backtrace
}
}
};
}
return function(err) {
return function (err) {
if (err) {
throw err; // Forgot a callback but don't know where? Use NODE_DEBUG=fs
throw err // Forgot a callback but don't know where? Use NODE_DEBUG=fs
}
}
};
}
function maybeCallback(cb) {
return typeof cb === 'function' ? cb : rethrow();
function maybeCallback (cb) {
return typeof cb === 'function' ? cb : rethrow()
}
function isFd(path) {
return (path >>> 0) === path;
function isFd (path) {
return (path >>> 0) === path
}
function assertEncoding(encoding) {
function assertEncoding (encoding) {
if (encoding && !Buffer.isEncoding(encoding)) {
throw new Error('Unknown encoding: ' + encoding);
throw new Error('Unknown encoding: ' + encoding)
}
}
var onePassDone = false;
function writeAll(fd, isUserFd, buffer, offset, length, position, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
let onePassDone = false
if (onePassDone) { process.exit(1); } // Crash on purpose before rewrite done
var l = Math.min(5000, length); // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
function writeAll (fd, isUserFd, buffer, offset, length, position, callback_) {
const callback = maybeCallback(arguments[arguments.length - 1])
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
const l = Math.min(5000, length) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
// write(fd, buffer, offset, length, position, callback)
fs.write(fd, buffer, offset, l, position, function(writeErr, written) {
fs.write(fd, buffer, offset, l, position, function (writeErr, written) {
if (writeErr) {
if (isUserFd) {
if (callback) callback(writeErr);
if (callback) callback(writeErr)
} else {
fs.close(fd, function() {
if (callback) callback(writeErr);
});
fs.close(fd, function () {
if (callback) callback(writeErr)
})
}
} else {
onePassDone = true;
onePassDone = true
if (written === length) {
if (isUserFd) {
if (callback) callback(null);
if (callback) callback(null)
} else {
fs.close(fd, callback);
fs.close(fd, callback)
}
} else {
offset += written;
length -= written;
offset += written
length -= written
if (position !== null) {
position += written;
position += written
}
writeAll(fd, isUserFd, buffer, offset, length, position, callback);
writeAll(fd, isUserFd, buffer, offset, length, position, callback)
}
}
});
})
}
fs.writeFile = function(path, data, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
fs.writeFile = function (path, data, options, callback_) {
const callback = maybeCallback(arguments[arguments.length - 1])
if (!options || typeof options === 'function') {
options = { encoding: 'utf8', mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases)
options = { encoding: 'utf8', mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options === 'string') {
options = { encoding: options, mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases)
options = { encoding: options, mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options !== 'object') {
throwOptionsError(options);
throw new Error(`throwOptionsError${options}`)
}
assertEncoding(options.encoding);
assertEncoding(options.encoding)
var flag = options.flag || 'w';
const flag = options.flag || 'w'
if (isFd(path)) {
writeFd(path, true);
return;
writeFd(path, true)
return
}
fs.open(path, flag, options.mode, function(openErr, fd) {
fs.open(path, flag, options.mode, function (openErr, fd) {
if (openErr) {
if (callback) callback(openErr);
if (callback) callback(openErr)
} else {
writeFd(fd, false);
writeFd(fd, false)
}
});
})
function writeFd(fd, isUserFd) {
var buffer = (data instanceof Buffer) ? data : new Buffer('' + data,
options.encoding || 'utf8');
var position = /a/.test(flag) ? null : 0;
function writeFd (fd, isUserFd) {
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8')
const position = /a/.test(flag) ? null : 0
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback);
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback)
}
};
}
// End of fs modification
var Nedb = require('../lib/datastore.js')
, db = new Nedb({ filename: 'workspace/lac.db' })
;
const Nedb = require('../lib/datastore.js')
const db = new Nedb({ filename: 'workspace/lac.db' })
db.loadDatabase();
db.loadDatabase()

@ -1,67 +1,64 @@
var fs = require('fs')
, child_process = require('child_process')
, async = require('async')
, Nedb = require('../lib/datastore')
, db = new Nedb({ filename: './workspace/openfds.db', autoload: true })
, N = 64 // Half the allowed file descriptors
, i, fds
;
const fs = require('fs')
const async = require('async')
const Nedb = require('../lib/datastore')
const db = new Nedb({ filename: './workspace/openfds.db', autoload: true })
const N = 64
let i
let fds
function multipleOpen (filename, N, callback) {
async.whilst( function () { return i < N; }
async.whilst(function () { return i < N }
, function (cb) {
fs.open(filename, 'r', function (err, fd) {
i += 1;
if (fd) { fds.push(fd); }
return cb(err);
});
i += 1
if (fd) { fds.push(fd) }
return cb(err)
})
}
, callback);
, callback)
}
async.waterfall([
// Check that ulimit has been set to the correct value
function (cb) {
i = 0;
fds = [];
i = 0
fds = []
multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) {
if (!err) { console.log("No error occured while opening a file too many times"); }
fds.forEach(function (fd) { fs.closeSync(fd); });
return cb();
if (!err) { console.log('No error occured while opening a file too many times') }
fds.forEach(function (fd) { fs.closeSync(fd) })
return cb()
})
}
, function (cb) {
i = 0;
fds = [];
},
function (cb) {
i = 0
fds = []
multipleOpen('./test_lac/openFdsTestFile2', N, function (err) {
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err); }
fds.forEach(function (fd) { fs.closeSync(fd); });
return cb();
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err) }
fds.forEach(function (fd) { fs.closeSync(fd) })
return cb()
})
}
},
// Then actually test NeDB persistence
, function () {
function () {
db.remove({}, { multi: true }, function (err) {
if (err) { console.log(err); }
if (err) { console.log(err) }
db.insert({ hello: 'world' }, function (err) {
if (err) { console.log(err); }
if (err) { console.log(err) }
i = 0;
async.whilst( function () { return i < 2 * N + 1; }
i = 0
async.whilst(function () { return i < 2 * N + 1 }
, function (cb) {
db.persistence.persistCachedDatabase(function (err) {
if (err) { return cb(err); }
i += 1;
return cb();
});
if (err) { return cb(err) }
i += 1
return cb()
})
}
, function (err) {
if (err) { console.log("Got unexpected error during one peresistence operation: " + err); }
if (err) { console.log('Got unexpected error during one peresistence operation: ' + err) }
}
);
});
});
)
})
})
}
]);
])

Loading…
Cancel
Save