TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

standard everything except browser-version

Browse Source
pull/2/head
Timothée Rebours 4 years ago
parent aa3302e5dc
commit 185ae680b8
32 changed files with 7648 additions and 7423 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 22
      LICENSE
  2. 19
      LICENSE.md
  3. 12
      README.md
  4. 328
      benchmarks/commonUtilities.js
  5. 65
      benchmarks/ensureIndex.js
  6. 46
      benchmarks/find.js
  7. 48
      benchmarks/findOne.js
  8. 46
      benchmarks/findWithIn.js
  9. 50
      benchmarks/insert.js
  10. 52
      benchmarks/loadDatabase.js
  11. 60
      benchmarks/remove.js
  12. 61
      benchmarks/update.js
  13. 4
      index.js
  14. 345
      lib/cursor.js
  15. 11
      lib/customUtils.js
  16. 1252
      lib/datastore.js
  17. 123
      lib/executor.js
  18. 461
      lib/indexes.js
  19. 628
      lib/model.js
  20. 548
      lib/persistence.js
  21. 127
      lib/storage.js
  22. 31
      package.json
  23. 1318
      test/cursor.test.js
  24. 32
      test/customUtil.test.js
  25. 4155
      test/db.test.js
  26. 268
      test/executor.test.js
  27. 1231
      test/indexes.test.js
  28. 2
      test/mocha.opts
  29. 2178
      test/model.test.js
  30. 1331
      test/persistence.test.js
  31. 118
      test_lac/loadAndCrash.test.js
  32. 99
      test_lac/openFds.test.js

22
LICENSE
Unescape View File

@ -1,22 +0,0 @@
(The MIT License)
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

19
LICENSE.md
Unescape View File

@ -0,0 +1,19 @@
Copyright (c) 2013 Louis Chatriot <louis.chatriot@gmail.com>
Copyright (c) 2021 Seald [contact@seald.io](mailto:contact@seald.io);
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including without
limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom
the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

12
README.md
Unescape View File

@ -34,7 +34,7 @@ It is a subset of MongoDB's API (the most used operations).
* <a href="#inserting-documents">Inserting documents</a>
* <a href="#finding-documents">Finding documents</a>
* <a href="#basic-querying">Basic Querying</a>
* <a href="#operators-lt-lte-gt-gte-in-nin-ne-exists-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)</a>
* <a href="#operators-lt-lte-gt-gte-in-nin-ne-stat-regex">Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)</a>
* <a href="#array-fields">Array fields</a>
* <a href="#logical-operators-or-and-not-where">Logical operators $or, $and, $not, $where</a>
* <a href="#sorting-and-paginating">Sorting and paginating</a>
@ -236,14 +236,14 @@ db.findOne({ _id: 'id1' }, function (err, doc) {
});
```
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $stat, $regex)
The syntax is `{ field: { $op: value } }` where `$op` is any comparison operator:
* `$lt`, `$lte`: less than, less than or equal
* `$gt`, `$gte`: greater than, greater than or equal
* `$in`: member of. `value` must be an array of values
* `$ne`, `$nin`: not equal, not a member of
* `$exists`: checks whether the document posses the property `field`. `value` should be true or false
* `$stat`: checks whether the document posses the property `field`. `value` should be true or false
* `$regex`: checks whether a string is matched by the regular expression. Contrary to MongoDB, the use of `$options` with `$regex` is not supported, because it doesn't give you more power than regex flags. Basic queries are more readable so only use the `$regex` operator when you need to use another operator with it (see example below)
```javascript
@ -262,8 +262,8 @@ db.find({ planet: { $in: ['Earth', 'Jupiter'] }}, function (err, docs) {
// docs contains Earth and Jupiter
});
// Using $exists
db.find({ satellites: { $exists: true } }, function (err, docs) {
// Using $stat
db.find({ satellites: { $stat: true } }, function (err, docs) {
// docs contains only Mars
});
@ -724,4 +724,4 @@ You don't have time? You can support NeDB by sending bitcoins to this address: 1
## License
See [License](LICENSE)
See [License](LICENSE.md)

328
benchmarks/commonUtilities.js
Unescape View File

@ -1,65 +1,58 @@
/**
* Functions that are used in several benchmark tests
*/
var customUtils = require('../lib/customUtils')
, fs = require('fs')
, path = require('path')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
, executeAsap // process.nextTick or setImmediate depending on your Node version
;
const fs = require('fs')
const path = require('path')
const Datastore = require('../lib/datastore')
const Persistence = require('../lib/persistence')
let executeAsap
try {
executeAsap = setImmediate;
executeAsap = setImmediate
} catch (e) {
executeAsap = process.nextTick;
executeAsap = process.nextTick
}
/**
* Configure the benchmark
*/
module.exports.getConfiguration = function (benchDb) {
var d, n
, program = require('commander')
;
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Use an index')
.option('-m --in-memory', 'Test with an in-memory only store')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log(program.withIndex ? "Use an index" : "Don't use an index");
console.log(program.inMemory ? "Use an in-memory datastore" : "Use a persistent datastore");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log(program.inMemory ? 'Use an in-memory datastore' : 'Use a persistent datastore')
console.log('----------------------------')
d = new Datastore({ filename: benchDb
, inMemoryOnly: program.inMemory
});
return { n: n, d: d, program: program };
};
const d = new Datastore({
filename: benchDb,
inMemoryOnly: program.inMemory
})
return { n: n, d: d, program: program }
}
/**
* Ensure the workspace exists and the db datafile is empty
* Ensure the workspace stat and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
};
fs.access(filename, fs.constants.FS_OK, function (err) {
if (!err) {
fs.unlink(filename, cb)
} else { return cb() }
})
})
}
/**
* Return an array with the numbers from 0 to n-1, in a random order
@ -67,242 +60,221 @@ module.exports.prepareDb = function (filename, cb) {
* Useful to get fair tests
*/
function getRandomArray (n) {
var res = []
, i, j, temp
;
const res = []
let i
let j
let temp
for (i = 0; i < n; i += 1) { res[i] = i; }
for (i = 0; i < n; i += 1) { res[i] = i }
for (i = n - 1; i >= 1; i -= 1) {
j = Math.floor((i + 1) * Math.random());
temp = res[i];
res[i] = res[j];
res[j] = temp;
j = Math.floor((i + 1) * Math.random())
temp = res[i]
res[i] = res[j]
res[j] = temp
}
return res;
return res
};
module.exports.getRandomArray = getRandomArray;
module.exports.getRandomArray = getRandomArray
/**
* Insert a certain number of documents for testing
*/
module.exports.insertDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step('Begin inserting ' + n + ' docs');
function runFrom(i) {
if (i === n) { // Finished
var opsPerSecond = Math.floor(1000* n / profiler.elapsedSinceLastStep());
console.log("===== RESULT (insert) ===== " + opsPerSecond + " ops/s");
profiler.step('Finished inserting ' + n + ' docs');
profiler.insertOpsPerSecond = opsPerSecond;
return cb();
const order = getRandomArray(n)
profiler.step('Begin inserting ' + n + ' docs')
function runFrom (i) {
if (i === n) { // Finished
const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
console.log('===== RESULT (insert) ===== ' + opsPerSecond + ' ops/s')
profiler.step('Finished inserting ' + n + ' docs')
profiler.insertOpsPerSecond = opsPerSecond
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.insert({ docNumber: order[i] }, function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with find
*/
module.exports.findDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Finding " + n + " documents");
profiler.step('Finding ' + n + ' documents')
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (find) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
function runFrom (i) {
if (i === n) { // Finished
console.log('===== RESULT (find) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: order[i] }, function (err, docs) {
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb('One find didnt work'); }
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with find and the $in operator
*/
module.exports.findDocsWithIn = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
, ins = [], i, j
, arraySize = Math.min(10, n) // The array for $in needs to be smaller than n (inclusive)
;
const ins = []
const arraySize = Math.min(10, n)
// Preparing all the $in arrays, will take some time
for (i = 0; i < n; i += 1) {
ins[i] = [];
for (let i = 0; i < n; i += 1) {
ins[i] = []
for (j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n);
for (let j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n)
}
}
profiler.step("Finding " + n + " documents WITH $IN OPERATOR");
profiler.step('Finding ' + n + ' documents WITH $IN OPERATOR')
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (find with in selector) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
function runFrom (i) {
if (i === n) { // Finished
console.log('===== RESULT (find with in selector) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.find({ docNumber: { $in: ins[i] } }, function (err, docs) {
if (docs.length !== arraySize) { return cb('One find didnt work'); }
if (docs.length !== arraySize) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Find documents with findOne
*/
module.exports.findOneDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("FindingOne " + n + " documents");
profiler.step('FindingOne ' + n + ' documents')
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (findOne) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
function runFrom (i) {
if (i === n) { // Finished
console.log('===== RESULT (findOne) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished finding ' + n + ' docs')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.findOne({ docNumber: order[i] }, function (err, doc) {
if (!doc || doc.docNumber !== order[i]) { return cb('One find didnt work'); }
if (!doc || doc.docNumber !== order[i]) { return cb(new Error('One find didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Update documents
* options is the same as the options object for update
*/
module.exports.updateDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Updating " + n + " documents");
profiler.step('Updating ' + n + ' documents')
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (update) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished updating ' + n + ' docs');
return cb();
function runFrom (i) {
if (i === n) { // Finished
console.log('===== RESULT (update) ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished updating ' + n + ' docs')
return cb()
}
// Will not actually modify the document but will take the same time
d.update({ docNumber: order[i] }, { docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One update didnt work'); }
if (err) { return cb(err) }
if (nr !== 1) { return cb(new Error('One update didnt work')) }
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Remove documents
* options is the same as the options object for update
*/
module.exports.removeDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
const order = getRandomArray(n)
profiler.step("Removing " + n + " documents");
profiler.step('Removing ' + n + ' documents')
function runFrom(i) {
if (i === n) { // Finished
function runFrom (i) {
if (i === n) { // Finished
// opsPerSecond corresponds to 1 insert + 1 remove, needed to keep collection size at 10,000
// We need to subtract the time taken by one insert to get the time actually taken by one remove
var opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep());
var removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log("===== RESULT (remove) ===== " + removeOpsPerSecond + " ops/s");
profiler.step('Finished removing ' + n + ' docs');
return cb();
const opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep())
const removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log('===== RESULT (remove) ===== ' + removeOpsPerSecond + ' ops/s')
profiler.step('Finished removing ' + n + ' docs')
return cb()
}
d.remove({ docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One remove didnt work'); }
d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size at n
// So actually we're calculating the average time taken by one insert + one remove
if (err) { return cb(err) }
if (nr !== 1) { return cb(new Error('One remove didnt work')) }
// eslint-disable-next-line node/handle-callback-err
d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size at n
// So actually we're calculating the average time taken by one insert + one remove
executeAsap(function () {
runFrom(i + 1);
});
});
});
runFrom(i + 1)
})
})
})
}
runFrom(0);
};
runFrom(0)
}
/**
* Load database
*/
module.exports.loadDatabase = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Loading the database " + n + " times");
profiler.step('Loading the database ' + n + ' times')
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished loading a database' + n + ' times');
return cb();
function runFrom (i) {
if (i === n) { // Finished
console.log('===== RESULT ===== ' + Math.floor(1000 * n / profiler.elapsedSinceLastStep()) + ' ops/s')
profiler.step('Finished loading a database' + n + ' times')
return cb()
}
// eslint-disable-next-line node/handle-callback-err
d.loadDatabase(function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
runFrom(i + 1)
})
})
}
runFrom(0);
};
runFrom(0)
}

65
benchmarks/ensureIndex.js
Unescape View File

@ -1,51 +1,48 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/insert.bench.db'
, async = require('async')
, commonUtilities = require('./commonUtilities')
, execTime = require('exec-time')
, profiler = new execTime('INSERT BENCH')
, d = new Datastore(benchDb)
, program = require('commander')
, n
;
const Datastore = require('../lib/datastore')
const benchDb = 'workspace/insert.bench.db'
const async = require('async')
const commonUtilities = require('./commonUtilities')
const ExecTime = require('exec-time')
const profiler = new ExecTime('INSERT BENCH')
const d = new Datastore(benchDb)
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log('----------------------------')
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) {
var i;
if (err) { return cb(err) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) {
let i
profiler.step('Begin calling ensureIndex ' + n + ' times');
profiler.step('Begin calling ensureIndex ' + n + ' times')
for (i = 0; i < n; i += 1) {
d.ensureIndex({ fieldName: 'docNumber' });
delete d.indexes.docNumber;
d.ensureIndex({ fieldName: 'docNumber' })
delete d.indexes.docNumber
}
console.log("Average time for one ensureIndex: " + (profiler.elapsedSinceLastStep() / n) + "ms");
profiler.step('Finished calling ensureIndex ' + n + ' times');
console.log('Average time for one ensureIndex: ' + (profiler.elapsedSinceLastStep() / n) + 'ms')
profiler.step('Finished calling ensureIndex ' + n + ' times')
}
], function (err) {
profiler.step("Benchmark finished");
if (err) { return console.log("An error was encountered: ", err); }
});
profiler.step('Benchmark finished')
if (err) { return console.log('An error was encountered: ', err) }
})

46
benchmarks/find.js
Unescape View File

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/find.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FIND BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/find.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FIND BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.findDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.findDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

48
benchmarks/findOne.js
Unescape View File

@ -1,31 +1,27 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/findOne.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FINDONE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/findOne.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FINDONE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { setTimeout(function () {cb();}, 500); }
, async.apply(commonUtilities.findOneDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { setTimeout(function () { cb() }, 500) },
async.apply(commonUtilities.findOneDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

46
benchmarks/findWithIn.js
Unescape View File

@ -1,30 +1,26 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/find.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FIND BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/find.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('FIND BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.findDocsWithIn, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.findDocsWithIn, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

50
benchmarks/insert.js
Unescape View File

@ -1,33 +1,31 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/insert.bench.db'
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('INSERT BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/insert.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('INSERT BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
let n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (err) { return cb(err) }
if (config.program.withIndex) {
d.ensureIndex({ fieldName: 'docNumber' });
n = 2 * n; // We will actually insert twice as many documents
// because the index is slower when the collection is already
// big. So the result given by the algorithm will be a bit worse than
// actual performance
d.ensureIndex({ fieldName: 'docNumber' })
n = 2 * n // We will actually insert twice as many documents
// because the index is slower when the collection is already
// big. So the result given by the algorithm will be a bit worse than
// actual performance
}
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

52
benchmarks/loadDatabase.js
Unescape View File

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/loaddb.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, commonUtilities = require('./commonUtilities')
, execTime = require('exec-time')
, profiler = new execTime('LOADDB BENCH')
, d = new Datastore(benchDb)
, program = require('commander')
, n
;
const Datastore = require('../lib/datastore')
const benchDb = 'workspace/loaddb.bench.db'
const async = require('async')
const commonUtilities = require('./commonUtilities')
const ExecTime = require('exec-time')
const profiler = new ExecTime('LOADDB BENCH')
const d = new Datastore(benchDb)
const program = require('commander')
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Test with an index')
.parse(process.argv);
.parse(process.argv)
n = program.number || 10000;
const n = program.number || 10000
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log(program.withIndex ? "Use an index" : "Don't use an index");
console.log("----------------------------");
console.log('----------------------------')
console.log('Test with ' + n + ' documents')
console.log(program.withIndex ? 'Use an index' : "Don't use an index")
console.log('----------------------------')
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
d.loadDatabase(cb);
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, async.apply(commonUtilities.loadDatabase, d, n, profiler)
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(cb)
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
async.apply(commonUtilities.loadDatabase, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

60
benchmarks/remove.js
Unescape View File

@ -1,38 +1,34 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/remove.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('REMOVE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/remove.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('REMOVE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with remove only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); }
, async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler)
// Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function () { return cb(); }); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); }
, async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler)
// Test with remove only one document
function (cb) { profiler.step('MULTI: FALSE'); return cb() },
async.apply(commonUtilities.removeDocs, { multi: false }, d, n, profiler),
// Test with multiple documents
function (cb) { d.remove({}, { multi: true }, function () { return cb() }) },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { profiler.step('MULTI: TRUE'); return cb() },
async.apply(commonUtilities.removeDocs, { multi: true }, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

61
benchmarks/update.js
Unescape View File

@ -1,39 +1,36 @@
var Datastore = require('../lib/datastore')
, benchDb = 'workspace/update.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('UPDATE BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
const benchDb = 'workspace/update.bench.db'
const async = require('async')
const ExecTime = require('exec-time')
const profiler = new ExecTime('UPDATE BENCH')
const commonUtilities = require('./commonUtilities')
const config = commonUtilities.getConfiguration(benchDb)
const d = config.d
const n = config.n
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
async.apply(commonUtilities.prepareDb, benchDb),
function (cb) {
d.loadDatabase(function (err) {
if (err) { return cb(err); }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }); }
cb();
});
}
, function (cb) { profiler.beginProfiling(); return cb(); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
if (err) { return cb(err) }
if (config.program.withIndex) { d.ensureIndex({ fieldName: 'docNumber' }) }
cb()
})
},
function (cb) { profiler.beginProfiling(); return cb() },
async.apply(commonUtilities.insertDocs, d, n, profiler),
// Test with update only one document
, function (cb) { profiler.step('MULTI: FALSE'); return cb(); }
, async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler)
// Test with update only one document
function (cb) { profiler.step('MULTI: FALSE'); return cb() },
async.apply(commonUtilities.updateDocs, { multi: false }, d, n, profiler),
// Test with multiple documents
, function (cb) { d.remove({}, { multi: true }, function (err) { return cb(); }); }
, async.apply(commonUtilities.insertDocs, d, n, profiler)
, function (cb) { profiler.step('MULTI: TRUE'); return cb(); }
, async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler)
// Test with multiple documents
// eslint-disable-next-line node/handle-callback-err
function (cb) { d.remove({}, { multi: true }, function (err) { return cb() }) },
async.apply(commonUtilities.insertDocs, d, n, profiler),
function (cb) { profiler.step('MULTI: TRUE'); return cb() },
async.apply(commonUtilities.updateDocs, { multi: true }, d, n, profiler)
], function (err) {
profiler.step("Benchmark finished");
profiler.step('Benchmark finished')
if (err) { return console.log("An error was encountered: ", err); }
});
if (err) { return console.log('An error was encountered: ', err) }
})

4
index.js
Unescape View File

@ -1,3 +1,3 @@
var Datastore = require('./lib/datastore');
const Datastore = require('./lib/datastore')
module.exports = Datastore;
module.exports = Datastore

345
lib/cursor.js
Unescape View File

@ -1,204 +1,201 @@
/**
* Manage access to data, be it to find, update or remove it
*/
var model = require('./model')
, _ = require('underscore')
;
/**
* Create a new cursor for this collection
* @param {Datastore} db - The datastore this cursor is bound to
* @param {Query} query - The query this cursor will operate on
* @param {Function} execFn - Handler to be executed after cursor has found the results and before the callback passed to find/findOne/update/remove
*/
function Cursor (db, query, execFn) {
this.db = db;
this.query = query || {};
if (execFn) { this.execFn = execFn; }
}
/**
* Set a limit to the number of results
*/
Cursor.prototype.limit = function(limit) {
this._limit = limit;
return this;
};
/**
* Skip a the number of results
*/
Cursor.prototype.skip = function(skip) {
this._skip = skip;
return this;
};
/**
* Sort results of the query
* @param {SortQuery} sortQuery - SortQuery is { field: order }, field can use the dot-notation, order is 1 for ascending and -1 for descending
*/
Cursor.prototype.sort = function(sortQuery) {
this._sort = sortQuery;
return this;
};
const model = require('./model')
const _ = require('underscore')
class Cursor {
/**
* Create a new cursor for this collection
* @param {Datastore} db - The datastore this cursor is bound to
* @param {Query} query - The query this cursor will operate on
* @param {Function} execFn - Handler to be executed after cursor has found the results and before the callback passed to find/findOne/update/remove
*/
constructor (db, query, execFn) {
this.db = db
this.query = query || {}
if (execFn) { this.execFn = execFn }
}
/**
* Add the use of a projection
* @param {Object} projection - MongoDB-style projection. {} means take all fields. Then it's { key1: 1, key2: 1 } to take only key1 and key2
* { key1: 0, key2: 0 } to omit only key1 and key2. Except _id, you can't mix takes and omits
*/
Cursor.prototype.projection = function(projection) {
this._projection = projection;
return this;
};
/**
* Set a limit to the number of results
*/
limit (limit) {
this._limit = limit
return this
}
/**
* Skip a the number of results
*/
skip (skip) {
this._skip = skip
return this
}
/**
* Apply the projection
*/
Cursor.prototype.project = function (candidates) {
var res = [], self = this
, keepId, action, keys
;
/**
* Sort results of the query
* @param {SortQuery} sortQuery - SortQuery is { field: order }, field can use the dot-notation, order is 1 for ascending and -1 for descending
*/
sort (sortQuery) {
this._sort = sortQuery
return this
}
if (this._projection === undefined || Object.keys(this._projection).length === 0) {
return candidates;
/**
* Add the use of a projection
* @param {Object} projection - MongoDB-style projection. {} means take all fields. Then it's { key1: 1, key2: 1 } to take only key1 and key2
* { key1: 0, key2: 0 } to omit only key1 and key2. Except _id, you can't mix takes and omits
*/
projection (projection) {
this._projection = projection
return this
}
keepId = this._projection._id === 0 ? false : true;
this._projection = _.omit(this._projection, '_id');
// Check for consistency
keys = Object.keys(this._projection);
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error("Can't both keep and omit fields except for _id"); }
action = self._projection[k];
});
// Do the actual projection
candidates.forEach(function (candidate) {
var toPush;
if (action === 1) { // pick-type projection
toPush = { $set: {} };
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
}
if (keepId) {
toPush._id = candidate._id;
} else {
delete toPush._id;
}
res.push(toPush);
});
/**
* Apply the projection
*/
project (candidates) {
const res = []
const self = this
let action
return res;
};
if (this._projection === undefined || Object.keys(this._projection).length === 0) {
return candidates
}
const keepId = this._projection._id !== 0
this._projection = _.omit(this._projection, '_id')
// Check for consistency
const keys = Object.keys(this._projection)
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error('Can\'t both keep and omit fields except for _id') }
action = self._projection[k]
})
// Do the actual projection
candidates.forEach(function (candidate) {
let toPush
if (action === 1) { // pick-type projection
toPush = { $set: {} }
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k)
if (toPush.$set[k] === undefined) { delete toPush.$set[k] }
})
toPush = model.modify({}, toPush)
} else { // omit-type projection
toPush = { $unset: {} }
keys.forEach(function (k) { toPush.$unset[k] = true })
toPush = model.modify(candidate, toPush)
}
if (keepId) {
toPush._id = candidate._id
} else {
delete toPush._id
}
res.push(toPush)
})
/**
* Get all matching elements
* Will return pointers to matched elements (shallow copies), returning full copies is the role of find or findOne
* This is an internal function, use exec which uses the executor
*
* @param {Function} callback - Signature: err, results
*/
Cursor.prototype._exec = function(_callback) {
var res = [], added = 0, skipped = 0, self = this
, error = null
, i, keys, key
;
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback);
} else {
return _callback(error, res);
}
return res
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err); }
/**
* Get all matching elements
* Will return pointers to matched elements (shallow copies), returning full copies is the role of find or findOne
* This is an internal function, use exec which uses the executor
*
* @param {Function} callback - Signature: err, results
*/
_exec (_callback) {
let res = []
let added = 0
let skipped = 0
const self = this
let error = null
let i
let keys
let key
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback)
} else {
return _callback(error, res)
}
}
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1;
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err) }
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1
} else {
res.push(candidates[i])
added += 1
if (self._limit && self._limit <= added) { break }
}
} else {
res.push(candidates[i]);
added += 1;
if (self._limit && self._limit <= added) { break; }
res.push(candidates[i])
}
} else {
res.push(candidates[i]);
}
}
} catch (err) {
return callback(err)
}
} catch (err) {
return callback(err);
}
// Apply all sorts
if (self._sort) {
keys = Object.keys(self._sort);
// Apply all sorts
if (self._sort) {
keys = Object.keys(self._sort)
// Sorting
var criteria = [];
for (i = 0; i < keys.length; i++) {
key = keys[i];
criteria.push({ key: key, direction: self._sort[key] });
}
res.sort(function(a, b) {
var criterion, compare, i;
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i];
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings);
if (compare !== 0) {
return compare;
}
// Sorting
const criteria = []
for (i = 0; i < keys.length; i++) {
key = keys[i]
criteria.push({ key: key, direction: self._sort[key] })
}
return 0;
});
// Applying limit and skip
var limit = self._limit || res.length
, skip = self._skip || 0;
res = res.slice(skip, skip + limit);
}
res.sort(function (a, b) {
let criterion
let compare
let i
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i]
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings)
if (compare !== 0) {
return compare
}
}
return 0
})
// Apply projection
try {
res = self.project(res);
} catch (e) {
error = e;
res = undefined;
}
// Applying limit and skip
const limit = self._limit || res.length
const skip = self._skip || 0
return callback(error, res);
});
};
res = res.slice(skip, skip + limit)
}
Cursor.prototype.exec = function () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments });
};
// Apply projection
try {
res = self.project(res)
} catch (e) {
error = e
res = undefined
}
return callback(error, res)
})
}
exec () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments })
}
}
// Interface
module.exports = Cursor;
module.exports = Cursor

11
lib/customUtils.js
Unescape View File

@ -1,5 +1,4 @@
var crypto = require('crypto')
;
const crypto = require('crypto')
/**
* Return a random alphanumerical string of length len
@ -12,11 +11,9 @@ var crypto = require('crypto')
function uid (len) {
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
.toString('base64')
.replace(/[+\/]/g, '')
.slice(0, len);
.replace(/[+/]/g, '')
.slice(0, len)
}
// Interface
module.exports.uid = uid;
module.exports.uid = uid

1252
lib/datastore.js
View File

File diff suppressed because it is too large Load Diff

123
lib/executor.js
Unescape View File

@ -1,78 +1,73 @@
/**
* Responsible for sequentially executing actions on the database
*/
const async = require('async')
var async = require('async')
;
class Executor {
constructor () {
this.buffer = []
this.ready = false
function Executor () {
this.buffer = [];
this.ready = false;
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
const newArguments = []
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
var newArguments = [];
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (let i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]) }
const lastArg = task.arguments[task.arguments.length - 1]
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (var i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]); }
var lastArg = task.arguments[task.arguments.length - 1];
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
// Callback was supplied
newArguments[newArguments.length - 1] = function () {
if (typeof setImmediate === 'function') {
setImmediate(cb);
} else {
process.nextTick(cb);
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
// Callback was supplied
newArguments[newArguments.length - 1] = function () {
if (typeof setImmediate === 'function') {
setImmediate(cb)
} else {
process.nextTick(cb)
}
lastArg.apply(null, arguments)
}
lastArg.apply(null, arguments);
};
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callbback
newArguments[newArguments.length - 1] = function () { cb(); };
} else {
// Nothing supplied as callback
newArguments.push(function () { cb(); });
}
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callback
newArguments[newArguments.length - 1] = function () { cb() }
} else {
// Nothing supplied as callback
newArguments.push(function () { cb() })
}
task.fn.apply(task.this, newArguments);
}, 1);
}
/**
* If executor is ready, queue task (and process it immediately if executor was idle)
* If not, buffer task for later processing
* @param {Object} task
* task.this - Object to use as this
* task.fn - Function to execute
* task.arguments - Array of arguments, IMPORTANT: only the last argument may be a function (the callback)
* and the last argument cannot be false/undefined/null
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
Executor.prototype.push = function (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task);
} else {
this.buffer.push(task);
task.fn.apply(task.this, newArguments)
}, 1)
}
};
/**
* Queue all tasks in buffer (in the same order they came in)
* Automatically sets executor as ready
*/
Executor.prototype.processBuffer = function () {
var i;
this.ready = true;
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]); }
this.buffer = [];
};
/**
* If executor is ready, queue task (and process it immediately if executor was idle)
* If not, buffer task for later processing
* @param {Object} task
* task.this - Object to use as this
* task.fn - Function to execute
* task.arguments - Array of arguments, IMPORTANT: only the last argument may be a function (the callback)
* and the last argument cannot be false/undefined/null
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
push (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task)
} else {
this.buffer.push(task)
}
}
/**
* Queue all tasks in buffer (in the same order they came in)
* Automatically sets executor as ready
*/
processBuffer () {
let i
this.ready = true
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]) }
this.buffer = []
}
}
// Interface
module.exports = Executor;
module.exports = Executor

461
lib/indexes.js
Unescape View File

@ -1,294 +1,295 @@
var BinarySearchTree = require('binary-search-tree').AVLTree
, model = require('./model')
, _ = require('underscore')
, util = require('util')
;
const BinarySearchTree = require('@seald-io/binary-search-tree').BinarySearchTree
const model = require('./model')
const _ = require('underscore')
/**
* Two indexed pointers are equal iif they point to the same place
*/
function checkValueEquality (a, b) {
return a === b;
return a === b
}
/**
* Type-aware projection
*/
function projectForUnique (elt) {
if (elt === null) { return '$null'; }
if (typeof elt === 'string') { return '$string' + elt; }
if (typeof elt === 'boolean') { return '$boolean' + elt; }
if (typeof elt === 'number') { return '$number' + elt; }
if (util.isArray(elt)) { return '$date' + elt.getTime(); }
if (elt === null) { return '$null' }
if (typeof elt === 'string') { return '$string' + elt }
if (typeof elt === 'boolean') { return '$boolean' + elt }
if (typeof elt === 'number') { return '$number' + elt }
if (Array.isArray(elt)) { return '$date' + elt.getTime() }
return elt; // Arrays and objects, will check for pointer equality
return elt // Arrays and objects, will check for pointer equality
}
class Index {
/**
* Create a new index
* All methods on an index guarantee that either the whole operation was successful and the index changed
* or the operation was unsuccessful and an error is thrown while the index is unchanged
* @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields)
* @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
* @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
*/
constructor (options) {
this.fieldName = options.fieldName
this.unique = options.unique || false
this.sparse = options.sparse || false
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality }
this.reset() // No data in the beginning
}
/**
* Create a new index
* All methods on an index guarantee that either the whole operation was successful and the index changed
* or the operation was unsuccessful and an error is thrown while the index is unchanged
* @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields)
* @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
* @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
*/
function Index (options) {
this.fieldName = options.fieldName;
this.unique = options.unique || false;
this.sparse = options.sparse || false;
this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
this.reset(); // No data in the beginning
}
/**
* Reset an index
* @param {Document or Array of documents} newData Optional, data to initialize the index with
* If an error is thrown during insertion, the index is not modified
*/
Index.prototype.reset = function (newData) {
this.tree = new BinarySearchTree(this.treeOptions);
if (newData) { this.insert(newData); }
};
/**
* Reset an index
* @param {Document or Array of documents} newData Optional, data to initialize the index with
* If an error is thrown during insertion, the index is not modified
*/
reset (newData) {
this.tree = new BinarySearchTree(this.treeOptions)
/**
* Insert a new document in the index
* If an array is passed, we insert all its elements (if one insertion fails the index is not modified)
* O(log(n))
*/
Index.prototype.insert = function (doc) {
var key, self = this
, keys, i, failingI, error
;
if (newData) { this.insert(newData) }
}
if (util.isArray(doc)) { this.insertMultipleDocs(doc); return; }
/**
* Insert a new document in the index
* If an array is passed, we insert all its elements (if one insertion fails the index is not modified)
* O(log(n))
*/
insert (doc) {
let keys
let i
let failingI
let error
if (Array.isArray(doc)) {
this.insertMultipleDocs(doc)
return
}
key = model.getDotValue(doc, this.fieldName);
const key = model.getDotValue(doc, this.fieldName)
// We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return }
if (!Array.isArray(key)) {
this.tree.insert(key, doc)
} else {
// If an insert fails due to a unique constraint, roll back all inserts before it
keys = _.uniq(key, projectForUnique)
for (i = 0; i < keys.length; i += 1) {
try {
this.tree.insert(keys[i], doc)
} catch (e) {
error = e
failingI = i
break
}
}
// We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return; }
if (error) {
for (i = 0; i < failingI; i += 1) {
this.tree.delete(keys[i], doc)
}
if (!util.isArray(key)) {
this.tree.insert(key, doc);
} else {
// If an insert fails due to a unique constraint, roll back all inserts before it
keys = _.uniq(key, projectForUnique);
throw error
}
}
}
for (i = 0; i < keys.length; i += 1) {
/**
* Insert an array of documents in the index
* If a constraint is violated, the changes should be rolled back and an error thrown
*
* @API private
*/
insertMultipleDocs (docs) {
let i
let error
let failingI
for (i = 0; i < docs.length; i += 1) {
try {
this.tree.insert(keys[i], doc);
this.insert(docs[i])
} catch (e) {
error = e;
failingI = i;
break;
error = e
failingI = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.tree.delete(keys[i], doc);
this.remove(docs[i])
}
throw error;
throw error
}
}
};
/**
* Insert an array of documents in the index
* If a constraint is violated, the changes should be rolled back and an error thrown
*
* @API private
*/
Index.prototype.insertMultipleDocs = function (docs) {
var i, error, failingI;
for (i = 0; i < docs.length; i += 1) {
try {
this.insert(docs[i]);
} catch (e) {
error = e;
failingI = i;
break;
/**
* Remove a document from the index
* If an array is passed, we remove all its elements
* The remove operation is safe with regards to the 'unique' constraint
* O(log(n))
*/
remove (doc) {
const self = this
if (Array.isArray(doc)) {
doc.forEach(function (d) { self.remove(d) })
return
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.remove(docs[i]);
}
throw error;
}
};
/**
* Remove a document from the index
* If an array is passed, we remove all its elements
* The remove operation is safe with regards to the 'unique' constraint
* O(log(n))
*/
Index.prototype.remove = function (doc) {
var key, self = this;
if (util.isArray(doc)) { doc.forEach(function (d) { self.remove(d); }); return; }
key = model.getDotValue(doc, this.fieldName);
if (key === undefined && this.sparse) { return; }
if (!util.isArray(key)) {
this.tree.delete(key, doc);
} else {
_.uniq(key, projectForUnique).forEach(function (_key) {
self.tree.delete(_key, doc);
});
}
};
/**
* Update a document in the index
* If a constraint is violated, changes are rolled back and an error thrown
* Naive implementation, still in O(log(n))
*/
Index.prototype.update = function (oldDoc, newDoc) {
if (util.isArray(oldDoc)) { this.updateMultipleDocs(oldDoc); return; }
const key = model.getDotValue(doc, this.fieldName)
this.remove(oldDoc);
if (key === undefined && this.sparse) { return }
try {
this.insert(newDoc);
} catch (e) {
this.insert(oldDoc);
throw e;
if (!Array.isArray(key)) {
this.tree.delete(key, doc)
} else {
_.uniq(key, projectForUnique).forEach(function (_key) {
self.tree.delete(_key, doc)
})
}
}
};
/**
* Update multiple documents in the index
* If a constraint is violated, the changes need to be rolled back
* and an error thrown
* @param {Array of oldDoc, newDoc pairs} pairs
*
* @API private
*/
Index.prototype.updateMultipleDocs = function (pairs) {
var i, failingI, error;
/**
* Update a document in the index
* If a constraint is violated, changes are rolled back and an error thrown
* Naive implementation, still in O(log(n))
*/
update (oldDoc, newDoc) {
if (Array.isArray(oldDoc)) {
this.updateMultipleDocs(oldDoc)
return
}
for (i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc);
}
this.remove(oldDoc)
for (i = 0; i < pairs.length; i += 1) {
try {
this.insert(pairs[i].newDoc);
this.insert(newDoc)
} catch (e) {
error = e;
failingI = i;
break;
this.insert(oldDoc)
throw e
}
}
// If an error was raised, roll back changes in the inverse order
if (error) {
for (i = 0; i < failingI; i += 1) {
this.remove(pairs[i].newDoc);
/**
* Update multiple documents in the index
* If a constraint is violated, the changes need to be rolled back
* and an error thrown
* @param {Array of oldDoc, newDoc pairs} pairs
*
* @API private
*/
updateMultipleDocs (pairs) {
let i
let failingI
let error
for (i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc)
}
for (i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc);
try {
this.insert(pairs[i].newDoc)
} catch (e) {
error = e
failingI = i
break
}
}
throw error;
}
};
// If an error was raised, roll back changes in the inverse order
if (error) {
for (i = 0; i < failingI; i += 1) {
this.remove(pairs[i].newDoc)
}
for (i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc)
}
/**
* Revert an update
*/
Index.prototype.revertUpdate = function (oldDoc, newDoc) {
var revert = [];
if (!util.isArray(oldDoc)) {
this.update(newDoc, oldDoc);
} else {
oldDoc.forEach(function (pair) {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc });
});
this.update(revert);
throw error
}
}
};
/**
* Get all documents in index whose key match value (if it is a Thing) or one of the elements of value (if it is an array of Things)
* @param {Thing} value Value to match the key against
* @return {Array of documents}
*/
Index.prototype.getMatching = function (value) {
var self = this;
if (!util.isArray(value)) {
return self.tree.search(value);
} else {
var _res = {}, res = [];
value.forEach(function (v) {
self.getMatching(v).forEach(function (doc) {
_res[doc._id] = doc;
});
});
Object.keys(_res).forEach(function (_id) {
res.push(_res[_id]);
});
return res;
/**
* Revert an update
*/
revertUpdate (oldDoc, newDoc) {
const revert = []
if (!Array.isArray(oldDoc)) {
this.update(newDoc, oldDoc)
} else {
oldDoc.forEach(function (pair) {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc })
})
this.update(revert)
}
}
};
/**
* Get all documents in index whose key is between bounds are they are defined by query
* Documents are sorted by key
* @param {Query} query
* @return {Array of documents}
*/
Index.prototype.getBetweenBounds = function (query) {
return this.tree.betweenBounds(query);
};
/**
* Get all elements in the index
* @return {Array of documents}
*/
Index.prototype.getAll = function () {
var res = [];
this.tree.executeOnEveryNode(function (node) {
var i;
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i]);
/**
* Get all documents in index whose key match value (if it is a Thing) or one of the elements of value (if it is an array of Things)
* @param {Thing} value Value to match the key against
* @return {Array of documents}
*/
getMatching (value) {
const self = this
if (!Array.isArray(value)) {
return self.tree.search(value)
} else {
const _res = {}
const res = []
value.forEach(function (v) {
self.getMatching(v).forEach(function (doc) {
_res[doc._id] = doc
})
})
Object.keys(_res).forEach(function (_id) {
res.push(_res[_id])
})
return res
}
});
}
return res;
};
/**
* Get all documents in index whose key is between bounds are they are defined by query
* Documents are sorted by key
* @param {Query} query
* @return {Array of documents}
*/
getBetweenBounds (query) {
return this.tree.betweenBounds(query)
}
/**
* Get all elements in the index
* @return {Array of documents}
*/
getAll () {
const res = []
this.tree.executeOnEveryNode(function (node) {
let i
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i])
}
})
return res
}
}
// Interface
module.exports = Index;
module.exports = Index

628
lib/model.js
View File

File diff suppressed because it is too large Load Diff

548
lib/persistence.js
Unescape View File

@ -4,311 +4,303 @@
* * Persistence.loadDatabase(callback) and callback has signature err
* * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
*/
const storage = require('./storage')
const path = require('path')
const model = require('./model')
const async = require('async')
const customUtils = require('./customUtils')
const Index = require('./indexes')
class Persistence {
/**
* Create a new Persistence object for database options.db
* @param {Datastore} options.db
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/
constructor (options) {
let i
let j
let randomString
this.db = options.db
this.inMemoryOnly = this.db.inMemoryOnly
this.filename = this.db.filename
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
}
var storage = require('./storage')
, path = require('path')
, model = require('./model')
, async = require('async')
, customUtils = require('./customUtils')
, Index = require('./indexes')
;
/**
* Create a new Persistence object for database options.db
* @param {Datastore} options.db
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/
function Persistence (options) {
var i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss");
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss");
}
this.afterSerialization = options.afterSerialization || function (s) { return s; };
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i);
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss");
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
}
this.afterSerialization = options.afterSerialization || function (s) { return s }
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s }
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i)
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss')
}
}
}
}
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log("==================================================================");
console.log("WARNING: The nodeWebkitAppName option is deprecated");
console.log("To get the path to the directory where Node Webkit stores the data");
console.log("for your app, use the internal nw.gui module like this");
console.log("require('nw.gui').App.dataPath");
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log('==================================================================')
console.log('WARNING: The nodeWebkitAppName option is deprecated')
console.log('To get the path to the directory where Node Webkit stores the data')
console.log('for your app, use the internal nw.gui module like this')
console.log('require(\'nw.gui\').App.dataPath')
console.log('See https://github.com/rogerwang/node-webkit/issues/500')
console.log('==================================================================')
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename)
}
}
};
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
Persistence.ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
/**
* Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection
* while the data file is append-only so it may grow larger
* @param {Function} cb Optional callback, signature: err
*/
persistCachedDatabase (cb) {
const callback = cb || function () {}
let toPersist = ''
const self = this
if (this.inMemoryOnly) { return callback(null) }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({
$$indexCreated: {
fieldName: fieldName,
unique: self.db.indexes[fieldName].unique,
sparse: self.db.indexes[fieldName].sparse
}
})) + '\n'
}
})
storage.mkdirp(dir, function (err) { return callback(err); });
};
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err) }
self.db.emit('compaction.done')
return callback(null)
})
}
/**
* Queue a rewrite of the datafile
*/
compactDatafile () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] })
}
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
setAutocompactionInterval (interval) {
const self = this
const minInterval = 5000
const realInterval = Math.max(interval || 0, minInterval)
this.stopAutocompaction()
/**
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
* data for this application. Probably the best place to store data
*/
Persistence.getNWAppFilename = function (appName, relativeFilename) {
var home;
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA;
if (!home) { throw new Error("Couldn't find the base application data folder"); }
home = path.join(home, appName);
break;
case 'darwin':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, 'Library', 'Application Support', appName);
break;
case 'linux':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, '.config', appName);
break;
default:
throw new Error("Can't use the Node Webkit relative path for platform " + process.platform);
break;
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile()
}, realInterval)
}
return path.join(home, 'nedb-data', relativeFilename);
}
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
stopAutocompaction () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId) }
}
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
*/
persistNewState (newDocs, cb) {
const self = this
let toPersist = ''
const callback = cb || function () {}
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null) }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
})
if (toPersist.length === 0) { return callback(null) }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err)
})
}
/**
* Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection
* while the data file is append-only so it may grow larger
* @param {Function} cb Optional callback, signature: err
*/
Persistence.prototype.persistCachedDatabase = function (cb) {
var callback = cb || function () {}
, toPersist = ''
, self = this
;
if (this.inMemoryOnly) { return callback(null); }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
treatRawData (rawData) {
const data = rawData.split('\n')
const dataById = {}
const tdata = []
let i
const indexes = {}
let corruptItems = -1
for (i = 0; i < data.length; i += 1) {
let doc
try {
doc = model.deserialize(this.beforeDeserialization(data[i]))
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id]
} else {
dataById[doc._id] = doc
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
} else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved]
}
} catch (e) {
corruptItems += 1
}
}
});
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err); }
self.db.emit('compaction.done');
return callback(null);
});
};
/**
* Queue a rewrite of the datafile
*/
Persistence.prototype.compactDatafile = function () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
};
/**
* Set automatic compaction every interval ms
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
Persistence.prototype.setAutocompactionInterval = function (interval) {
var self = this
, minInterval = 5000
, realInterval = Math.max(interval || 0, minInterval)
;
this.stopAutocompaction();
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile();
}, realInterval);
};
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
Persistence.prototype.stopAutocompaction = function () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
};
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
*/
Persistence.prototype.persistNewState = function (newDocs, cb) {
var self = this
, toPersist = ''
, callback = cb || function () {}
;
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
if (toPersist.length === 0) { return callback(null); }
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error('More than ' + Math.floor(100 * this.corruptAlertThreshold) + '% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss')
}
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err);
});
};
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k])
})
return { data: tdata, indexes: indexes }
}
/**
* From a database's raw data, return the corresponding
* machine understandable collection
*/
Persistence.prototype.treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, tdata = []
, i
, indexes = {}
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(this.beforeDeserialization(data[i]));
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
} else if (typeof doc.$$indexRemoved === "string") {
delete indexes[doc.$$indexRemoved];
/**
* Load the database
* 1) Create all indexes
* 2) Insert all data
* 3) Compact the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*/
loadDatabase (cb) {
const callback = cb || function () {}
const self = this
self.db.resetIndexes()
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null) }
async.waterfall([
function (cb) {
// eslint-disable-next-line node/handle-callback-err
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
// TODO: handle error
// eslint-disable-next-line node/handle-callback-err
storage.ensureDatafileIntegrity(self.filename, function (err) {
// TODO: handle error
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err) }
let treatedData
try {
treatedData = self.treatRawData(rawData)
} catch (e) {
return cb(e)
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key])
})
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data)
} catch (e) {
self.db.resetIndexes() // Rollback any index which didn't fail
return cb(e)
}
self.db.persistence.persistCachedDatabase(cb)
})
})
})
}
} catch (e) {
corruptItems += 1;
}
}
], function (err) {
if (err) { return callback(err) }
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss");
self.db.executor.processBuffer()
return callback(null)
})
}
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k]);
});
return { data: tdata, indexes: indexes };
};
/**
* Check if a directory stat and create it on the fly if it is not the case
* cb is optional, signature: err
*/
static ensureDirectoryExists (dir, cb) {
const callback = cb || function () {}
storage.mkdir(dir, { recursive: true }, err => { callback(err) })
}
/**
* Load the database
* 1) Create all indexes
* 2) Insert all data
* 3) Compact the database
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
*/
Persistence.prototype.loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.db.resetIndexes();
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);
});
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
return cb(e);
}
self.db.persistence.persistCachedDatabase(cb);
});
});
});
/**
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
* data for this application. Probably the best place to store data
*/
static getNWAppFilename (appName, relativeFilename) {
let home
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA
if (!home) { throw new Error('Couldn\'t find the base application data folder') }
home = path.join(home, appName)
break
case 'darwin':
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
home = path.join(home, 'Library', 'Application Support', appName)
break
case 'linux':
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
home = path.join(home, '.config', appName)
break
default:
throw new Error('Can\'t use the Node Webkit relative path for platform ' + process.platform)
}
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
};
return path.join(home, 'nedb-data', relativeFilename)
}
}
// Interface
module.exports = Persistence;
module.exports = Persistence

127
lib/storage.js
Unescape View File

@ -6,34 +6,30 @@
* This version is the Node.js/Node Webkit version
* It's essentially fs, mkdirp and crash safe write and read functions
*/
var fs = require('fs')
, mkdirp = require('mkdirp')
, async = require('async')
, path = require('path')
, storage = {}
;
storage.exists = fs.exists;
storage.rename = fs.rename;
storage.writeFile = fs.writeFile;
storage.unlink = fs.unlink;
storage.appendFile = fs.appendFile;
storage.readFile = fs.readFile;
storage.mkdirp = mkdirp;
const fs = require('fs')
const async = require('async')
const path = require('path')
const storage = {}
// eslint-disable-next-line node/no-callback-literal
storage.exists = (path, cb) => fs.access(path, fs.constants.F_OK, (err) => { cb(!err) })
storage.rename = fs.rename
storage.writeFile = fs.writeFile
storage.unlink = fs.unlink
storage.appendFile = fs.appendFile
storage.readFile = fs.readFile
storage.mkdir = fs.mkdir
/**
* Explicit name ...
*/
storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
};
if (!exists) { return callback(null) }
storage.unlink(file, function (err) { return callback(err) })
})
}
/**
* Flush data in OS buffer to storage if corresponding option is set
@ -42,36 +38,36 @@ storage.ensureFileDoesntExist = function (file, callback) {
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/
storage.flushToStorage = function (options, callback) {
var filename, flags;
let filename
let flags
if (typeof options === 'string') {
filename = options;
flags = 'r+';
filename = options
flags = 'r+'
} else {
filename = options.filename;
flags = options.isDir ? 'r' : 'r+';
filename = options.filename
flags = options.isDir ? 'r' : 'r+'
}
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null); }
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null) }
fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err); }
if (err) { return callback(err) }
fs.fsync(fd, function (errFS) {
fs.close(fd, function (errC) {
if (errFS || errC) {
var e = new Error('Failed to flush to storage');
e.errorOnFsync = errFS;
e.errorOnClose = errC;
return callback(e);
const e = new Error('Failed to flush to storage')
e.errorOnFsync = errFS
e.errorOnClose = errC
return callback(e)
} else {
return callback(null);
return callback(null)
}
});
});
});
};
})
})
})
}
/**
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
@ -80,31 +76,30 @@ storage.flushToStorage = function (options, callback) {
* @param {Function} cb Optional callback, signature: err
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~';
const callback = cb || function () {}
const tempFilename = filename + '~'
async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, function (cb) {
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }),
function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err); });
storage.flushToStorage(filename, function (err) { return cb(err) })
} else {
return cb();
return cb()
}
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err); })
};
})
},
function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err) })
},
async.apply(storage.flushToStorage, tempFilename),
function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err) })
},
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err) })
}
/**
* Ensure the datafile contains all the data, even if there was a crash during a full file write
@ -112,25 +107,23 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~';
const tempFilename = filename + '~'
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
if (filenameExists) { return callback(null) }
storage.exists(tempFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err) })
}
// Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err); });
});
});
};
storage.rename(tempFilename, filename, function (err) { return callback(err) })
})
})
}
// Interface
module.exports = storage;
module.exports = storage

31
package.json
Unescape View File

@ -20,27 +20,36 @@
"url": "git@github.com:louischatriot/nedb.git"
},
"dependencies": {
"@seald-io/binary-search-tree": "^1.0.0",
"async": "0.2.10",
"binary-search-tree": "0.2.5",
"localforage": "^1.3.0",
"mkdirp": "~0.5.1",
"underscore": "~1.4.4"
"localforage": "^1.9.0",
"underscore": "^1.13.1"
},
"devDependencies": {
"chai": "^3.2.0",
"mocha": "1.4.x",
"chai": "^4.3.4",
"commander": "1.1.1",
"exec-time": "0.0.2",
"mocha": "^8.4.0",
"request": "2.9.x",
"semver": "^7.3.5",
"sinon": "1.3.x",
"exec-time": "0.0.2",
"commander": "1.1.1"
"standard": "^16.0.3"
},
"scripts": {
"test": "./node_modules/.bin/mocha --reporter spec --timeout 10000"
"test": "mocha --reporter spec --timeout 10000"
},
"main": "index",
"main": "index.js",
"browser": {
"./lib/customUtils.js": "./browser-version/browser-specific/lib/customUtils.js",
"./lib/storage.js": "./browser-version/browser-specific/lib/storage.js"
},
"license": "SEE LICENSE IN LICENSE"
"license": "MIT",
"publishConfig": {
"access": "public"
},
"standard": {
"ignore": [
"browser-version"
]
}
}

1318
test/cursor.test.js
View File

File diff suppressed because it is too large Load Diff

32
test/customUtil.test.js
Unescape View File

@ -1,26 +1,20 @@
var should = require('chai').should()
, assert = require('chai').assert
, customUtils = require('../lib/customUtils')
, fs = require('fs')
;
/* eslint-env mocha */
const chai = require('chai')
const customUtils = require('../lib/customUtils')
chai.should()
describe('customUtils', function () {
describe('uid', function () {
it('Generates a string of the expected length', function () {
customUtils.uid(3).length.should.equal(3);
customUtils.uid(16).length.should.equal(16);
customUtils.uid(42).length.should.equal(42);
customUtils.uid(1000).length.should.equal(1000);
});
customUtils.uid(3).length.should.equal(3)
customUtils.uid(16).length.should.equal(16)
customUtils.uid(42).length.should.equal(42)
customUtils.uid(1000).length.should.equal(1000)
})
// Very small probability of conflict
it('Generated uids should not be the same', function () {
customUtils.uid(56).should.not.equal(customUtils.uid(56));
});
});
});
customUtils.uid(56).should.not.equal(customUtils.uid(56))
})
})
})

4155
test/db.test.js
View File

File diff suppressed because it is too large Load Diff

268
test/executor.test.js
Unescape View File

@ -1,213 +1,215 @@
var should = require('chai').should()
, assert = require('chai').assert
, testDb = 'workspace/test.db'
, fs = require('fs')
, path = require('path')
, _ = require('underscore')
, async = require('async')
, model = require('../lib/model')
, Datastore = require('../lib/datastore')
, Persistence = require('../lib/persistence')
;
/* eslint-env mocha */
const chai = require('chai')
const testDb = 'workspace/test.db'
const fs = require('fs')
const path = require('path')
const async = require('async')
const Datastore = require('../lib/datastore')
const Persistence = require('../lib/persistence')
const { assert } = chai
chai.should()
// Test that even if a callback throws an exception, the next DB operations will still be executed
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testThrowInCallback (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException');
const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException');
process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track
});
})
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err) {
process.nextTick(function () {
// eslint-disable-next-line node/handle-callback-err
d.insert({ bar: 1 }, function (err) {
process.removeAllListeners('uncaughtException');
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]);
process.removeAllListeners('uncaughtException')
for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
}
done();
});
});
done()
})
})
throw new Error('Some error');
});
throw new Error('Some error')
})
}
// Test that if the callback is falsy, the next DB operations will still be executed
function testFalsyCallback (d, done) {
d.insert({ a: 1 }, null);
d.insert({ a: 1 }, null)
process.nextTick(function () {
d.update({ a: 1 }, { a: 2 }, {}, null);
d.update({ a: 1 }, { a: 2 }, {}, null)
process.nextTick(function () {
d.update({ a: 2 }, { a: 1 }, null);
d.update({ a: 2 }, { a: 1 }, null)
process.nextTick(function () {
d.remove({ a: 2 }, {}, null);
d.remove({ a: 2 }, {}, null)
process.nextTick(function () {
d.remove({ a: 2 }, null);
d.remove({ a: 2 }, null)
process.nextTick(function () {
d.find({}, done);
});
});
});
});
});
d.find({}, done)
})
})
})
})
})
}
// Test that operations are executed in the right order
// We prevent Mocha from catching the exception we throw on purpose by remembering all current handlers, remove them and register them back after test ends
function testRightOrder (d, done) {
var currentUncaughtExceptionHandlers = process.listeners('uncaughtException');
const currentUncaughtExceptionHandlers = process.listeners('uncaughtException')
process.removeAllListeners('uncaughtException');
process.removeAllListeners('uncaughtException')
// eslint-disable-next-line node/handle-callback-err
process.on('uncaughtException', function (err) {
// Do nothing with the error which is only there to test we stay on track
});
})
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs.length.should.equal(0);
docs.length.should.equal(0)
d.insert({ a: 1 }, function () {
d.update({ a: 1 }, { a: 2 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs[0].a.should.equal(2);
docs[0].a.should.equal(2)
process.nextTick(function () {
d.update({ a: 2 }, { a: 3 }, {}, function () {
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs[0].a.should.equal(3);
docs[0].a.should.equal(3)
process.removeAllListeners('uncaughtException');
for (var i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i]);
process.removeAllListeners('uncaughtException')
for (let i = 0; i < currentUncaughtExceptionHandlers.length; i += 1) {
process.on('uncaughtException', currentUncaughtExceptionHandlers[i])
}
done();
});
});
});
done()
})
})
})
throw new Error('Some error');
});
});
});
});
throw new Error('Some error')
})
})
})
})
}
// Note: The following test does not have any assertion because it
// is meant to address the deprecation warning:
// (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
// see
var testEventLoopStarvation = function(d, done){
var times = 1001;
var i = 0;
while ( i <times) {
i++;
d.find({"bogus": "search"}, function (err, docs) {
});
}
done();
};
const testEventLoopStarvation = function (d, done) {
const times = 1001
let i = 0
while (i < times) {
i++
// eslint-disable-next-line node/handle-callback-err
d.find({ bogus: 'search' }, function (err, docs) {
})
}
done()
}
// Test that operations are executed in the right order even with no callback
function testExecutorWorksWithoutCallback (d, done) {
d.insert({ a: 1 });
d.insert({ a: 2 }, false);
d.insert({ a: 1 })
d.insert({ a: 2 }, false)
// eslint-disable-next-line node/handle-callback-err
d.find({}, function (err, docs) {
docs.length.should.equal(2);
done();
});
docs.length.should.equal(2)
done()
})
}
describe('Executor', function () {
describe('With persistent database', function () {
var d;
let d
beforeEach(function (done) {
d = new Datastore({ filename: testDb });
d.filename.should.equal(testDb);
d.inMemoryOnly.should.equal(false);
d = new Datastore({ filename: testDb })
d.filename.should.equal(testDb)
d.inMemoryOnly.should.equal(false)
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
fs.exists(testDb, function (exists) {
if (exists) {
fs.unlink(testDb, cb);
} else { return cb(); }
});
});
}
, function (cb) {
fs.access(testDb, fs.constants.F_OK, function (err) {
if (!err) {
fs.unlink(testDb, cb)
} else { return cb() }
})
})
},
function (cb) {
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return cb();
});
assert.isNull(err)
d.getAllData().length.should.equal(0)
return cb()
})
}
], done);
});
it('A throw in a callback doesnt prevent execution of next operations', function(done) {
testThrowInCallback(d, done);
});
], done)
})
it('A falsy callback doesnt prevent execution of next operations', function(done) {
testFalsyCallback(d, done);
});
it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testThrowInCallback(d, done)
})
it('Operations are executed in the right order', function(done) {
testRightOrder(d, done);
});
it('A falsy callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done)
})
it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function(done){
testEventLoopStarvation(d, done);
});
it('Operations are executed in the right order', function (done) {
testRightOrder(d, done)
})
it('Works in the right order even with no supplied callback', function(done){
testExecutorWorksWithoutCallback(d, done);
});
}); // ==== End of 'With persistent database' ====
it('Does not starve event loop and raise warning when more than 1000 callbacks are in queue', function (done) {
testEventLoopStarvation(d, done)
})
it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done)
})
}) // ==== End of 'With persistent database' ====
describe('With non persistent database', function () {
var d;
let d
beforeEach(function (done) {
d = new Datastore({ inMemoryOnly: true });
d.inMemoryOnly.should.equal(true);
d = new Datastore({ inMemoryOnly: true })
d.inMemoryOnly.should.equal(true)
d.loadDatabase(function (err) {
assert.isNull(err);
d.getAllData().length.should.equal(0);
return done();
});
});
it('A throw in a callback doesnt prevent execution of next operations', function(done) {
testThrowInCallback(d, done);
});
it('A falsy callback doesnt prevent execution of next operations', function(done) {
testFalsyCallback(d, done);
});
it('Operations are executed in the right order', function(done) {
testRightOrder(d, done);
});
it('Works in the right order even with no supplied callback', function(done){
testExecutorWorksWithoutCallback(d, done);
});
}); // ==== End of 'With non persistent database' ====
});
assert.isNull(err)
d.getAllData().length.should.equal(0)
return done()
})
})
it('A throw in a callback doesnt prevent execution of next operations', function (done) {
testThrowInCallback(d, done)
})
it('A falsy callback doesnt prevent execution of next operations', function (done) {
testFalsyCallback(d, done)
})
it('Operations are executed in the right order', function (done) {
testRightOrder(d, done)
})
it('Works in the right order even with no supplied callback', function (done) {
testExecutorWorksWithoutCallback(d, done)
})
}) // ==== End of 'With non persistent database' ====
})

1231
test/indexes.test.js
View File

File diff suppressed because it is too large Load Diff

2
test/mocha.opts
Unescape View File

@ -1,2 +0,0 @@
--reporter spec
--timeout 30000

2178
test/model.test.js
View File

File diff suppressed because it is too large Load Diff

1331
test/persistence.test.js
View File

File diff suppressed because it is too large Load Diff

118
test_lac/loadAndCrash.test.js
Unescape View File

@ -1,123 +1,121 @@
/* eslint-env mocha */
/* global DEBUG */
/**
* Load and modify part of fs to ensure writeFile will crash after writing 5000 bytes
*/
var fs = require('fs');
const fs = require('fs')
function rethrow() {
function rethrow () {
// Only enable in debug mode. A backtrace uses ~1000 bytes of heap space and
// is fairly slow to generate.
if (DEBUG) {
var backtrace = new Error();
return function(err) {
const backtrace = new Error()
return function (err) {
if (err) {
backtrace.stack = err.name + ': ' + err.message +
backtrace.stack.substr(backtrace.name.length);
throw backtrace;
backtrace.stack.substr(backtrace.name.length)
throw backtrace
}
};
}
}
return function(err) {
return function (err) {
if (err) {
throw err; // Forgot a callback but don't know where? Use NODE_DEBUG=fs
throw err // Forgot a callback but don't know where? Use NODE_DEBUG=fs
}
};
}
}
function maybeCallback(cb) {
return typeof cb === 'function' ? cb : rethrow();
function maybeCallback (cb) {
return typeof cb === 'function' ? cb : rethrow()
}
function isFd(path) {
return (path >>> 0) === path;
function isFd (path) {
return (path >>> 0) === path
}
function assertEncoding(encoding) {
function assertEncoding (encoding) {
if (encoding && !Buffer.isEncoding(encoding)) {
throw new Error('Unknown encoding: ' + encoding);
throw new Error('Unknown encoding: ' + encoding)
}
}
var onePassDone = false;
function writeAll(fd, isUserFd, buffer, offset, length, position, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
let onePassDone = false
if (onePassDone) { process.exit(1); } // Crash on purpose before rewrite done
var l = Math.min(5000, length); // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
function writeAll (fd, isUserFd, buffer, offset, length, position, callback_) {
const callback = maybeCallback(arguments[arguments.length - 1])
if (onePassDone) { process.exit(1) } // Crash on purpose before rewrite done
const l = Math.min(5000, length) // Force write by chunks of 5000 bytes to ensure data will be incomplete on crash
// write(fd, buffer, offset, length, position, callback)
fs.write(fd, buffer, offset, l, position, function(writeErr, written) {
fs.write(fd, buffer, offset, l, position, function (writeErr, written) {
if (writeErr) {
if (isUserFd) {
if (callback) callback(writeErr);
if (callback) callback(writeErr)
} else {
fs.close(fd, function() {
if (callback) callback(writeErr);
});
fs.close(fd, function () {
if (callback) callback(writeErr)
})
}
} else {
onePassDone = true;
onePassDone = true
if (written === length) {
if (isUserFd) {
if (callback) callback(null);
if (callback) callback(null)
} else {
fs.close(fd, callback);
fs.close(fd, callback)
}
} else {
offset += written;
length -= written;
offset += written
length -= written
if (position !== null) {
position += written;
position += written
}
writeAll(fd, isUserFd, buffer, offset, length, position, callback);
writeAll(fd, isUserFd, buffer, offset, length, position, callback)
}
}
});
})
}
fs.writeFile = function(path, data, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
fs.writeFile = function (path, data, options, callback_) {
const callback = maybeCallback(arguments[arguments.length - 1])
if (!options || typeof options === 'function') {
options = { encoding: 'utf8', mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases)
options = { encoding: 'utf8', mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options === 'string') {
options = { encoding: options, mode: 438, flag: 'w' }; // Mode 438 == 0o666 (compatibility with older Node releases)
options = { encoding: options, mode: 438, flag: 'w' } // Mode 438 == 0o666 (compatibility with older Node releases)
} else if (typeof options !== 'object') {
throwOptionsError(options);
throw new Error(`throwOptionsError${options}`)
}
assertEncoding(options.encoding);
assertEncoding(options.encoding)
var flag = options.flag || 'w';
const flag = options.flag || 'w'
if (isFd(path)) {
writeFd(path, true);
return;
writeFd(path, true)
return
}
fs.open(path, flag, options.mode, function(openErr, fd) {
fs.open(path, flag, options.mode, function (openErr, fd) {
if (openErr) {
if (callback) callback(openErr);
if (callback) callback(openErr)
} else {
writeFd(fd, false);
writeFd(fd, false)
}
});
})
function writeFd(fd, isUserFd) {
var buffer = (data instanceof Buffer) ? data : new Buffer('' + data,
options.encoding || 'utf8');
var position = /a/.test(flag) ? null : 0;
function writeFd (fd, isUserFd) {
const buffer = (data instanceof Buffer) ? data : Buffer.from('' + data, options.encoding || 'utf8')
const position = /a/.test(flag) ? null : 0
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback);
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback)
}
};
}
// End of fs modification
var Nedb = require('../lib/datastore.js')
, db = new Nedb({ filename: 'workspace/lac.db' })
;
const Nedb = require('../lib/datastore.js')
const db = new Nedb({ filename: 'workspace/lac.db' })
db.loadDatabase();
db.loadDatabase()

99
test_lac/openFds.test.js
Unescape View File

@ -1,67 +1,64 @@
var fs = require('fs')
, child_process = require('child_process')
, async = require('async')
, Nedb = require('../lib/datastore')
, db = new Nedb({ filename: './workspace/openfds.db', autoload: true })
, N = 64 // Half the allowed file descriptors
, i, fds
;
const fs = require('fs')
const async = require('async')
const Nedb = require('../lib/datastore')
const db = new Nedb({ filename: './workspace/openfds.db', autoload: true })
const N = 64
let i
let fds
function multipleOpen (filename, N, callback) {
async.whilst( function () { return i < N; }
, function (cb) {
fs.open(filename, 'r', function (err, fd) {
i += 1;
if (fd) { fds.push(fd); }
return cb(err);
});
}
, callback);
async.whilst(function () { return i < N }
, function (cb) {
fs.open(filename, 'r', function (err, fd) {
i += 1
if (fd) { fds.push(fd) }
return cb(err)
})
}
, callback)
}
async.waterfall([
// Check that ulimit has been set to the correct value
function (cb) {
i = 0;
fds = [];
i = 0
fds = []
multipleOpen('./test_lac/openFdsTestFile', 2 * N + 1, function (err) {
if (!err) { console.log("No error occured while opening a file too many times"); }
fds.forEach(function (fd) { fs.closeSync(fd); });
return cb();
if (!err) { console.log('No error occured while opening a file too many times') }
fds.forEach(function (fd) { fs.closeSync(fd) })
return cb()
})
}
, function (cb) {
i = 0;
fds = [];
},
function (cb) {
i = 0
fds = []
multipleOpen('./test_lac/openFdsTestFile2', N, function (err) {
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err); }
fds.forEach(function (fd) { fs.closeSync(fd); });
return cb();
if (err) { console.log('An unexpected error occured when opening file not too many times: ' + err) }
fds.forEach(function (fd) { fs.closeSync(fd) })
return cb()
})
}
},
// Then actually test NeDB persistence
, function () {
function () {
db.remove({}, { multi: true }, function (err) {
if (err) { console.log(err); }
if (err) { console.log(err) }
db.insert({ hello: 'world' }, function (err) {
if (err) { console.log(err); }
if (err) { console.log(err) }
i = 0;
async.whilst( function () { return i < 2 * N + 1; }
, function (cb) {
db.persistence.persistCachedDatabase(function (err) {
if (err) { return cb(err); }
i += 1;
return cb();
});
}
, function (err) {
if (err) { console.log("Got unexpected error during one peresistence operation: " + err); }
}
);
});
});
i = 0
async.whilst(function () { return i < 2 * N + 1 }
, function (cb) {
db.persistence.persistCachedDatabase(function (err) {
if (err) { return cb(err) }
i += 1
return cb()
})
}
, function (err) {
if (err) { console.log('Got unexpected error during one peresistence operation: ' + err) }
}
)
})
})
}
]);
])

Write Preview
Loading…
Cancel
Save