From 856b1744f677639856cc426061bc998e140d3c1a Mon Sep 17 00:00:00 2001 From: "James M. Greene" Date: Wed, 16 Dec 2015 12:40:07 -0600 Subject: [PATCH 1/3] Mention that compactions eliminate corrupt records And a little housekeeping beyond that. --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 139040b..de4ba77 100755 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -## The Javascript Database +## The JavaScript Database -**Embedded persistent or in memory database for Node.js, nw.js, Electron and browsers, 100% Javascript, no binary dependency**. API is a subset of MongoDB's and it's plenty fast. +**Embedded persistent or in memory database for Node.js, nw.js, Electron and browsers, 100% JavaScript, no binary dependency**. API is a subset of MongoDB's and it's plenty fast. **IMPORTANT NOTE**: Please don't submit issues for questions regarding your code. Only actual bugs or feature requests will be answered, all others will be closed without comment. Also, please follow the bug reporting guidelines and check the change log before submitting an already fixed bug :) @@ -16,14 +16,15 @@ Bitcoin address: 1dDZLnWpBbodPiN8sizzYrgaz5iahFyb1 ## Installation, tests Module name on npm and bower is `nedb`. -```javascript -npm install nedb --save // Put latest version in your package.json -npm test // You'll need the dev dependencies to launch tests -bower install nedb // For the browser versions, which will be in browser-version/out + +``` +npm install nedb --save # Put latest version in your package.json +npm test # You'll need the dev dependencies to launch tests +bower install nedb # For the browser versions, which will be in browser-version/out ``` ## API -It's a subset of MongoDB's API (the most used operations). +It is a subset of MongoDB's API (the most used operations). * Creating/loading a database * Persistence @@ -44,18 +45,14 @@ It's a subset of MongoDB's API (the most used operations). ### Creating/loading a database You can use NeDB as an in-memory only datastore or as a persistent datastore. One datastore is the equivalent of a MongoDB collection. The constructor is used as follows `new Datastore(options)` where `options` is an object with the following fields: -* `filename` (optional): path to the file where the data is persisted. If left blank, the datastore is automatically considered in-memory only. It cannot end with a `~` which is used in the temporary files NeDB uses to perform crash-safe writes -* `inMemoryOnly` (optional, defaults to false): as the name implies. -* `timestampData` (optional, defaults to false): timestamp the insertion and last update of all documents, with the fields `createdAt` and `updatedAt`. User-specified values override automatic generation, usually useful for testing. -* `autoload` (optional, defaults to false): if used, the database will - automatically be loaded from the datafile upon creation (you don't -need to call `loadDatabase`). Any command -issued before load is finished is buffered and will be executed when -load is done. +* `filename` (optional): path to the file where the data is persisted. If left blank, the datastore is automatically considered in-memory only. It cannot end with a `~` which is used in the temporary files NeDB uses to perform crash-safe writes. +* `inMemoryOnly` (optional, defaults to `false`): as the name implies. +* `timestampData` (optional, defaults to `false`): timestamp the insertion and last update of all documents, with the fields `createdAt` and `updatedAt`. User-specified values override automatic generation, usually useful for testing. +* `autoload` (optional, defaults to `false`): if used, the database will automatically be loaded from the datafile upon creation (you don't need to call `loadDatabase`). Any command issued before load is finished is buffered and will be executed when load is done. * `onload` (optional): if you use autoloading, this is the handler called after the `loadDatabase`. It takes one `error` argument. If you use autoloading without specifying this handler, and an error happens during load, an error will be thrown. -* `afterSerialization` (optional): hook you can use to transform data after it was serialized and before it is written to disk. Can be used for example to encrypt data before writing database to disk. This function takes a string as parameter (one line of an NeDB data file) and outputs the transformed string, **which must absolutely not contain a `\n` character** (or data will be lost) -* `beforeDeserialization` (optional): reverse of `afterSerialization`. Make sure to include both and not just one or you risk data loss. For the same reason, make sure both functions are inverses of one another. Some failsafe mechanisms are in place to prevent data loss if you misuse the serialization hooks: NeDB checks that never one is declared without the other, and checks that they are reverse of one another by testing on random strings of various lengths. In addition, if too much data is detected as corrupt, NeDB will refuse to start as it could mean you're not using the deserialization hook corresponding to the serialization hook used before (see below) -* `corruptAlertThreshold` (optional): between 0 and 1, defaults to 10%. NeDB will refuse to start if more than this percentage of the datafile is corrupt. 0 means you don't tolerate any corruption, 1 means you don't care +* `afterSerialization` (optional): hook you can use to transform data after it was serialized and before it is written to disk. Can be used for example to encrypt data before writing database to disk. This function takes a string as parameter (one line of an NeDB data file) and outputs the transformed string, **which must absolutely not contain a `\n` character** (or data will be lost). +* `beforeDeserialization` (optional): inverse of `afterSerialization`. Make sure to include both and not just one or you risk data loss. For the same reason, make sure both functions are inverses of one another. Some failsafe mechanisms are in place to prevent data loss if you misuse the serialization hooks: NeDB checks that never one is declared without the other, and checks that they are reverse of one another by testing on random strings of various lengths. In addition, if too much data is detected as corrupt, NeDB will refuse to start as it could mean you're not using the deserialization hook corresponding to the serialization hook used before (see below). +* `corruptAlertThreshold` (optional): between 0 and 1, defaults to 10%. NeDB will refuse to start if more than this percentage of the datafile is corrupt. 0 means you don't tolerate any corruption, 1 means you don't care. * `nodeWebkitAppName` (optional, **DEPRECATED**): if you are using NeDB from whithin a Node Webkit app, specify its name (the same one you use in the `package.json`) in this field and the `filename` will be relative to the directory Node Webkit uses to store the rest of the application's data (local storage etc.). It works on Linux, OS X and Windows. Now that you can use `require('nw.gui').App.dataPath` in Node Webkit to get the path to the data directory for your application, you should not use this option anymore and it will be removed. If you use a persistent datastore without the `autoload` option, you need to call `loadDatabase` manually. @@ -102,7 +99,7 @@ db.robots.loadDatabase(); ``` ### Persistence -Under the hood, NeDB's persistence uses an append-only format, meaning that all updates and deletes actually result in lines added at the end of the datafile, for performance reasons. The database is automatically compacted (i.e. put back in the one-line-per-document format) everytime your application restarts. +Under the hood, NeDB's persistence uses an append-only format, meaning that all updates and deletes actually result in lines added at the end of the datafile, for performance reasons. The database is automatically compacted (i.e. put back in the one-line-per-document format) every time you load each database within your application. You can manually call the compaction function with `yourDatabase.persistence.compactDatafile` which takes no argument. It queues a compaction of the datafile in the executor, to be executed sequentially after all pending operations. @@ -110,6 +107,8 @@ You can also set automatic compaction at regular intervals with `yourDatabase.pe Keep in mind that compaction takes a bit of time (not too much: 130ms for 50k records on a typical development machine) and no other operation can happen when it does, so most projects actually don't need to use it. +Compaction will also immediately remove any documents whose data line has become corrupted, assuming that the total percentage of all corrupted documents in that database still falls below the specified `corruptAlertThreshold` option's value. + Durability works similarly to major databases: compaction forces the OS to physically flush data to disk, while appends to the data file do not (the OS is responsible for flushing the data). That guarantees that a server crash can never cause complete data loss, while preserving performance. The worst that can happen is a crash between two syncs, causing a loss of all data between the two syncs. Usually syncs are 30 seconds appart so that's at most 30 seconds of data. This post by Antirez on Redis persistence explains this in more details, NeDB being very close to Redis AOF persistence with `appendfsync` option set to `no`. @@ -140,6 +139,7 @@ db.insert(doc, function (err, newDoc) { // Callback is optional ``` You can also bulk-insert an array of documents. This operation is atomic, meaning that if one insert fails due to a unique constraint being violated, all changes are rolled back. + ```javascript db.insert([{ a: 5 }, { a: 42 }], function (err, newDocs) { // Two documents were inserted in the database From 446f420774ce11946161bb2553720c5018a3ea4b Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Mon, 21 Dec 2015 11:02:12 +0100 Subject: [PATCH 2/3] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index de4ba77..7168cab 100755 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ This function fetches the data from datafile and prepares the database. **Don't persistent datastore, no command (insert, find, update, remove) will be executed before `loadDatabase` is called, so make sure to call it yourself or use the `autoload` option. +Also, if `loadDatabase` fails, all commands registered to the executor afterwards will not be executed. They will be registered and executed, in sequence, only after a successful `loadDatabase`. + ```javascript // Type 1: In-memory only datastore (no need to load the database) var Datastore = require('nedb') From dab6fe24a579e1b5504d35d25c3a5c00cc137584 Mon Sep 17 00:00:00 2001 From: "James M. Greene" Date: Mon, 21 Dec 2015 13:45:25 -0600 Subject: [PATCH 3/3] Throw real Errors instead of strings and plain objects --- lib/cursor.js | 2 +- lib/model.js | 58 +++++++++++++++++++++++----------------------- lib/persistence.js | 18 +++++++------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/cursor.js b/lib/cursor.js index 20c6bb1..430a9b4 100755 --- a/lib/cursor.js +++ b/lib/cursor.js @@ -77,7 +77,7 @@ Cursor.prototype.project = function (candidates) { // Check for consistency keys = Object.keys(this._projection); keys.forEach(function (k) { - if (action !== undefined && self._projection[k] !== action) { throw "Can't both keep and omit fields except for _id"; } + if (action !== undefined && self._projection[k] !== action) { throw new Error("Can't both keep and omit fields except for _id"); } action = self._projection[k]; }); diff --git a/lib/model.js b/lib/model.js index ba85dba..6d7b950 100755 --- a/lib/model.js +++ b/lib/model.js @@ -29,11 +29,11 @@ function checkKey (k, v) { } if (k[0] === '$' && !(k === '$$date' && typeof v === 'number') && !(k === '$$deleted' && v === true) && !(k === '$$indexCreated') && !(k === '$$indexRemoved')) { - throw 'Field names cannot begin with the $ character'; + throw new Error('Field names cannot begin with the $ character'); } if (k.indexOf('.') !== -1) { - throw 'Field names cannot contain a .'; + throw new Error('Field names cannot contain a .'); } } @@ -266,11 +266,11 @@ lastStepModifierFunctions.$push = function (obj, field, value) { // Create the array if it doesn't exist if (!obj.hasOwnProperty(field)) { obj[field] = []; } - if (!util.isArray(obj[field])) { throw "Can't $push an element on non-array values"; } + if (!util.isArray(obj[field])) { throw new Error("Can't $push an element on non-array values"); } if (value !== null && typeof value === 'object' && value.$each) { - if (Object.keys(value).length > 1) { throw "Can't use another field in conjunction with $each"; } - if (!util.isArray(value.$each)) { throw "$each requires an array value"; } + if (Object.keys(value).length > 1) { throw new Error("Can't use another field in conjunction with $each"); } + if (!util.isArray(value.$each)) { throw new Error("$each requires an array value"); } value.$each.forEach(function (v) { obj[field].push(v); @@ -292,11 +292,11 @@ lastStepModifierFunctions.$addToSet = function (obj, field, value) { // Create the array if it doesn't exist if (!obj.hasOwnProperty(field)) { obj[field] = []; } - if (!util.isArray(obj[field])) { throw "Can't $addToSet an element on non-array values"; } + if (!util.isArray(obj[field])) { throw new Error("Can't $addToSet an element on non-array values"); } if (value !== null && typeof value === 'object' && value.$each) { - if (Object.keys(value).length > 1) { throw "Can't use another field in conjunction with $each"; } - if (!util.isArray(value.$each)) { throw "$each requires an array value"; } + if (Object.keys(value).length > 1) { throw new Error("Can't use another field in conjunction with $each"); } + if (!util.isArray(value.$each)) { throw new Error("$each requires an array value"); } value.$each.forEach(function (v) { lastStepModifierFunctions.$addToSet(obj, field, v); @@ -314,8 +314,8 @@ lastStepModifierFunctions.$addToSet = function (obj, field, value) { * Remove the first or last element of an array */ lastStepModifierFunctions.$pop = function (obj, field, value) { - if (!util.isArray(obj[field])) { throw "Can't $pop an element from non-array values"; } - if (typeof value !== 'number') { throw value + " isn't an integer, can't use it with $pop"; } + if (!util.isArray(obj[field])) { throw new Error("Can't $pop an element from non-array values"); } + if (typeof value !== 'number') { throw new Error(value + " isn't an integer, can't use it with $pop"); } if (value === 0) { return; } if (value > 0) { @@ -332,7 +332,7 @@ lastStepModifierFunctions.$pop = function (obj, field, value) { lastStepModifierFunctions.$pull = function (obj, field, value) { var arr, i; - if (!util.isArray(obj[field])) { throw "Can't $pull an element from non-array values"; } + if (!util.isArray(obj[field])) { throw new Error("Can't $pull an element from non-array values"); } arr = obj[field]; for (i = arr.length - 1; i >= 0; i -= 1) { @@ -347,13 +347,13 @@ lastStepModifierFunctions.$pull = function (obj, field, value) { * Increment a numeric field's value */ lastStepModifierFunctions.$inc = function (obj, field, value) { - if (typeof value !== 'number') { throw value + " must be a number"; } + if (typeof value !== 'number') { throw new Error(value + " must be a number"); } if (typeof obj[field] !== 'number') { if (!_.has(obj, field)) { obj[field] = value; } else { - throw "Don't use the $inc modifier on non-number fields"; + throw new Error("Don't use the $inc modifier on non-number fields"); } } else { obj[field] += value; @@ -390,10 +390,10 @@ function modify (obj, updateQuery) { , newDoc, modifiers ; - if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) { throw "You cannot change a document's _id"; } + if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) { throw new Error("You cannot change a document's _id"); } if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) { - throw "You cannot mix modifiers and normal fields"; + throw new Error("You cannot mix modifiers and normal fields"); } if (dollarFirstChars.length === 0) { @@ -407,12 +407,12 @@ function modify (obj, updateQuery) { modifiers.forEach(function (m) { var keys; - if (!modifierFunctions[m]) { throw "Unknown modifier " + m; } + if (!modifierFunctions[m]) { throw new Error("Unknown modifier " + m); } // Can't rely on Object.keys throwing on non objects since ES6{ // Not 100% satisfying as non objects can be interpreted as objects but no false negatives so we can live with it if (typeof updateQuery[m] !== 'object') { - throw "Modifier " + m + "'s argument must be an object"; + throw new Error("Modifier " + m + "'s argument must be an object"); } keys = Object.keys(updateQuery[m]); @@ -425,7 +425,7 @@ function modify (obj, updateQuery) { // Check result is valid and return it checkObject(newDoc); - if (obj._id !== newDoc._id) { throw "You can't change a document's _id"; } + if (obj._id !== newDoc._id) { throw new Error("You can't change a document's _id"); } return newDoc; }; @@ -550,7 +550,7 @@ comparisonFunctions.$ne = function (a, b) { comparisonFunctions.$in = function (a, b) { var i; - if (!util.isArray(b)) { throw "$in operator called with a non-array"; } + if (!util.isArray(b)) { throw new Error("$in operator called with a non-array"); } for (i = 0; i < b.length; i += 1) { if (areThingsEqual(a, b[i])) { return true; } @@ -560,13 +560,13 @@ comparisonFunctions.$in = function (a, b) { }; comparisonFunctions.$nin = function (a, b) { - if (!util.isArray(b)) { throw "$nin operator called with a non-array"; } + if (!util.isArray(b)) { throw new Error("$nin operator called with a non-array"); } return !comparisonFunctions.$in(a, b); }; comparisonFunctions.$regex = function (a, b) { - if (!util.isRegExp(b)) { throw "$regex operator called with non regular expression"; } + if (!util.isRegExp(b)) { throw new Error("$regex operator called with non regular expression"); } if (typeof a !== 'string') { return false @@ -592,7 +592,7 @@ comparisonFunctions.$exists = function (value, exists) { // Specific to arrays comparisonFunctions.$size = function (obj, value) { if (!util.isArray(obj)) { return false; } - if (value % 1 !== 0) { throw "$size operator called without an integer"; } + if (value % 1 !== 0) { throw new Error("$size operator called without an integer"); } return (obj.length == value); }; @@ -607,7 +607,7 @@ arrayComparisonFunctions.$size = true; logicalOperators.$or = function (obj, query) { var i; - if (!util.isArray(query)) { throw "$or operator used without an array"; } + if (!util.isArray(query)) { throw new Error("$or operator used without an array"); } for (i = 0; i < query.length; i += 1) { if (match(obj, query[i])) { return true; } @@ -625,7 +625,7 @@ logicalOperators.$or = function (obj, query) { logicalOperators.$and = function (obj, query) { var i; - if (!util.isArray(query)) { throw "$and operator used without an array"; } + if (!util.isArray(query)) { throw new Error("$and operator used without an array"); } for (i = 0; i < query.length; i += 1) { if (!match(obj, query[i])) { return false; } @@ -653,10 +653,10 @@ logicalOperators.$not = function (obj, query) { logicalOperators.$where = function (obj, fn) { var result; - if (!_.isFunction(fn)) { throw "$where operator used without a function"; } + if (!_.isFunction(fn)) { throw new Error("$where operator used without a function"); } result = fn.call(obj); - if (!_.isBoolean(result)) { throw "$where function must return boolean"; } + if (!_.isBoolean(result)) { throw new Error("$where function must return boolean"); } return result; }; @@ -684,7 +684,7 @@ function match (obj, query) { queryValue = query[queryKey]; if (queryKey[0] === '$') { - if (!logicalOperators[queryKey]) { throw "Unknown logical operator " + queryKey; } + if (!logicalOperators[queryKey]) { throw new Error("Unknown logical operator " + queryKey); } if (!logicalOperators[queryKey](obj, queryValue)) { return false; } } else { if (!matchQueryPart(obj, queryKey, queryValue)) { return false; } @@ -728,13 +728,13 @@ function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) { dollarFirstChars = _.filter(firstChars, function (c) { return c === '$'; }); if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) { - throw "You cannot mix operators and normal fields"; + throw new Error("You cannot mix operators and normal fields"); } // queryValue is an object of this form: { $comparisonOperator1: value1, ... } if (dollarFirstChars.length > 0) { for (i = 0; i < keys.length; i += 1) { - if (!comparisonFunctions[keys[i]]) { throw "Unknown comparison function " + keys[i]; } + if (!comparisonFunctions[keys[i]]) { throw new Error("Unknown comparison function " + keys[i]); } if (!comparisonFunctions[keys[i]](objValue, queryValue[keys[i]])) { return false; } } diff --git a/lib/persistence.js b/lib/persistence.js index 12fdfac..a83563a 100755 --- a/lib/persistence.js +++ b/lib/persistence.js @@ -29,15 +29,15 @@ function Persistence (options) { this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1; if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') { - throw "The datafile name can't end with a ~, which is reserved for crash safe backup files"; + throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files"); } // After serialization and before deserialization hooks with some basic sanity checks if (options.afterSerialization && !options.beforeDeserialization) { - throw "Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss"; + throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss"); } if (!options.afterSerialization && options.beforeDeserialization) { - throw "Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss"; + throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss"); } this.afterSerialization = options.afterSerialization || function (s) { return s; }; this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; }; @@ -45,7 +45,7 @@ function Persistence (options) { for (j = 0; j < 10; j += 1) { randomString = customUtils.uid(i); if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) { - throw "beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss"; + throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss"); } } } @@ -89,21 +89,21 @@ Persistence.getNWAppFilename = function (appName, relativeFilename) { case 'win32': case 'win64': home = process.env.LOCALAPPDATA || process.env.APPDATA; - if (!home) { throw "Couldn't find the base application data folder"; } + if (!home) { throw new Error("Couldn't find the base application data folder"); } home = path.join(home, appName); break; case 'darwin': home = process.env.HOME; - if (!home) { throw "Couldn't find the base application data directory"; } + if (!home) { throw new Error("Couldn't find the base application data directory"); } home = path.join(home, 'Library', 'Application Support', appName); break; case 'linux': home = process.env.HOME; - if (!home) { throw "Couldn't find the base application data directory"; } + if (!home) { throw new Error("Couldn't find the base application data directory"); } home = path.join(home, '.config', appName); break; default: - throw "Can't use the Node Webkit relative path for platform " + process.platform; + throw new Error("Can't use the Node Webkit relative path for platform " + process.platform); break; } @@ -235,7 +235,7 @@ Persistence.prototype.treatRawData = function (rawData) { // A bit lenient on corruption if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) { - throw "More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss" + throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss"); } Object.keys(dataById).forEach(function (k) {