cleanup code

pull/2/head
Timothée Rebours 4 years ago
parent 5c6561307d
commit e8cba5273a
  1. 23
      browser-version/lib/customUtils.js
  2. 22
      browser-version/lib/storage.js
  3. 88
      lib/cursor.js
  4. 4
      lib/customUtils.js
  5. 282
      lib/datastore.js
  6. 23
      lib/executor.js
  7. 92
      lib/indexes.js
  8. 550
      lib/model.js
  9. 185
      lib/persistence.js
  10. 60
      lib/storage.js

@ -7,7 +7,7 @@
* https://github.com/dominictarr/crypto-browserify
* NOTE: Math.random() does not guarantee "cryptographic quality" but we actually don't need it
*/
function randomBytes (size) {
const randomBytes = size => {
const bytes = new Array(size)
for (let i = 0, r; i < size; i++) {
@ -22,39 +22,32 @@ function randomBytes (size) {
* Taken from the base64-js module
* https://github.com/beatgammit/base64-js/
*/
function byteArrayToBase64 (uint8) {
const byteArrayToBase64 = uint8 => {
const lookup = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
const extraBytes = uint8.length % 3 // if we have 1 byte left, pad 2 bytes
let output = ''
let temp
let length
let i
function tripletToBase64 (num) {
return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F]
}
const tripletToBase64 = num => lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F]
// go through the array every three bytes, we'll deal with trailing stuff later
for (i = 0, length = uint8.length - extraBytes; i < length; i += 3) {
for (let i = 0, length = uint8.length - extraBytes; i < length; i += 3) {
temp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2])
output += tripletToBase64(temp)
}
// pad the end with zeros, but make sure to not forget the extra bytes
switch (extraBytes) {
case 1:
if (extraBytes === 1) {
temp = uint8[uint8.length - 1]
output += lookup[temp >> 2]
output += lookup[(temp << 4) & 0x3F]
output += '=='
break
case 2:
} else if (extraBytes === 2) {
temp = (uint8[uint8.length - 2] << 8) + (uint8[uint8.length - 1])
output += lookup[temp >> 10]
output += lookup[(temp >> 4) & 0x3F]
output += lookup[(temp << 2) & 0x3F]
output += '='
break
}
return output
@ -68,8 +61,6 @@ function byteArrayToBase64 (uint8) {
* The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision)
* See http://en.wikipedia.org/wiki/Birthday_problem
*/
function uid (len) {
return byteArrayToBase64(randomBytes(Math.ceil(Math.max(8, len * 2)))).replace(/[+/]/g, '').slice(0, len)
}
const uid = len => byteArrayToBase64(randomBytes(Math.ceil(Math.max(8, len * 2)))).replace(/[+/]/g, '').slice(0, len)
module.exports.uid = uid

@ -13,7 +13,7 @@ const store = localforage.createInstance({
storeName: 'nedbdata'
})
function exists (filename, cback) {
const exists = (filename, cback) => {
// eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, value) => {
if (value !== null) return cback(true) // Even if value is undefined, localforage returns null
@ -21,7 +21,7 @@ function exists (filename, cback) {
})
}
function rename (filename, newFilename, callback) {
const rename = (filename, newFilename, callback) => {
// eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, value) => {
if (value === null) store.removeItem(newFilename, () => callback())
@ -33,13 +33,13 @@ function rename (filename, newFilename, callback) {
})
}
function writeFile (filename, contents, options, callback) {
const writeFile = (filename, contents, options, callback) => {
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options }
store.setItem(filename, contents, () => callback())
}
function appendFile (filename, toAppend, options, callback) {
const appendFile = (filename, toAppend, options, callback) => {
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options }
@ -51,26 +51,22 @@ function appendFile (filename, toAppend, options, callback) {
})
}
function readFile (filename, options, callback) {
const readFile = (filename, options, callback) => {
// Options do not matter in browser setup
if (typeof options === 'function') { callback = options }
// eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, contents) => callback(null, contents || ''))
}
function unlink (filename, callback) {
const unlink = (filename, callback) => {
store.removeItem(filename, () => callback())
}
// Nothing to do, no directories will be used on the browser
function mkdir (dir, options, callback) {
return callback()
}
const mkdir = (dir, options, callback) => callback()
// Nothing to do, no data corruption possible in the brower
function ensureDatafileIntegrity (filename, callback) {
return callback(null)
}
// Nothing to do, no data corruption possible in the browser
const ensureDatafileIntegrity = (filename, callback) => callback(null)
// Interface
module.exports.exists = exists

@ -56,7 +56,6 @@ class Cursor {
*/
project (candidates) {
const res = []
const self = this
let action
if (this._projection === undefined || Object.keys(this._projection).length === 0) {
@ -69,31 +68,28 @@ class Cursor {
// Check for consistency
const keys = Object.keys(this._projection)
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error('Can\'t both keep and omit fields except for _id') }
action = self._projection[k]
keys.forEach(k => {
if (action !== undefined && this._projection[k] !== action) throw new Error('Can\'t both keep and omit fields except for _id')
action = this._projection[k]
})
// Do the actual projection
candidates.forEach(function (candidate) {
candidates.forEach(candidate => {
let toPush
if (action === 1) { // pick-type projection
toPush = { $set: {} }
keys.forEach(function (k) {
keys.forEach(k => {
toPush.$set[k] = model.getDotValue(candidate, k)
if (toPush.$set[k] === undefined) { delete toPush.$set[k] }
if (toPush.$set[k] === undefined) delete toPush.$set[k]
})
toPush = model.modify({}, toPush)
} else { // omit-type projection
toPush = { $unset: {} }
keys.forEach(function (k) { toPush.$unset[k] = true })
keys.forEach(k => { toPush.$unset[k] = true })
toPush = model.modify(candidate, toPush)
}
if (keepId) {
toPush._id = candidate._id
} else {
delete toPush._id
}
if (keepId) toPush._id = candidate._id
else delete toPush._id
res.push(toPush)
})
@ -111,38 +107,30 @@ class Cursor {
let res = []
let added = 0
let skipped = 0
const self = this
let error = null
let i
let keys
let key
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback)
} else {
return _callback(error, res)
}
const callback = (error, res) => {
if (this.execFn) return this.execFn(error, res, _callback)
else return _callback(error, res)
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err) }
this.db.getCandidates(this.query, (err, candidates) => {
if (err) return callback(err)
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
for (const candidate of candidates) {
if (model.match(candidate, this.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1
} else {
res.push(candidates[i])
if (!this._sort) {
if (this._skip && this._skip > skipped) skipped += 1
else {
res.push(candidate)
added += 1
if (self._limit && self._limit <= added) { break }
}
} else {
res.push(candidates[i])
if (this._limit && this._limit <= added) break
}
} else res.push(candidate)
}
}
} catch (err) {
@ -150,39 +138,33 @@ class Cursor {
}
// Apply all sorts
if (self._sort) {
keys = Object.keys(self._sort)
if (this._sort) {
keys = Object.keys(this._sort)
// Sorting
const criteria = []
for (i = 0; i < keys.length; i++) {
key = keys[i]
criteria.push({ key: key, direction: self._sort[key] })
}
res.sort(function (a, b) {
let criterion
let compare
let i
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i]
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings)
if (compare !== 0) {
return compare
}
keys.forEach(item => {
key = item
criteria.push({ key: key, direction: this._sort[key] })
})
res.sort((a, b) => {
for (const criterion of criteria) {
const compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), this.db.compareStrings)
if (compare !== 0) return compare
}
return 0
})
// Applying limit and skip
const limit = self._limit || res.length
const skip = self._skip || 0
const limit = this._limit || res.length
const skip = this._skip || 0
res = res.slice(skip, skip + limit)
}
// Apply projection
try {
res = self.project(res)
res = this.project(res)
} catch (e) {
error = e
res = undefined

@ -8,12 +8,10 @@ const crypto = require('crypto')
* The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision)
* See http://en.wikipedia.org/wiki/Birthday_problem
*/
function uid (len) {
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
const uid = len => crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
.toString('base64')
.replace(/[+/]/g, '')
.slice(0, len)
}
// Interface
module.exports.uid = uid

@ -64,7 +64,7 @@ class Datastore extends EventEmitter {
// This new executor is ready if we don't use persistence
// If we do, it will only be ready once loadDatabase is called
this.executor = new Executor()
if (this.inMemoryOnly) { this.executor.ready = true }
if (this.inMemoryOnly) this.executor.ready = true
// Indexed by field name, dot notation can be used
// _id is always indexed and since _ids are generated randomly the underlying
@ -76,9 +76,9 @@ class Datastore extends EventEmitter {
// Queue a load of the database right away and call the onload handler
// By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
if (this.autoload) {
this.loadDatabase(options.onload || function (err) {
if (err) { throw err }
})
this.loadDatabase(options.onload || (err => {
if (err) throw err
}))
}
}
@ -100,38 +100,32 @@ class Datastore extends EventEmitter {
* Reset all currently defined indexes
*/
resetIndexes (newData) {
const self = this
Object.keys(this.indexes).forEach(function (i) {
self.indexes[i].reset(newData)
})
for (const index of Object.values(this.indexes)) {
index.reset(newData)
}
}
/**
* Ensure an index is kept for this field. Same parameters as lib/indexes
* For now this function is synchronous, we need to test how much time it takes
* We use an async API for consistency with the rest of the code
* @param {Object} options
* @param {String} options.fieldName
* @param {Boolean} options.unique
* @param {Boolean} options.sparse
* @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index (only works on Date fields, not arrays of Date)
* @param {Function} cb Optional callback, signature: err
* @param {Function} callback Optional callback, signature: err
*/
ensureIndex (options, cb) {
let err
const callback = cb || function () {}
options = options || {}
ensureIndex (options = {}, callback = () => {}) {
if (!options.fieldName) {
err = new Error('Cannot create an index without a fieldName')
const err = new Error('Cannot create an index without a fieldName')
err.missingFieldName = true
return callback(err)
}
if (this.indexes[options.fieldName]) { return callback(null) }
if (this.indexes[options.fieldName]) return callback(null)
this.indexes[options.fieldName] = new Index(options)
if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds } // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
if (options.expireAfterSeconds !== undefined) this.ttlIndexes[options.fieldName] = options.expireAfterSeconds // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
try {
this.indexes[options.fieldName].insert(this.getAllData())
@ -141,8 +135,8 @@ class Datastore extends EventEmitter {
}
// We may want to force all options to be persisted including defaults, not just the ones passed the index creation function
this.persistence.persistNewState([{ $$indexCreated: options }], function (err) {
if (err) { return callback(err) }
this.persistence.persistNewState([{ $$indexCreated: options }], err => {
if (err) return callback(err)
return callback(null)
})
}
@ -150,15 +144,13 @@ class Datastore extends EventEmitter {
/**
* Remove an index
* @param {String} fieldName
* @param {Function} cb Optional callback, signature: err
* @param {Function} callback Optional callback, signature: err
*/
removeIndex (fieldName, cb) {
const callback = cb || function () {}
removeIndex (fieldName, callback = () => {}) {
delete this.indexes[fieldName]
this.persistence.persistNewState([{ $$indexRemoved: fieldName }], function (err) {
if (err) { return callback(err) }
this.persistence.persistNewState([{ $$indexRemoved: fieldName }], err => {
if (err) return callback(err)
return callback(null)
})
}
@ -167,12 +159,11 @@ class Datastore extends EventEmitter {
* Add one or several document(s) to all indexes
*/
addToIndexes (doc) {
let i
let failingIndex
let error
const keys = Object.keys(this.indexes)
for (i = 0; i < keys.length; i += 1) {
for (let i = 0; i < keys.length; i += 1) {
try {
this.indexes[keys[i]].insert(doc)
} catch (e) {
@ -184,7 +175,7 @@ class Datastore extends EventEmitter {
// If an error happened, we need to rollback the insert on all other indexes
if (error) {
for (i = 0; i < failingIndex; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].remove(doc)
}
@ -196,11 +187,9 @@ class Datastore extends EventEmitter {
* Remove one or several document(s) from all indexes
*/
removeFromIndexes (doc) {
const self = this
Object.keys(this.indexes).forEach(function (i) {
self.indexes[i].remove(doc)
})
for (const index of Object.values(this.indexes)) {
index.remove(doc)
}
}
/**
@ -209,12 +198,11 @@ class Datastore extends EventEmitter {
* If one update violates a constraint, all changes are rolled back
*/
updateIndexes (oldDoc, newDoc) {
let i
let failingIndex
let error
const keys = Object.keys(this.indexes)
for (i = 0; i < keys.length; i += 1) {
for (let i = 0; i < keys.length; i += 1) {
try {
this.indexes[keys[i]].update(oldDoc, newDoc)
} catch (e) {
@ -226,7 +214,7 @@ class Datastore extends EventEmitter {
// If an error happened, we need to rollback the update on all other indexes
if (error) {
for (i = 0; i < failingIndex; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].revertUpdate(oldDoc, newDoc)
}
@ -249,7 +237,6 @@ class Datastore extends EventEmitter {
*/
getCandidates (query, dontExpireStaleDocs, callback) {
const indexNames = Object.keys(this.indexes)
const self = this
let usableQueryKeys
if (typeof dontExpireStaleDocs === 'function') {
@ -259,71 +246,72 @@ class Datastore extends EventEmitter {
async.waterfall([
// STEP 1: get candidates list by checking indexes from most to least frequent usecase
function (cb) {
cb => {
// For a basic match
usableQueryKeys = []
Object.keys(query).forEach(function (k) {
Object.keys(query).forEach(k => {
if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.types.isDate(query[k]) || query[k] === null) {
usableQueryKeys.push(k)
}
})
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]))
return cb(null, this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]))
}
// For a $in match
usableQueryKeys = []
Object.keys(query).forEach(function (k) {
Object.keys(query).forEach(k => {
if (query[k] && Object.prototype.hasOwnProperty.call(query[k], '$in')) {
usableQueryKeys.push(k)
}
})
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in))
return cb(null, this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in))
}
// For a comparison match
usableQueryKeys = []
Object.keys(query).forEach(function (k) {
Object.keys(query).forEach(k => {
if (query[k] && (Object.prototype.hasOwnProperty.call(query[k], '$lt') || Object.prototype.hasOwnProperty.call(query[k], '$lte') || Object.prototype.hasOwnProperty.call(query[k], '$gt') || Object.prototype.hasOwnProperty.call(query[k], '$gte'))) {
usableQueryKeys.push(k)
}
})
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]))
return cb(null, this.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]))
}
// By default, return all the DB data
return cb(null, self.getAllData())
return cb(null, this.getAllData())
},
// STEP 2: remove all expired documents
function (docs) {
if (dontExpireStaleDocs) { return callback(null, docs) }
docs => {
if (dontExpireStaleDocs) return callback(null, docs)
const expiredDocsIds = []
const validDocs = []
const ttlIndexesFieldNames = Object.keys(self.ttlIndexes)
const ttlIndexesFieldNames = Object.keys(this.ttlIndexes)
docs.forEach(function (doc) {
docs.forEach(doc => {
let valid = true
ttlIndexesFieldNames.forEach(function (i) {
if (doc[i] !== undefined && util.types.isDate(doc[i]) && Date.now() > doc[i].getTime() + self.ttlIndexes[i] * 1000) {
ttlIndexesFieldNames.forEach(i => {
if (doc[i] !== undefined && util.types.isDate(doc[i]) && Date.now() > doc[i].getTime() + this.ttlIndexes[i] * 1000) {
valid = false
}
})
if (valid) { validDocs.push(doc) } else { expiredDocsIds.push(doc._id) }
if (valid) validDocs.push(doc)
else expiredDocsIds.push(doc._id)
})
async.eachSeries(expiredDocsIds, function (_id, cb) {
self._remove({ _id: _id }, {}, function (err) {
if (err) { return callback(err) }
async.eachSeries(expiredDocsIds, (_id, cb) => {
this._remove({ _id: _id }, {}, err => {
if (err) return callback(err)
return cb()
})
// eslint-disable-next-line node/handle-callback-err
}, function (err) {
}, err => {
// TODO: handle error
return callback(null, validDocs)
})
@ -332,12 +320,12 @@ class Datastore extends EventEmitter {
/**
* Insert a new document
* @param {Function} cb Optional callback, signature: err, insertedDoc
* @param {Document} newDoc
* @param {Function} callback Optional callback, signature: err, insertedDoc
*
* @api private Use Datastore.insert which has the same signature
*/
_insert (newDoc, cb) {
const callback = cb || function () {}
_insert (newDoc, callback = () => {}) {
let preparedDoc
try {
@ -347,8 +335,8 @@ class Datastore extends EventEmitter {
return callback(e)
}
this.persistence.persistNewState(Array.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) {
if (err) { return callback(err) }
this.persistence.persistNewState(Array.isArray(preparedDoc) ? preparedDoc : [preparedDoc], err => {
if (err) return callback(err)
return callback(null, model.deepCopy(preparedDoc))
})
}
@ -357,12 +345,10 @@ class Datastore extends EventEmitter {
* Create a new _id that's not already in use
*/
createNewId () {
let tentativeId = customUtils.uid(16)
let attemptId = customUtils.uid(16)
// Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely small, so this is O(1)
if (this.indexes._id.getMatching(tentativeId).length > 0) {
tentativeId = this.createNewId()
}
return tentativeId
if (this.indexes._id.getMatching(attemptId).length > 0) attemptId = this.createNewId()
return attemptId
}
/**
@ -372,17 +358,16 @@ class Datastore extends EventEmitter {
*/
prepareDocumentForInsertion (newDoc) {
let preparedDoc
const self = this
if (Array.isArray(newDoc)) {
preparedDoc = []
newDoc.forEach(function (doc) { preparedDoc.push(self.prepareDocumentForInsertion(doc)) })
newDoc.forEach(doc => { preparedDoc.push(this.prepareDocumentForInsertion(doc)) })
} else {
preparedDoc = model.deepCopy(newDoc)
if (preparedDoc._id === undefined) { preparedDoc._id = this.createNewId() }
if (preparedDoc._id === undefined) preparedDoc._id = this.createNewId()
const now = new Date()
if (this.timestampData && preparedDoc.createdAt === undefined) { preparedDoc.createdAt = now }
if (this.timestampData && preparedDoc.updatedAt === undefined) { preparedDoc.updatedAt = now }
if (this.timestampData && preparedDoc.createdAt === undefined) preparedDoc.createdAt = now
if (this.timestampData && preparedDoc.updatedAt === undefined) preparedDoc.updatedAt = now
model.checkObject(preparedDoc)
}
@ -394,11 +379,8 @@ class Datastore extends EventEmitter {
* @api private
*/
_insertInCache (preparedDoc) {
if (Array.isArray(preparedDoc)) {
this._insertMultipleDocsInCache(preparedDoc)
} else {
this.addToIndexes(preparedDoc)
}
if (Array.isArray(preparedDoc)) this._insertMultipleDocsInCache(preparedDoc)
else this.addToIndexes(preparedDoc)
}
/**
@ -407,22 +389,21 @@ class Datastore extends EventEmitter {
* @api private
*/
_insertMultipleDocsInCache (preparedDocs) {
let i
let failingI
let failingIndex
let error
for (i = 0; i < preparedDocs.length; i += 1) {
for (let i = 0; i < preparedDocs.length; i += 1) {
try {
this.addToIndexes(preparedDocs[i])
} catch (e) {
error = e
failingI = i
failingIndex = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.removeFromIndexes(preparedDocs[i])
}
@ -437,6 +418,7 @@ class Datastore extends EventEmitter {
/**
* Count all documents matching the query
* @param {Object} query MongoDB-style query
* @param {Function} callback Optional callback, signature: err, count
*/
count (query, callback) {
const cursor = new Cursor(this, query, function (err, docs, callback) {
@ -444,11 +426,8 @@ class Datastore extends EventEmitter {
return callback(null, docs.length)
})
if (typeof callback === 'function') {
cursor.exec(callback)
} else {
return cursor
}
if (typeof callback === 'function') cursor.exec(callback)
else return cursor
}
/**
@ -456,75 +435,58 @@ class Datastore extends EventEmitter {
* If no callback is passed, we return the cursor so that user can limit, skip and finally exec
* @param {Object} query MongoDB-style query
* @param {Object} projection MongoDB-style projection
* @param {Function} callback Optional callback, signature: err, docs
*/
find (query, projection, callback) {
switch (arguments.length) {
case 1:
if (arguments.length === 1) {
projection = {}
// callback is undefined, will return a cursor
break
case 2:
} else if (arguments.length === 2) {
if (typeof projection === 'function') {
callback = projection
projection = {}
} // If not assume projection is an object and callback undefined
break
}
const cursor = new Cursor(this, query, function (err, docs, callback) {
const res = []
let i
if (err) { return callback(err) }
for (i = 0; i < docs.length; i += 1) {
res.push(model.deepCopy(docs[i]))
}
const res = docs.map(doc => model.deepCopy(doc))
return callback(null, res)
})
cursor.projection(projection)
if (typeof callback === 'function') {
cursor.exec(callback)
} else {
return cursor
}
if (typeof callback === 'function') cursor.exec(callback)
else return cursor
}
/**
* Find one document matching the query
* @param {Object} query MongoDB-style query
* @param {Object} projection MongoDB-style projection
* @param {Function} callback Optional callback, signature: err, doc
*/
findOne (query, projection, callback) {
switch (arguments.length) {
case 1:
if (arguments.length === 1) {
projection = {}
// callback is undefined, will return a cursor
break
case 2:
} else if (arguments.length === 2) {
if (typeof projection === 'function') {
callback = projection
projection = {}
} // If not assume projection is an object and callback undefined
break
}
const cursor = new Cursor(this, query, function (err, docs, callback) {
if (err) { return callback(err) }
if (docs.length === 1) {
return callback(null, model.deepCopy(docs[0]))
} else {
return callback(null, null)
}
const cursor = new Cursor(this, query, (err, docs, callback) => {
if (err) return callback(err)
if (docs.length === 1) return callback(null, model.deepCopy(docs[0]))
else return callback(null, null)
})
cursor.projection(projection).limit(1)
if (typeof callback === 'function') {
cursor.exec(callback)
} else {
return cursor
}
if (typeof callback === 'function') cursor.exec(callback)
else return cursor
}
/**
@ -553,29 +515,24 @@ class Datastore extends EventEmitter {
* @api private Use Datastore.update which has the same signature
*/
_update (query, updateQuery, options, cb) {
const self = this
let numReplaced = 0
let i
if (typeof options === 'function') {
cb = options
options = {}
}
const callback = cb || function () {}
const callback = cb || (() => {})
const multi = options.multi !== undefined ? options.multi : false
const upsert = options.upsert !== undefined ? options.upsert : false
async.waterfall([
function (cb) { // If upsert option is set, check whether we need to insert the doc
if (!upsert) { return cb() }
cb => { // If upsert option is set, check whether we need to insert the doc
if (!upsert) return cb()
// Need to use an internal function not tied to the executor to avoid deadlock
const cursor = new Cursor(self, query)
cursor.limit(1)._exec(function (err, docs) {
if (err) { return callback(err) }
if (docs.length === 1) {
return cb()
} else {
const cursor = new Cursor(this, query)
cursor.limit(1)._exec((err, docs) => {
if (err) return callback(err)
if (docs.length === 1) return cb()
else {
let toBeInserted
try {
@ -592,34 +549,35 @@ class Datastore extends EventEmitter {
}
}
return self._insert(toBeInserted, function (err, newDoc) {
if (err) { return callback(err) }
return this._insert(toBeInserted, (err, newDoc) => {
if (err) return callback(err)
return callback(null, 1, newDoc, true)
})
}
})
},
function () { // Perform the update
() => { // Perform the update
let numReplaced = 0
let modifiedDoc
const modifications = []
let createdAt
self.getCandidates(query, function (err, candidates) {
if (err) { return callback(err) }
this.getCandidates(query, (err, candidates) => {
if (err) return callback(err)
// Preparing update (if an error is thrown here neither the datafile nor
// the in-memory indexes are affected)
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], query) && (multi || numReplaced === 0)) {
for (const candidate of candidates) {
if (model.match(candidate, query) && (multi || numReplaced === 0)) {
numReplaced += 1
if (self.timestampData) { createdAt = candidates[i].createdAt }
modifiedDoc = model.modify(candidates[i], updateQuery)
if (self.timestampData) {
if (this.timestampData) { createdAt = candidate.createdAt }
modifiedDoc = model.modify(candidate, updateQuery)
if (this.timestampData) {
modifiedDoc.createdAt = createdAt
modifiedDoc.updatedAt = new Date()
}
modifications.push({ oldDoc: candidates[i], newDoc: modifiedDoc })
modifications.push({ oldDoc: candidate, newDoc: modifiedDoc })
}
}
} catch (err) {
@ -628,21 +586,21 @@ class Datastore extends EventEmitter {
// Change the docs in memory
try {
self.updateIndexes(modifications)
this.updateIndexes(modifications)
} catch (err) {
return callback(err)
}
// Update the datafile
const updatedDocs = modifications.map(x => x.newDoc)
self.persistence.persistNewState(updatedDocs, function (err) {
if (err) { return callback(err) }
this.persistence.persistNewState(updatedDocs, err => {
if (err) return callback(err)
if (!options.returnUpdatedDocs) {
return callback(null, numReplaced)
} else {
let updatedDocsDC = []
updatedDocs.forEach(function (doc) { updatedDocsDC.push(model.deepCopy(doc)) })
if (!multi) { updatedDocsDC = updatedDocsDC[0] }
updatedDocs.forEach(doc => { updatedDocsDC.push(model.deepCopy(doc)) })
if (!multi) updatedDocsDC = updatedDocsDC[0]
return callback(null, numReplaced, updatedDocsDC)
}
})
@ -665,32 +623,32 @@ class Datastore extends EventEmitter {
* @api private Use Datastore.remove which has the same signature
*/
_remove (query, options, cb) {
const self = this
let numRemoved = 0
const removedDocs = []
if (typeof options === 'function') {
cb = options
options = {}
}
const callback = cb || function () {}
const callback = cb || (() => {})
const multi = options.multi !== undefined ? options.multi : false
this.getCandidates(query, true, function (err, candidates) {
if (err) { return callback(err) }
this.getCandidates(query, true, (err, candidates) => {
if (err) return callback(err)
const removedDocs = []
let numRemoved = 0
try {
candidates.forEach(function (d) {
candidates.forEach(d => {
if (model.match(d, query) && (multi || numRemoved === 0)) {
numRemoved += 1
removedDocs.push({ $$deleted: true, _id: d._id })
self.removeFromIndexes(d)
this.removeFromIndexes(d)
}
})
} catch (err) { return callback(err) }
} catch (err) {
return callback(err)
}
self.persistence.persistNewState(removedDocs, function (err) {
if (err) { return callback(err) }
this.persistence.persistNewState(removedDocs, err => {
if (err) return callback(err)
return callback(null, numRemoved)
})
})

@ -9,12 +9,11 @@ class Executor {
this.ready = false
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
const newArguments = []
this.queue = async.queue((task, cb) => {
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (let i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]) }
const lastArg = task.arguments[task.arguments.length - 1]
const newArguments = Array.from(task.arguments)
const lastArg = newArguments[newArguments.length - 1]
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
@ -29,10 +28,10 @@ class Executor {
}
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callback
newArguments[newArguments.length - 1] = function () { cb() }
newArguments[newArguments.length - 1] = () => { cb() }
} else {
// Nothing supplied as callback
newArguments.push(function () { cb() })
newArguments.push(() => { cb() })
}
task.fn.apply(task.this, newArguments)
@ -50,11 +49,8 @@ class Executor {
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/
push (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task)
} else {
this.buffer.push(task)
}
if (this.ready || forceQueuing) this.queue.push(task)
else this.buffer.push(task)
}
/**
@ -62,9 +58,8 @@ class Executor {
* Automatically sets executor as ready
*/
processBuffer () {
let i
this.ready = true
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]) }
this.buffer.forEach(task => { this.queue.push(task) })
this.buffer = []
}
}

@ -6,19 +6,17 @@ const { uniq } = require('./utils.js')
/**
* Two indexed pointers are equal iif they point to the same place
*/
function checkValueEquality (a, b) {
return a === b
}
const checkValueEquality = (a, b) => a === b
/**
* Type-aware projection
*/
function projectForUnique (elt) {
if (elt === null) { return '$null' }
if (typeof elt === 'string') { return '$string' + elt }
if (typeof elt === 'boolean') { return '$boolean' + elt }
if (typeof elt === 'number') { return '$number' + elt }
if (util.types.isDate(elt)) { return '$date' + elt.getTime() } // TODO: there is an obvious error here,
if (elt === null) return '$null'
if (typeof elt === 'string') return '$string' + elt
if (typeof elt === 'boolean') return '$boolean' + elt
if (typeof elt === 'number') return '$number' + elt
if (util.types.isDate(elt)) return '$date' + elt.getTime()
return elt // Arrays and objects, will check for pointer equality
}
@ -50,7 +48,7 @@ class Index {
reset (newData) {
this.tree = new BinarySearchTree(this.treeOptions)
if (newData) { this.insert(newData) }
if (newData) this.insert(newData)
}
/**
@ -60,8 +58,7 @@ class Index {
*/
insert (doc) {
let keys
let i
let failingI
let failingIndex
let error
if (Array.isArray(doc)) {
@ -72,26 +69,25 @@ class Index {
const key = model.getDotValue(doc, this.fieldName)
// We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return }
if (key === undefined && this.sparse) return
if (!Array.isArray(key)) {
this.tree.insert(key, doc)
} else {
if (!Array.isArray(key)) this.tree.insert(key, doc)
else {
// If an insert fails due to a unique constraint, roll back all inserts before it
keys = uniq(key, projectForUnique)
for (i = 0; i < keys.length; i += 1) {
for (let i = 0; i < keys.length; i += 1) {
try {
this.tree.insert(keys[i], doc)
} catch (e) {
error = e
failingI = i
failingIndex = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.tree.delete(keys[i], doc)
}
@ -107,22 +103,21 @@ class Index {
* @API private
*/
insertMultipleDocs (docs) {
let i
let error
let failingI
let failingIndex
for (i = 0; i < docs.length; i += 1) {
for (let i = 0; i < docs.length; i += 1) {
try {
this.insert(docs[i])
} catch (e) {
error = e
failingI = i
failingIndex = i
break
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.remove(docs[i])
}
@ -137,22 +132,20 @@ class Index {
* O(log(n))
*/
remove (doc) {
const self = this
if (Array.isArray(doc)) {
doc.forEach(function (d) { self.remove(d) })
doc.forEach(d => { this.remove(d) })
return
}
const key = model.getDotValue(doc, this.fieldName)
if (key === undefined && this.sparse) { return }
if (key === undefined && this.sparse) return
if (!Array.isArray(key)) {
this.tree.delete(key, doc)
} else {
uniq(key, projectForUnique).forEach(function (_key) {
self.tree.delete(_key, doc)
uniq(key, projectForUnique).forEach(_key => {
this.tree.delete(_key, doc)
})
}
}
@ -187,31 +180,30 @@ class Index {
* @API private
*/
updateMultipleDocs (pairs) {
let i
let failingI
let failingIndex
let error
for (i = 0; i < pairs.length; i += 1) {
for (let i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc)
}
for (i = 0; i < pairs.length; i += 1) {
for (let i = 0; i < pairs.length; i += 1) {
try {
this.insert(pairs[i].newDoc)
} catch (e) {
error = e
failingI = i
failingIndex = i
break
}
}
// If an error was raised, roll back changes in the inverse order
if (error) {
for (i = 0; i < failingI; i += 1) {
for (let i = 0; i < failingIndex; i += 1) {
this.remove(pairs[i].newDoc)
}
for (i = 0; i < pairs.length; i += 1) {
for (let i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc)
}
@ -225,10 +217,9 @@ class Index {
revertUpdate (oldDoc, newDoc) {
const revert = []
if (!Array.isArray(oldDoc)) {
this.update(newDoc, oldDoc)
} else {
oldDoc.forEach(function (pair) {
if (!Array.isArray(oldDoc)) this.update(newDoc, oldDoc)
else {
oldDoc.forEach(pair => {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc })
})
this.update(revert)
@ -241,21 +232,18 @@ class Index {
* @return {Array of documents}
*/
getMatching (value) {
const self = this
if (!Array.isArray(value)) {
return self.tree.search(value)
} else {
if (!Array.isArray(value)) return this.tree.search(value)
else {
const _res = {}
const res = []
value.forEach(function (v) {
self.getMatching(v).forEach(function (doc) {
value.forEach(v => {
this.getMatching(v).forEach(doc => {
_res[doc._id] = doc
})
})
Object.keys(_res).forEach(function (_id) {
Object.keys(_res).forEach(_id => {
res.push(_res[_id])
})
@ -280,12 +268,8 @@ class Index {
getAll () {
const res = []
this.tree.executeOnEveryNode(function (node) {
let i
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i])
}
this.tree.executeOnEveryNode(node => {
res.push(...node.data)
})
return res

@ -20,36 +20,38 @@ const arrayComparisonFunctions = {}
* Its serialized-then-deserialized version it will transformed into a Date object
* But you really need to want it to trigger such behaviour, even when warned not to use '$' at the beginning of the field names...
*/
function checkKey (k, v) {
if (typeof k === 'number') {
k = k.toString()
}
const checkKey = (k, v) => {
if (typeof k === 'number') k = k.toString()
if (k[0] === '$' && !(k === '$$date' && typeof v === 'number') && !(k === '$$deleted' && v === true) && !(k === '$$indexCreated') && !(k === '$$indexRemoved')) {
throw new Error('Field names cannot begin with the $ character')
}
if (
k[0] === '$' &&
!(k === '$$date' && typeof v === 'number') &&
!(k === '$$deleted' && v === true) &&
!(k === '$$indexCreated') &&
!(k === '$$indexRemoved')
) throw new Error('Field names cannot begin with the $ character')
if (k.indexOf('.') !== -1) {
throw new Error('Field names cannot contain a .')
}
if (k.indexOf('.') !== -1) throw new Error('Field names cannot contain a .')
}
/**
* Check a DB object and throw an error if it's not valid
* Works by applying the above checkKey function to all fields recursively
*/
function checkObject (obj) {
const checkObject = obj => {
if (Array.isArray(obj)) {
obj.forEach(function (o) {
obj.forEach(o => {
checkObject(o)
})
}
if (typeof obj === 'object' && obj !== null) {
Object.keys(obj).forEach(function (k) {
for (const k in obj) {
if (Object.prototype.hasOwnProperty.call(obj, k)) {
checkKey(k, obj[k])
checkObject(obj[k])
})
}
}
}
}
@ -61,36 +63,37 @@ function checkObject (obj) {
* Accepted primitive types: Number, String, Boolean, Date, null
* Accepted secondary types: Objects, Arrays
*/
function serialize (obj) {
const res = JSON.stringify(obj, function (k, v) {
const serialize = obj => {
return JSON.stringify(obj, function (k, v) {
checkKey(k, v)
if (v === undefined) { return undefined }
if (v === null) { return null }
if (v === undefined) return undefined
if (v === null) return null
// Hackish way of checking if object is Date (this way it works between execution contexts in node-webkit).
// We can't use value directly because for dates it is already string in this function (date.toJSON was already called), so we use this
if (typeof this[k].getTime === 'function') { return { $$date: this[k].getTime() } }
if (typeof this[k].getTime === 'function') return { $$date: this[k].getTime() }
return v
})
return res
}
/**
* From a one-line representation of an object generate by the serialize function
* Return the object itself
*/
function deserialize (rawData) {
return JSON.parse(rawData, function (k, v) {
if (k === '$$date') { return new Date(v) }
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null) { return v }
if (v && v.$$date) { return v.$$date }
const deserialize = rawData => JSON.parse(rawData, function (k, v) {
if (k === '$$date') return new Date(v)
if (
typeof v === 'string' ||
typeof v === 'number' ||
typeof v === 'boolean' ||
v === null
) return v
if (v && v.$$date) return v.$$date
return v
})
}
/**
* Deep copy a DB object
@ -98,29 +101,26 @@ function deserialize (rawData) {
* where the keys are valid, i.e. don't begin with $ and don't contain a .
*/
function deepCopy (obj, strictKeys) {
let res
if (typeof obj === 'boolean' ||
if (
typeof obj === 'boolean' ||
typeof obj === 'number' ||
typeof obj === 'string' ||
obj === null ||
(util.types.isDate(obj))) {
return obj
}
(util.types.isDate(obj))
) return obj
if (Array.isArray(obj)) {
res = []
obj.forEach(function (o) { res.push(deepCopy(o, strictKeys)) })
return res
}
if (Array.isArray(obj)) return obj.map(o => deepCopy(o, strictKeys))
if (typeof obj === 'object') {
res = {}
Object.keys(obj).forEach(function (k) {
if (!strictKeys || (k[0] !== '$' && k.indexOf('.') === -1)) {
const res = {}
for (const k in obj) {
if (
Object.prototype.hasOwnProperty.call(obj, k) &&
(!strictKeys || (k[0] !== '$' && k.indexOf('.') === -1))
) {
res[k] = deepCopy(obj[k], strictKeys)
}
})
}
return res
}
@ -131,34 +131,32 @@ function deepCopy (obj, strictKeys) {
* Tells if an object is a primitive type or a "real" object
* Arrays are considered primitive
*/
function isPrimitiveType (obj) {
return (typeof obj === 'boolean' ||
const isPrimitiveType = obj => (
typeof obj === 'boolean' ||
typeof obj === 'number' ||
typeof obj === 'string' ||
obj === null ||
util.types.isDate(obj) ||
Array.isArray(obj))
}
Array.isArray(obj)
)
/**
* Utility functions for comparing things
* Assumes type checking was already done (a and b already have the same type)
* compareNSB works for numbers, strings and booleans
*/
function compareNSB (a, b) {
if (a < b) { return -1 }
if (a > b) { return 1 }
const compareNSB = (a, b) => {
if (a < b) return -1
if (a > b) return 1
return 0
}
function compareArrays (a, b) {
let i
let comp
const compareArrays = (a, b) => {
const minLength = Math.min(a.length, b.length)
for (let i = 0; i < minLength; i += 1) {
const comp = compareThings(a[i], b[i])
for (i = 0; i < Math.min(a.length, b.length); i += 1) {
comp = compareThings(a[i], b[i])
if (comp !== 0) { return comp }
if (comp !== 0) return comp
}
// Common section was identical, longest one wins
@ -175,47 +173,45 @@ function compareArrays (a, b) {
*
* @param {Function} _compareStrings String comparing function, returning -1, 0 or 1, overriding default string comparison (useful for languages with accented letters)
*/
function compareThings (a, b, _compareStrings) {
let comp
let i
const compareThings = (a, b, _compareStrings) => {
const compareStrings = _compareStrings || compareNSB
// undefined
if (a === undefined) { return b === undefined ? 0 : -1 }
if (b === undefined) { return a === undefined ? 0 : 1 }
if (a === undefined) return b === undefined ? 0 : -1
if (b === undefined) return 1 // no need to test if a === undefined
// null
if (a === null) { return b === null ? 0 : -1 }
if (b === null) { return a === null ? 0 : 1 }
if (a === null) return b === null ? 0 : -1
if (b === null) return 1 // no need to test if a === null
// Numbers
if (typeof a === 'number') { return typeof b === 'number' ? compareNSB(a, b) : -1 }
if (typeof b === 'number') { return typeof a === 'number' ? compareNSB(a, b) : 1 }
if (typeof a === 'number') return typeof b === 'number' ? compareNSB(a, b) : -1
if (typeof b === 'number') return typeof a === 'number' ? compareNSB(a, b) : 1
// Strings
if (typeof a === 'string') { return typeof b === 'string' ? compareStrings(a, b) : -1 }
if (typeof b === 'string') { return typeof a === 'string' ? compareStrings(a, b) : 1 }
if (typeof a === 'string') return typeof b === 'string' ? compareStrings(a, b) : -1
if (typeof b === 'string') return typeof a === 'string' ? compareStrings(a, b) : 1
// Booleans
if (typeof a === 'boolean') { return typeof b === 'boolean' ? compareNSB(a, b) : -1 }
if (typeof b === 'boolean') { return typeof a === 'boolean' ? compareNSB(a, b) : 1 }
if (typeof a === 'boolean') return typeof b === 'boolean' ? compareNSB(a, b) : -1
if (typeof b === 'boolean') return typeof a === 'boolean' ? compareNSB(a, b) : 1
// Dates
if (util.types.isDate(a)) { return util.types.isDate(b) ? compareNSB(a.getTime(), b.getTime()) : -1 }
if (util.types.isDate(b)) { return util.types.isDate(a) ? compareNSB(a.getTime(), b.getTime()) : 1 }
if (util.types.isDate(a)) return util.types.isDate(b) ? compareNSB(a.getTime(), b.getTime()) : -1
if (util.types.isDate(b)) return util.types.isDate(a) ? compareNSB(a.getTime(), b.getTime()) : 1
// Arrays (first element is most significant and so on)
if (Array.isArray(a)) { return Array.isArray(b) ? compareArrays(a, b) : -1 }
if (Array.isArray(b)) { return Array.isArray(a) ? compareArrays(a, b) : 1 }
if (Array.isArray(a)) return Array.isArray(b) ? compareArrays(a, b) : -1
if (Array.isArray(b)) return Array.isArray(a) ? compareArrays(a, b) : 1
// Objects
const aKeys = Object.keys(a).sort()
const bKeys = Object.keys(b).sort()
for (i = 0; i < Math.min(aKeys.length, bKeys.length); i += 1) {
comp = compareThings(a[aKeys[i]], b[bKeys[i]])
for (let i = 0; i < Math.min(aKeys.length, bKeys.length); i += 1) {
const comp = compareThings(a[aKeys[i]], b[bKeys[i]])
if (comp !== 0) { return comp }
if (comp !== 0) return comp
}
return compareNSB(aKeys.length, bKeys.length)
@ -237,14 +233,14 @@ function compareThings (a, b, _compareStrings) {
/**
* Set a field to a new value
*/
lastStepModifierFunctions.$set = function (obj, field, value) {
lastStepModifierFunctions.$set = (obj, field, value) => {
obj[field] = value
}
/**
* Unset a field
*/
lastStepModifierFunctions.$unset = function (obj, field, value) {
lastStepModifierFunctions.$unset = (obj, field, value) => {
delete obj[field]
}
@ -254,29 +250,34 @@ lastStepModifierFunctions.$unset = function (obj, field, value) {
* Optional modifier $slice to slice the resulting array, see https://docs.mongodb.org/manual/reference/operator/update/slice/
* Différeence with MongoDB: if $slice is specified and not $each, we act as if value is an empty array
*/
lastStepModifierFunctions.$push = function (obj, field, value) {
lastStepModifierFunctions.$push = (obj, field, value) => {
// Create the array if it doesn't exist
if (!Object.prototype.hasOwnProperty.call(obj, field)) { obj[field] = [] }
if (!Object.prototype.hasOwnProperty.call(obj, field)) obj[field] = []
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $push an element on non-array values') }
if (!Array.isArray(obj[field])) throw new Error('Can\'t $push an element on non-array values')
if (value !== null && typeof value === 'object' && value.$slice && value.$each === undefined) {
value.$each = []
}
if (
value !== null &&
typeof value === 'object' &&
value.$slice &&
value.$each === undefined
) value.$each = []
if (value !== null && typeof value === 'object' && value.$each) {
if (Object.keys(value).length >= 3 || (Object.keys(value).length === 2 && value.$slice === undefined)) { throw new Error('Can only use $slice in cunjunction with $each when $push to array') }
if (!Array.isArray(value.$each)) { throw new Error('$each requires an array value') }
if (
Object.keys(value).length >= 3 ||
(Object.keys(value).length === 2 && value.$slice === undefined)
) throw new Error('Can only use $slice in cunjunction with $each when $push to array')
if (!Array.isArray(value.$each)) throw new Error('$each requires an array value')
value.$each.forEach(function (v) {
value.$each.forEach(v => {
obj[field].push(v)
})
if (value.$slice === undefined || typeof value.$slice !== 'number') { return }
if (value.$slice === undefined || typeof value.$slice !== 'number') return
if (value.$slice === 0) {
obj[field] = []
} else {
if (value.$slice === 0) obj[field] = []
else {
let start
let end
const n = obj[field].length
@ -299,134 +300,112 @@ lastStepModifierFunctions.$push = function (obj, field, value) {
* No modification if the element is already in the array
* Note that it doesn't check whether the original array contains duplicates
*/
lastStepModifierFunctions.$addToSet = function (obj, field, value) {
let addToSet = true
lastStepModifierFunctions.$addToSet = (obj, field, value) => {
// Create the array if it doesn't exist
if (!Object.prototype.hasOwnProperty.call(obj, field)) { obj[field] = [] }
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $addToSet an element on non-array values') }
if (!Array.isArray(obj[field])) throw new Error('Can\'t $addToSet an element on non-array values')
if (value !== null && typeof value === 'object' && value.$each) {
if (Object.keys(value).length > 1) { throw new Error('Can\'t use another field in conjunction with $each') }
if (!Array.isArray(value.$each)) { throw new Error('$each requires an array value') }
if (Object.keys(value).length > 1) throw new Error('Can\'t use another field in conjunction with $each')
if (!Array.isArray(value.$each)) throw new Error('$each requires an array value')
value.$each.forEach(function (v) {
value.$each.forEach(v => {
lastStepModifierFunctions.$addToSet(obj, field, v)
})
} else {
obj[field].forEach(function (v) {
if (compareThings(v, value) === 0) { addToSet = false }
let addToSet = true
obj[field].forEach(v => {
if (compareThings(v, value) === 0) addToSet = false
})
if (addToSet) { obj[field].push(value) }
if (addToSet) obj[field].push(value)
}
}
/**
* Remove the first or last element of an array
*/
lastStepModifierFunctions.$pop = function (obj, field, value) {
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $pop an element from non-array values') }
if (typeof value !== 'number') { throw new Error(value + ' isn\'t an integer, can\'t use it with $pop') }
if (value === 0) { return }
lastStepModifierFunctions.$pop = (obj, field, value) => {
if (!Array.isArray(obj[field])) throw new Error('Can\'t $pop an element from non-array values')
if (typeof value !== 'number') throw new Error(`${value} isn't an integer, can't use it with $pop`)
if (value === 0) return
if (value > 0) {
obj[field] = obj[field].slice(0, obj[field].length - 1)
} else {
obj[field] = obj[field].slice(1)
}
if (value > 0) obj[field] = obj[field].slice(0, obj[field].length - 1)
else obj[field] = obj[field].slice(1)
}
/**
* Removes all instances of a value from an existing array
*/
lastStepModifierFunctions.$pull = function (obj, field, value) {
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $pull an element from non-array values') }
lastStepModifierFunctions.$pull = (obj, field, value) => {
if (!Array.isArray(obj[field])) throw new Error('Can\'t $pull an element from non-array values')
const arr = obj[field]
for (let i = arr.length - 1; i >= 0; i -= 1) {
if (match(arr[i], value)) {
arr.splice(i, 1)
}
if (match(arr[i], value)) arr.splice(i, 1)
}
}
/**
* Increment a numeric field's value
*/
lastStepModifierFunctions.$inc = function (obj, field, value) {
if (typeof value !== 'number') { throw new Error(value + ' must be a number') }
lastStepModifierFunctions.$inc = (obj, field, value) => {
if (typeof value !== 'number') throw new Error(`${value} must be a number`)
if (typeof obj[field] !== 'number') {
if (!Object.prototype.hasOwnProperty.call(obj, field)) {
obj[field] = value
} else {
throw new Error('Don\'t use the $inc modifier on non-number fields')
}
} else {
obj[field] += value
}
if (!Object.prototype.hasOwnProperty.call(obj, field)) obj[field] = value
else throw new Error('Don\'t use the $inc modifier on non-number fields')
} else obj[field] += value
}
/**
* Updates the value of the field, only if specified field is greater than the current value of the field
*/
lastStepModifierFunctions.$max = function (obj, field, value) {
if (typeof obj[field] === 'undefined') {
obj[field] = value
} else if (value > obj[field]) {
obj[field] = value
}
lastStepModifierFunctions.$max = (obj, field, value) => {
if (typeof obj[field] === 'undefined') obj[field] = value
else if (value > obj[field]) obj[field] = value
}
/**
* Updates the value of the field, only if specified field is smaller than the current value of the field
*/
lastStepModifierFunctions.$min = function (obj, field, value) {
if (typeof obj[field] === 'undefined') {
obj[field] = value
} else if (value < obj[field]) {
obj[field] = value
}
lastStepModifierFunctions.$min = (obj, field, value) => {
if (typeof obj[field] === 'undefined') obj[field] = value
else if (value < obj[field]) obj[field] = value
}
// Given its name, create the complete modifier function
function createModifierFunction (modifier) {
return function (obj, field, value) {
const createModifierFunction = modifier => (obj, field, value) => {
const fieldParts = typeof field === 'string' ? field.split('.') : field
if (fieldParts.length === 1) {
lastStepModifierFunctions[modifier](obj, field, value)
} else {
if (fieldParts.length === 1) lastStepModifierFunctions[modifier](obj, field, value)
else {
if (obj[fieldParts[0]] === undefined) {
if (modifier === '$unset') { return } // Bad looking specific fix, needs to be generalized modifiers that behave like $unset are implemented
if (modifier === '$unset') return // Bad looking specific fix, needs to be generalized modifiers that behave like $unset are implemented
obj[fieldParts[0]] = {}
}
modifierFunctions[modifier](obj[fieldParts[0]], fieldParts.slice(1), value)
}
}
}
// Actually create all modifier functions
Object.keys(lastStepModifierFunctions).forEach(function (modifier) {
Object.keys(lastStepModifierFunctions).forEach(modifier => {
modifierFunctions[modifier] = createModifierFunction(modifier)
})
/**
* Modify a DB object according to an update query
*/
function modify (obj, updateQuery) {
const modify = (obj, updateQuery) => {
const keys = Object.keys(updateQuery)
const firstChars = keys.map(item => item[0])
const dollarFirstChars = firstChars.filter(c => c === '$')
let newDoc
let modifiers
if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) { throw new Error('You cannot change a document\'s _id') }
if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) throw new Error('You cannot change a document\'s _id')
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) {
throw new Error('You cannot mix modifiers and normal fields')
}
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) throw new Error('You cannot mix modifiers and normal fields')
if (dollarFirstChars.length === 0) {
// Simply replace the object with the update query contents
@ -436,17 +415,15 @@ function modify (obj, updateQuery) {
// Apply modifiers
modifiers = uniq(keys)
newDoc = deepCopy(obj)
modifiers.forEach(function (m) {
if (!modifierFunctions[m]) { throw new Error('Unknown modifier ' + m) }
modifiers.forEach(m => {
if (!modifierFunctions[m]) throw new Error(`Unknown modifier ${m}`)
// Can't rely on Object.keys throwing on non objects since ES6
// Not 100% satisfying as non objects can be interpreted as objects but no false negatives so we can live with it
if (typeof updateQuery[m] !== 'object') {
throw new Error('Modifier ' + m + '\'s argument must be an object')
}
if (typeof updateQuery[m] !== 'object') throw new Error(`Modifier ${m}'s argument must be an object`)
const keys = Object.keys(updateQuery[m])
keys.forEach(function (k) {
keys.forEach(k => {
modifierFunctions[m](newDoc, k, updateQuery[m][k])
})
})
@ -455,7 +432,7 @@ function modify (obj, updateQuery) {
// Check result is valid and return it
checkObject(newDoc)
if (obj._id !== newDoc._id) { throw new Error('You can\'t change a document\'s _id') }
if (obj._id !== newDoc._id) throw new Error('You can\'t change a document\'s _id')
return newDoc
}
@ -468,33 +445,23 @@ function modify (obj, updateQuery) {
* @param {Object} obj
* @param {String} field
*/
function getDotValue (obj, field) {
const getDotValue = (obj, field) => {
const fieldParts = typeof field === 'string' ? field.split('.') : field
let i
let objs
if (!obj) { return undefined } // field cannot be empty so that means we should return undefined so that nothing can match
if (!obj) return undefined // field cannot be empty so that means we should return undefined so that nothing can match
if (fieldParts.length === 0) { return obj }
if (fieldParts.length === 0) return obj
if (fieldParts.length === 1) { return obj[fieldParts[0]] }
if (fieldParts.length === 1) return obj[fieldParts[0]]
if (Array.isArray(obj[fieldParts[0]])) {
// If the next field is an integer, return only this item of the array
i = parseInt(fieldParts[1], 10)
if (typeof i === 'number' && !isNaN(i)) {
return getDotValue(obj[fieldParts[0]][i], fieldParts.slice(2))
}
const i = parseInt(fieldParts[1], 10)
if (typeof i === 'number' && !isNaN(i)) return getDotValue(obj[fieldParts[0]][i], fieldParts.slice(2))
// Return the array of values
objs = []
for (i = 0; i < obj[fieldParts[0]].length; i += 1) {
objs.push(getDotValue(obj[fieldParts[0]][i], fieldParts.slice(1)))
}
return objs
} else {
return getDotValue(obj[fieldParts[0]], fieldParts.slice(1))
}
return obj[fieldParts[0]].map(el => getDotValue(el, fieldParts.slice(1)))
} else return getDotValue(obj[fieldParts[0]], fieldParts.slice(1))
}
/**
@ -503,24 +470,33 @@ function getDotValue (obj, field) {
* In the case of object, we check deep equality
* Returns true if they are, false otherwise
*/
function areThingsEqual (a, b) {
let aKeys
let bKeys
let i
const areThingsEqual = (a, b) => {
// Strings, booleans, numbers, null
if (a === null || typeof a === 'string' || typeof a === 'boolean' || typeof a === 'number' ||
b === null || typeof b === 'string' || typeof b === 'boolean' || typeof b === 'number') { return a === b }
if (
a === null ||
typeof a === 'string' ||
typeof a === 'boolean' ||
typeof a === 'number' ||
b === null ||
typeof b === 'string' ||
typeof b === 'boolean' ||
typeof b === 'number'
) return a === b
// Dates
if (util.types.isDate(a) || util.types.isDate(b)) { return util.types.isDate(a) && util.types.isDate(b) && a.getTime() === b.getTime() }
if (util.types.isDate(a) || util.types.isDate(b)) return util.types.isDate(a) && util.types.isDate(b) && a.getTime() === b.getTime()
// Arrays (no match since arrays are used as a $in)
// undefined (no match since they mean field doesn't exist and can't be serialized)
if ((!(Array.isArray(a) && Array.isArray(b)) && (Array.isArray(a) || Array.isArray(b))) || a === undefined || b === undefined) { return false }
if (
(!(Array.isArray(a) && Array.isArray(b)) && (Array.isArray(a) || Array.isArray(b))) ||
a === undefined || b === undefined
) return false
// General objects (check for deep equality)
// a and b should be objects at this point
let aKeys
let bKeys
try {
aKeys = Object.keys(a)
bKeys = Object.keys(b)
@ -528,10 +504,10 @@ function areThingsEqual (a, b) {
return false
}
if (aKeys.length !== bKeys.length) { return false }
for (i = 0; i < aKeys.length; i += 1) {
if (bKeys.indexOf(aKeys[i]) === -1) { return false }
if (!areThingsEqual(a[aKeys[i]], b[aKeys[i]])) { return false }
if (aKeys.length !== bKeys.length) return false
for (const el of aKeys) {
if (bKeys.indexOf(el) === -1) return false
if (!areThingsEqual(a[el], b[el])) return false
}
return true
}
@ -539,13 +515,17 @@ function areThingsEqual (a, b) {
/**
* Check that two values are comparable
*/
function areComparable (a, b) {
if (typeof a !== 'string' && typeof a !== 'number' && !util.types.isDate(a) &&
typeof b !== 'string' && typeof b !== 'number' && !util.types.isDate(b)) {
return false
}
if (typeof a !== typeof b) { return false }
const areComparable = (a, b) => {
if (
typeof a !== 'string' &&
typeof a !== 'number' &&
!util.types.isDate(a) &&
typeof b !== 'string' &&
typeof b !== 'number' &&
!util.types.isDate(b)
) return false
if (typeof a !== typeof b) return false
return true
}
@ -555,88 +535,62 @@ function areComparable (a, b) {
* @param {Native value} a Value in the object
* @param {Native value} b Value in the query
*/
comparisonFunctions.$lt = function (a, b) {
return areComparable(a, b) && a < b
}
comparisonFunctions.$lt = (a, b) => areComparable(a, b) && a < b
comparisonFunctions.$lte = function (a, b) {
return areComparable(a, b) && a <= b
}
comparisonFunctions.$lte = (a, b) => areComparable(a, b) && a <= b
comparisonFunctions.$gt = function (a, b) {
return areComparable(a, b) && a > b
}
comparisonFunctions.$gte = function (a, b) {
return areComparable(a, b) && a >= b
}
comparisonFunctions.$gt = (a, b) => areComparable(a, b) && a > b
comparisonFunctions.$ne = function (a, b) {
if (a === undefined) { return true }
return !areThingsEqual(a, b)
}
comparisonFunctions.$gte = (a, b) => areComparable(a, b) && a >= b
comparisonFunctions.$in = function (a, b) {
let i
comparisonFunctions.$ne = (a, b) => a === undefined || !areThingsEqual(a, b)
if (!Array.isArray(b)) { throw new Error('$in operator called with a non-array') }
comparisonFunctions.$in = (a, b) => {
if (!Array.isArray(b)) throw new Error('$in operator called with a non-array')
for (i = 0; i < b.length; i += 1) {
if (areThingsEqual(a, b[i])) { return true }
for (const el of b) {
if (areThingsEqual(a, el)) return true
}
return false
}
comparisonFunctions.$nin = function (a, b) {
if (!Array.isArray(b)) { throw new Error('$nin operator called with a non-array') }
comparisonFunctions.$nin = (a, b) => {
if (!Array.isArray(b)) throw new Error('$nin operator called with a non-array')
return !comparisonFunctions.$in(a, b)
}
comparisonFunctions.$regex = function (a, b) {
if (!util.types.isRegExp(b)) { throw new Error('$regex operator called with non regular expression') }
comparisonFunctions.$regex = (a, b) => {
if (!util.types.isRegExp(b)) throw new Error('$regex operator called with non regular expression')
if (typeof a !== 'string') {
return false
} else {
return b.test(a)
}
if (typeof a !== 'string') return false
else return b.test(a)
}
comparisonFunctions.$exists = function (value, exists) {
if (exists || exists === '') { // This will be true for all values of stat except false, null, undefined and 0
exists = true // That's strange behaviour (we should only use true/false) but that's the way Mongo does it...
} else {
exists = false
}
comparisonFunctions.$exists = (value, exists) => {
// This will be true for all values of stat except false, null, undefined and 0
// That's strange behaviour (we should only use true/false) but that's the way Mongo does it...
if (exists || exists === '') exists = true
else exists = false
if (value === undefined) {
return !exists
} else {
return exists
}
if (value === undefined) return !exists
else return exists
}
// Specific to arrays
comparisonFunctions.$size = function (obj, value) {
if (!Array.isArray(obj)) { return false }
if (value % 1 !== 0) { throw new Error('$size operator called without an integer') }
comparisonFunctions.$size = (obj, value) => {
if (!Array.isArray(obj)) return false
if (value % 1 !== 0) throw new Error('$size operator called without an integer')
return obj.length === value
}
comparisonFunctions.$elemMatch = function (obj, value) {
if (!Array.isArray(obj)) { return false }
let i = obj.length
let result = false // Initialize result
while (i--) {
if (match(obj[i], value)) { // If match for array element, return true
result = true
break
}
}
return result
comparisonFunctions.$elemMatch = (obj, value) => {
if (!Array.isArray(obj)) return false
return obj.some(el => match(el, value))
}
arrayComparisonFunctions.$size = true
arrayComparisonFunctions.$elemMatch = true
@ -645,13 +599,11 @@ arrayComparisonFunctions.$elemMatch = true
* @param {Model} obj
* @param {Array of Queries} query
*/
logicalOperators.$or = function (obj, query) {
let i
if (!Array.isArray(query)) { throw new Error('$or operator used without an array') }
logicalOperators.$or = (obj, query) => {
if (!Array.isArray(query)) throw new Error('$or operator used without an array')
for (i = 0; i < query.length; i += 1) {
if (match(obj, query[i])) { return true }
for (let i = 0; i < query.length; i += 1) {
if (match(obj, query[i])) return true
}
return false
@ -662,13 +614,11 @@ logicalOperators.$or = function (obj, query) {
* @param {Model} obj
* @param {Array of Queries} query
*/
logicalOperators.$and = function (obj, query) {
let i
logicalOperators.$and = (obj, query) => {
if (!Array.isArray(query)) throw new Error('$and operator used without an array')
if (!Array.isArray(query)) { throw new Error('$and operator used without an array') }
for (i = 0; i < query.length; i += 1) {
if (!match(obj, query[i])) { return false }
for (let i = 0; i < query.length; i += 1) {
if (!match(obj, query[i])) return false
}
return true
@ -679,20 +629,18 @@ logicalOperators.$and = function (obj, query) {
* @param {Model} obj
* @param {Query} query
*/
logicalOperators.$not = function (obj, query) {
return !match(obj, query)
}
logicalOperators.$not = (obj, query) => !match(obj, query)
/**
* Use a function to match
* @param {Model} obj
* @param {Query} query
*/
logicalOperators.$where = function (obj, fn) {
if (typeof fn !== 'function') { throw new Error('$where operator used without a function') }
logicalOperators.$where = (obj, fn) => {
if (typeof fn !== 'function') throw new Error('$where operator used without a function')
const result = fn.call(obj)
if (typeof result !== 'boolean') { throw new Error('$where function must return boolean') }
if (typeof result !== 'boolean') throw new Error('$where function must return boolean')
return result
}
@ -702,29 +650,20 @@ logicalOperators.$where = function (obj, fn) {
* @param {Object} obj Document to check
* @param {Object} query
*/
function match (obj, query) {
let queryKey
let queryValue
let i
const match = (obj, query) => {
// Primitive query against a primitive type
// This is a bit of a hack since we construct an object with an arbitrary key only to dereference it later
// But I don't have time for a cleaner implementation now
if (isPrimitiveType(obj) || isPrimitiveType(query)) {
return matchQueryPart({ needAKey: obj }, 'needAKey', query)
}
if (isPrimitiveType(obj) || isPrimitiveType(query)) return matchQueryPart({ needAKey: obj }, 'needAKey', query)
// Normal query
const queryKeys = Object.keys(query)
for (i = 0; i < queryKeys.length; i += 1) {
queryKey = queryKeys[i]
queryValue = query[queryKey]
for (const queryKey in query) {
if (Object.prototype.hasOwnProperty.call(query, queryKey)) {
const queryValue = query[queryKey]
if (queryKey[0] === '$') {
if (!logicalOperators[queryKey]) { throw new Error('Unknown logical operator ' + queryKey) }
if (!logicalOperators[queryKey](obj, queryValue)) { return false }
} else {
if (!matchQueryPart(obj, queryKey, queryValue)) { return false }
if (!logicalOperators[queryKey]) throw new Error(`Unknown logical operator ${queryKey}`)
if (!logicalOperators[queryKey](obj, queryValue)) return false
} else if (!matchQueryPart(obj, queryKey, queryValue)) return false
}
}
@ -737,29 +676,22 @@ function match (obj, query) {
*/
function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) {
const objValue = getDotValue(obj, queryKey)
let i
let keys
let firstChars
let dollarFirstChars
// Check if the value is an array if we don't force a treatment as value
if (Array.isArray(objValue) && !treatObjAsValue) {
// If the queryValue is an array, try to perform an exact match
if (Array.isArray(queryValue)) {
return matchQueryPart(obj, queryKey, queryValue, true)
}
if (Array.isArray(queryValue)) return matchQueryPart(obj, queryKey, queryValue, true)
// Check if we are using an array-specific comparison function
if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue)) {
keys = Object.keys(queryValue)
for (i = 0; i < keys.length; i += 1) {
if (arrayComparisonFunctions[keys[i]]) { return matchQueryPart(obj, queryKey, queryValue, true) }
for (const key in queryValue) {
if (Object.prototype.hasOwnProperty.call(queryValue, key) && arrayComparisonFunctions[key]) { return matchQueryPart(obj, queryKey, queryValue, true) }
}
}
// If not, treat it as an array of { obj, query } where there needs to be at least one match
for (i = 0; i < objValue.length; i += 1) {
if (matchQueryPart({ k: objValue[i] }, 'k', queryValue)) { return true } // k here could be any string
for (const el of objValue) {
if (matchQueryPart({ k: el }, 'k', queryValue)) return true // k here could be any string
}
return false
}
@ -767,33 +699,29 @@ function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) {
// queryValue is an actual object. Determine whether it contains comparison operators
// or only normal fields. Mixed objects are not allowed
if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue) && !Array.isArray(queryValue)) {
keys = Object.keys(queryValue)
firstChars = keys.map(item => item[0])
dollarFirstChars = firstChars.filter(c => c === '$')
const keys = Object.keys(queryValue)
const firstChars = keys.map(item => item[0])
const dollarFirstChars = firstChars.filter(c => c === '$')
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) {
throw new Error('You cannot mix operators and normal fields')
}
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) throw new Error('You cannot mix operators and normal fields')
// queryValue is an object of this form: { $comparisonOperator1: value1, ... }
if (dollarFirstChars.length > 0) {
for (i = 0; i < keys.length; i += 1) {
if (!comparisonFunctions[keys[i]]) { throw new Error('Unknown comparison function ' + keys[i]) }
for (const key of keys) {
if (!comparisonFunctions[key]) throw new Error(`Unknown comparison function ${key}`)
if (!comparisonFunctions[keys[i]](objValue, queryValue[keys[i]])) { return false }
if (!comparisonFunctions[key](objValue, queryValue[key])) return false
}
return true
}
}
// Using regular expressions with basic querying
if (util.types.isRegExp(queryValue)) { return comparisonFunctions.$regex(objValue, queryValue) }
if (util.types.isRegExp(queryValue)) return comparisonFunctions.$regex(objValue, queryValue)
// queryValue is either a native value or a normal object
// Basic matching is possible
if (!areThingsEqual(objValue, queryValue)) { return false }
return true
return areThingsEqual(objValue, queryValue)
}
// Interface

@ -19,31 +19,33 @@ class Persistence {
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/
constructor (options) {
let i
let j
let randomString
this.db = options.db
this.inMemoryOnly = this.db.inMemoryOnly
this.filename = this.db.filename
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
}
if (
!this.inMemoryOnly &&
this.filename &&
this.filename.charAt(this.filename.length - 1) === '~'
) throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
}
this.afterSerialization = options.afterSerialization || function (s) { return s }
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s }
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i)
if (
options.afterSerialization &&
!options.beforeDeserialization
) throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
if (
!options.afterSerialization &&
options.beforeDeserialization
) throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
this.afterSerialization = options.afterSerialization || (s => s)
this.beforeDeserialization = options.beforeDeserialization || (s => s)
for (let i = 1; i < 30; i += 1) {
for (let j = 0; j < 10; j += 1) {
const randomString = customUtils.uid(i)
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss')
}
@ -67,33 +69,31 @@ class Persistence {
* Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection
* while the data file is append-only so it may grow larger
* @param {Function} cb Optional callback, signature: err
* @param {Function} callback Optional callback, signature: err
*/
persistCachedDatabase (cb) {
const callback = cb || function () {}
persistCachedDatabase (callback = () => {}) {
let toPersist = ''
const self = this
if (this.inMemoryOnly) { return callback(null) }
if (this.inMemoryOnly) return callback(null)
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
this.db.getAllData().forEach(doc => {
toPersist += this.afterSerialization(model.serialize(doc)) + '\n'
})
Object.keys(this.db.indexes).forEach(function (fieldName) {
Object.keys(this.db.indexes).forEach(fieldName => {
if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({
toPersist += this.afterSerialization(model.serialize({
$$indexCreated: {
fieldName: fieldName,
unique: self.db.indexes[fieldName].unique,
sparse: self.db.indexes[fieldName].sparse
unique: this.db.indexes[fieldName].unique,
sparse: this.db.indexes[fieldName].sparse
}
})) + '\n'
}
})
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err) }
self.db.emit('compaction.done')
storage.crashSafeWriteFile(this.filename, toPersist, err => {
if (err) return callback(err)
this.db.emit('compaction.done')
return callback(null)
})
}
@ -110,14 +110,13 @@ class Persistence {
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/
setAutocompactionInterval (interval) {
const self = this
const minInterval = 5000
const realInterval = Math.max(interval || 0, minInterval)
this.stopAutocompaction()
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile()
this.autocompactionIntervalId = setInterval(() => {
this.compactDatafile()
}, realInterval)
}
@ -125,32 +124,28 @@ class Persistence {
* Stop autocompaction (do nothing if autocompaction was not running)
*/
stopAutocompaction () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId) }
if (this.autocompactionIntervalId) clearInterval(this.autocompactionIntervalId)
}
/**
* Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err
* @param {Function} callback Optional, signature: err
*/
persistNewState (newDocs, cb) {
const self = this
persistNewState (newDocs, callback = () => {}) {
let toPersist = ''
const callback = cb || function () {}
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null) }
if (this.inMemoryOnly) return callback(null)
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n'
newDocs.forEach(doc => {
toPersist += this.afterSerialization(model.serialize(doc)) + '\n'
})
if (toPersist.length === 0) { return callback(null) }
if (toPersist.length === 0) return callback(null)
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err)
})
storage.appendFile(this.filename, toPersist, 'utf8', err => callback(err))
}
/**
@ -161,39 +156,29 @@ class Persistence {
const data = rawData.split('\n')
const dataById = {}
const tdata = []
let i
const indexes = {}
let corruptItems = -1
for (i = 0; i < data.length; i += 1) {
let doc
for (const datum of data) {
try {
doc = model.deserialize(this.beforeDeserialization(data[i]))
const doc = model.deserialize(this.beforeDeserialization(datum))
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id]
} else {
dataById[doc._id] = doc
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
} else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved]
}
if (doc.$$deleted === true) delete dataById[doc._id]
else dataById[doc._id] = doc
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
else if (typeof doc.$$indexRemoved === 'string') delete indexes[doc.$$indexRemoved]
} catch (e) {
corruptItems += 1
}
}
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error('More than ' + Math.floor(100 * this.corruptAlertThreshold) + '% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss')
}
if (
data.length > 0 &&
corruptItems / data.length > this.corruptAlertThreshold
) throw new Error(`More than ${Math.floor(100 * this.corruptAlertThreshold)}% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss`)
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k])
})
tdata.push(...Object.values(dataById))
return { data: tdata, indexes: indexes }
}
@ -206,56 +191,53 @@ class Persistence {
* This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err
* @param {Function} callback Optional callback, signature: err
*/
loadDatabase (cb) {
const callback = cb || function () {}
const self = this
self.db.resetIndexes()
loadDatabase (callback = () => {}) {
this.db.resetIndexes()
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null) }
if (this.inMemoryOnly) return callback(null)
async.waterfall([
function (cb) {
cb => {
// eslint-disable-next-line node/handle-callback-err
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
Persistence.ensureDirectoryExists(path.dirname(this.filename), err => {
// TODO: handle error
// eslint-disable-next-line node/handle-callback-err
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.ensureDatafileIntegrity(this.filename, err => {
// TODO: handle error
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err) }
storage.readFile(this.filename, 'utf8', (err, rawData) => {
if (err) return cb(err)
let treatedData
try {
treatedData = self.treatRawData(rawData)
treatedData = this.treatRawData(rawData)
} catch (e) {
return cb(e)
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key])
Object.keys(treatedData.indexes).forEach(key => {
this.db.indexes[key] = new Index(treatedData.indexes[key])
})
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data)
this.db.resetIndexes(treatedData.data)
} catch (e) {
self.db.resetIndexes() // Rollback any index which didn't fail
this.db.resetIndexes() // Rollback any index which didn't fail
return cb(e)
}
self.db.persistence.persistCachedDatabase(cb)
this.db.persistence.persistCachedDatabase(cb)
})
})
})
}
], function (err) {
if (err) { return callback(err) }
], err => {
if (err) return callback(err)
self.db.executor.processBuffer()
this.db.executor.processBuffer()
return callback(null)
})
}
@ -264,9 +246,7 @@ class Persistence {
* Check if a directory stat and create it on the fly if it is not the case
* cb is optional, signature: err
*/
static ensureDirectoryExists (dir, cb) {
const callback = cb || function () {}
static ensureDirectoryExists (dir, callback = () => {}) {
storage.mkdir(dir, { recursive: true }, err => { callback(err) })
}
@ -277,26 +257,19 @@ class Persistence {
static getNWAppFilename (appName, relativeFilename) {
let home
switch (process.platform) {
case 'win32':
case 'win64':
if (process.platform === 'win32' || process.platform === 'win64') {
home = process.env.LOCALAPPDATA || process.env.APPDATA
if (!home) { throw new Error('Couldn\'t find the base application data folder') }
if (!home) throw new Error('Couldn\'t find the base application data folder')
home = path.join(home, appName)
break
case 'darwin':
} else if (process.platform === 'darwin') {
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
if (!home) throw new Error('Couldn\'t find the base application data directory')
home = path.join(home, 'Library', 'Application Support', appName)
break
case 'linux':
} else if (process.platform === 'linux') {
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
if (!home) throw new Error('Couldn\'t find the base application data directory')
home = path.join(home, '.config', appName)
break
default:
throw new Error('Can\'t use the Node Webkit relative path for platform ' + process.platform)
}
} else throw new Error(`Can't use the Node Webkit relative path for platform ${process.platform}`)
return path.join(home, 'nedb-data', relativeFilename)
}

@ -23,11 +23,11 @@ storage.mkdir = fs.mkdir
/**
* Explicit name ...
*/
storage.ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null) }
storage.ensureFileDoesntExist = (file, callback) => {
storage.exists(file, exists => {
if (!exists) return callback(null)
storage.unlink(file, function (err) { return callback(err) })
storage.unlink(file, err => callback(err))
})
}
@ -37,7 +37,7 @@ storage.ensureFileDoesntExist = function (file, callback) {
* @param {Boolean} options.isDir Optional, defaults to false
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/
storage.flushToStorage = function (options, callback) {
storage.flushToStorage = (options, callback) => {
let filename
let flags
if (typeof options === 'string') {
@ -50,12 +50,12 @@ storage.flushToStorage = function (options, callback) {
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null) }
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) return callback(null)
fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err) }
fs.fsync(fd, function (errFS) {
fs.close(fd, function (errC) {
fs.open(filename, flags, (err, fd) => {
if (err) return callback(err)
fs.fsync(fd, errFS => {
fs.close(fd, errC => {
if (errFS || errC) {
const e = new Error('Failed to flush to storage')
e.errorOnFsync = errFS
@ -73,32 +73,28 @@ storage.flushToStorage = function (options, callback) {
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
* @param {String} filename
* @param {String} data
* @param {Function} cb Optional callback, signature: err
* @param {Function} callback Optional callback, signature: err
*/
storage.crashSafeWriteFile = function (filename, data, cb) {
const callback = cb || function () {}
storage.crashSafeWriteFile = (filename, data, callback = () => {}) => {
const tempFilename = filename + '~'
async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }),
function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err) })
} else {
return cb()
}
cb => {
storage.exists(filename, exists => {
if (exists) storage.flushToStorage(filename, err => cb(err))
else return cb()
})
},
function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err) })
cb => {
storage.writeFile(tempFilename, data, err => cb(err))
},
async.apply(storage.flushToStorage, tempFilename),
function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err) })
cb => {
storage.rename(tempFilename, filename, err => cb(err))
},
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err) })
], err => callback(err))
}
/**
@ -106,21 +102,19 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {String} filename
* @param {Function} callback signature: err
*/
storage.ensureDatafileIntegrity = function (filename, callback) {
storage.ensureDatafileIntegrity = (filename, callback) => {
const tempFilename = filename + '~'
storage.exists(filename, function (filenameExists) {
storage.exists(filename, filenameExists => {
// Write was successful
if (filenameExists) { return callback(null) }
if (filenameExists) return callback(null)
storage.exists(tempFilename, function (oldFilenameExists) {
storage.exists(tempFilename, oldFilenameExists => {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err) })
}
if (!oldFilenameExists) return storage.writeFile(filename, '', 'utf8', err => { callback(err) })
// Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err) })
storage.rename(tempFilename, filename, err => callback(err))
})
})
}

Loading…
Cancel
Save