cleanup code

pull/2/head
Timothée Rebours 4 years ago
parent 5c6561307d
commit e8cba5273a
  1. 41
      browser-version/lib/customUtils.js
  2. 22
      browser-version/lib/storage.js
  3. 88
      lib/cursor.js
  4. 10
      lib/customUtils.js
  5. 302
      lib/datastore.js
  6. 23
      lib/executor.js
  7. 92
      lib/indexes.js
  8. 580
      lib/model.js
  9. 197
      lib/persistence.js
  10. 60
      lib/storage.js

@ -7,7 +7,7 @@
* https://github.com/dominictarr/crypto-browserify * https://github.com/dominictarr/crypto-browserify
* NOTE: Math.random() does not guarantee "cryptographic quality" but we actually don't need it * NOTE: Math.random() does not guarantee "cryptographic quality" but we actually don't need it
*/ */
function randomBytes (size) { const randomBytes = size => {
const bytes = new Array(size) const bytes = new Array(size)
for (let i = 0, r; i < size; i++) { for (let i = 0, r; i < size; i++) {
@ -22,39 +22,32 @@ function randomBytes (size) {
* Taken from the base64-js module * Taken from the base64-js module
* https://github.com/beatgammit/base64-js/ * https://github.com/beatgammit/base64-js/
*/ */
function byteArrayToBase64 (uint8) { const byteArrayToBase64 = uint8 => {
const lookup = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' const lookup = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
const extraBytes = uint8.length % 3 // if we have 1 byte left, pad 2 bytes const extraBytes = uint8.length % 3 // if we have 1 byte left, pad 2 bytes
let output = '' let output = ''
let temp let temp
let length
let i
function tripletToBase64 (num) { const tripletToBase64 = num => lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F]
return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F]
}
// go through the array every three bytes, we'll deal with trailing stuff later // go through the array every three bytes, we'll deal with trailing stuff later
for (i = 0, length = uint8.length - extraBytes; i < length; i += 3) { for (let i = 0, length = uint8.length - extraBytes; i < length; i += 3) {
temp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]) temp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2])
output += tripletToBase64(temp) output += tripletToBase64(temp)
} }
// pad the end with zeros, but make sure to not forget the extra bytes // pad the end with zeros, but make sure to not forget the extra bytes
switch (extraBytes) { if (extraBytes === 1) {
case 1: temp = uint8[uint8.length - 1]
temp = uint8[uint8.length - 1] output += lookup[temp >> 2]
output += lookup[temp >> 2] output += lookup[(temp << 4) & 0x3F]
output += lookup[(temp << 4) & 0x3F] output += '=='
output += '==' } else if (extraBytes === 2) {
break temp = (uint8[uint8.length - 2] << 8) + (uint8[uint8.length - 1])
case 2: output += lookup[temp >> 10]
temp = (uint8[uint8.length - 2] << 8) + (uint8[uint8.length - 1]) output += lookup[(temp >> 4) & 0x3F]
output += lookup[temp >> 10] output += lookup[(temp << 2) & 0x3F]
output += lookup[(temp >> 4) & 0x3F] output += '='
output += lookup[(temp << 2) & 0x3F]
output += '='
break
} }
return output return output
@ -68,8 +61,6 @@ function byteArrayToBase64 (uint8) {
* The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision) * The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision)
* See http://en.wikipedia.org/wiki/Birthday_problem * See http://en.wikipedia.org/wiki/Birthday_problem
*/ */
function uid (len) { const uid = len => byteArrayToBase64(randomBytes(Math.ceil(Math.max(8, len * 2)))).replace(/[+/]/g, '').slice(0, len)
return byteArrayToBase64(randomBytes(Math.ceil(Math.max(8, len * 2)))).replace(/[+/]/g, '').slice(0, len)
}
module.exports.uid = uid module.exports.uid = uid

@ -13,7 +13,7 @@ const store = localforage.createInstance({
storeName: 'nedbdata' storeName: 'nedbdata'
}) })
function exists (filename, cback) { const exists = (filename, cback) => {
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, value) => { store.getItem(filename, (err, value) => {
if (value !== null) return cback(true) // Even if value is undefined, localforage returns null if (value !== null) return cback(true) // Even if value is undefined, localforage returns null
@ -21,7 +21,7 @@ function exists (filename, cback) {
}) })
} }
function rename (filename, newFilename, callback) { const rename = (filename, newFilename, callback) => {
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, value) => { store.getItem(filename, (err, value) => {
if (value === null) store.removeItem(newFilename, () => callback()) if (value === null) store.removeItem(newFilename, () => callback())
@ -33,13 +33,13 @@ function rename (filename, newFilename, callback) {
}) })
} }
function writeFile (filename, contents, options, callback) { const writeFile = (filename, contents, options, callback) => {
// Options do not matter in browser setup // Options do not matter in browser setup
if (typeof options === 'function') { callback = options } if (typeof options === 'function') { callback = options }
store.setItem(filename, contents, () => callback()) store.setItem(filename, contents, () => callback())
} }
function appendFile (filename, toAppend, options, callback) { const appendFile = (filename, toAppend, options, callback) => {
// Options do not matter in browser setup // Options do not matter in browser setup
if (typeof options === 'function') { callback = options } if (typeof options === 'function') { callback = options }
@ -51,26 +51,22 @@ function appendFile (filename, toAppend, options, callback) {
}) })
} }
function readFile (filename, options, callback) { const readFile = (filename, options, callback) => {
// Options do not matter in browser setup // Options do not matter in browser setup
if (typeof options === 'function') { callback = options } if (typeof options === 'function') { callback = options }
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
store.getItem(filename, (err, contents) => callback(null, contents || '')) store.getItem(filename, (err, contents) => callback(null, contents || ''))
} }
function unlink (filename, callback) { const unlink = (filename, callback) => {
store.removeItem(filename, () => callback()) store.removeItem(filename, () => callback())
} }
// Nothing to do, no directories will be used on the browser // Nothing to do, no directories will be used on the browser
function mkdir (dir, options, callback) { const mkdir = (dir, options, callback) => callback()
return callback()
}
// Nothing to do, no data corruption possible in the brower // Nothing to do, no data corruption possible in the browser
function ensureDatafileIntegrity (filename, callback) { const ensureDatafileIntegrity = (filename, callback) => callback(null)
return callback(null)
}
// Interface // Interface
module.exports.exists = exists module.exports.exists = exists

@ -56,7 +56,6 @@ class Cursor {
*/ */
project (candidates) { project (candidates) {
const res = [] const res = []
const self = this
let action let action
if (this._projection === undefined || Object.keys(this._projection).length === 0) { if (this._projection === undefined || Object.keys(this._projection).length === 0) {
@ -69,31 +68,28 @@ class Cursor {
// Check for consistency // Check for consistency
const keys = Object.keys(this._projection) const keys = Object.keys(this._projection)
keys.forEach(function (k) { keys.forEach(k => {
if (action !== undefined && self._projection[k] !== action) { throw new Error('Can\'t both keep and omit fields except for _id') } if (action !== undefined && this._projection[k] !== action) throw new Error('Can\'t both keep and omit fields except for _id')
action = self._projection[k] action = this._projection[k]
}) })
// Do the actual projection // Do the actual projection
candidates.forEach(function (candidate) { candidates.forEach(candidate => {
let toPush let toPush
if (action === 1) { // pick-type projection if (action === 1) { // pick-type projection
toPush = { $set: {} } toPush = { $set: {} }
keys.forEach(function (k) { keys.forEach(k => {
toPush.$set[k] = model.getDotValue(candidate, k) toPush.$set[k] = model.getDotValue(candidate, k)
if (toPush.$set[k] === undefined) { delete toPush.$set[k] } if (toPush.$set[k] === undefined) delete toPush.$set[k]
}) })
toPush = model.modify({}, toPush) toPush = model.modify({}, toPush)
} else { // omit-type projection } else { // omit-type projection
toPush = { $unset: {} } toPush = { $unset: {} }
keys.forEach(function (k) { toPush.$unset[k] = true }) keys.forEach(k => { toPush.$unset[k] = true })
toPush = model.modify(candidate, toPush) toPush = model.modify(candidate, toPush)
} }
if (keepId) { if (keepId) toPush._id = candidate._id
toPush._id = candidate._id else delete toPush._id
} else {
delete toPush._id
}
res.push(toPush) res.push(toPush)
}) })
@ -111,38 +107,30 @@ class Cursor {
let res = [] let res = []
let added = 0 let added = 0
let skipped = 0 let skipped = 0
const self = this
let error = null let error = null
let i
let keys let keys
let key let key
function callback (error, res) { const callback = (error, res) => {
if (self.execFn) { if (this.execFn) return this.execFn(error, res, _callback)
return self.execFn(error, res, _callback) else return _callback(error, res)
} else {
return _callback(error, res)
}
} }
this.db.getCandidates(this.query, function (err, candidates) { this.db.getCandidates(this.query, (err, candidates) => {
if (err) { return callback(err) } if (err) return callback(err)
try { try {
for (i = 0; i < candidates.length; i += 1) { for (const candidate of candidates) {
if (model.match(candidates[i], self.query)) { if (model.match(candidate, this.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip // If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) { if (!this._sort) {
if (self._skip && self._skip > skipped) { if (this._skip && this._skip > skipped) skipped += 1
skipped += 1 else {
} else { res.push(candidate)
res.push(candidates[i])
added += 1 added += 1
if (self._limit && self._limit <= added) { break } if (this._limit && this._limit <= added) break
} }
} else { } else res.push(candidate)
res.push(candidates[i])
}
} }
} }
} catch (err) { } catch (err) {
@ -150,39 +138,33 @@ class Cursor {
} }
// Apply all sorts // Apply all sorts
if (self._sort) { if (this._sort) {
keys = Object.keys(self._sort) keys = Object.keys(this._sort)
// Sorting // Sorting
const criteria = [] const criteria = []
for (i = 0; i < keys.length; i++) { keys.forEach(item => {
key = keys[i] key = item
criteria.push({ key: key, direction: self._sort[key] }) criteria.push({ key: key, direction: this._sort[key] })
} })
res.sort(function (a, b) { res.sort((a, b) => {
let criterion for (const criterion of criteria) {
let compare const compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), this.db.compareStrings)
let i if (compare !== 0) return compare
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i]
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion.key), self.db.compareStrings)
if (compare !== 0) {
return compare
}
} }
return 0 return 0
}) })
// Applying limit and skip // Applying limit and skip
const limit = self._limit || res.length const limit = this._limit || res.length
const skip = self._skip || 0 const skip = this._skip || 0
res = res.slice(skip, skip + limit) res = res.slice(skip, skip + limit)
} }
// Apply projection // Apply projection
try { try {
res = self.project(res) res = this.project(res)
} catch (e) { } catch (e) {
error = e error = e
res = undefined res = undefined

@ -8,12 +8,10 @@ const crypto = require('crypto')
* The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision) * The probability of a collision is extremely small (need 3*10^12 documents to have one chance in a million of a collision)
* See http://en.wikipedia.org/wiki/Birthday_problem * See http://en.wikipedia.org/wiki/Birthday_problem
*/ */
function uid (len) { const uid = len => crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2))) .toString('base64')
.toString('base64') .replace(/[+/]/g, '')
.replace(/[+/]/g, '') .slice(0, len)
.slice(0, len)
}
// Interface // Interface
module.exports.uid = uid module.exports.uid = uid

@ -64,7 +64,7 @@ class Datastore extends EventEmitter {
// This new executor is ready if we don't use persistence // This new executor is ready if we don't use persistence
// If we do, it will only be ready once loadDatabase is called // If we do, it will only be ready once loadDatabase is called
this.executor = new Executor() this.executor = new Executor()
if (this.inMemoryOnly) { this.executor.ready = true } if (this.inMemoryOnly) this.executor.ready = true
// Indexed by field name, dot notation can be used // Indexed by field name, dot notation can be used
// _id is always indexed and since _ids are generated randomly the underlying // _id is always indexed and since _ids are generated randomly the underlying
@ -76,9 +76,9 @@ class Datastore extends EventEmitter {
// Queue a load of the database right away and call the onload handler // Queue a load of the database right away and call the onload handler
// By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception // By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
if (this.autoload) { if (this.autoload) {
this.loadDatabase(options.onload || function (err) { this.loadDatabase(options.onload || (err => {
if (err) { throw err } if (err) throw err
}) }))
} }
} }
@ -100,38 +100,32 @@ class Datastore extends EventEmitter {
* Reset all currently defined indexes * Reset all currently defined indexes
*/ */
resetIndexes (newData) { resetIndexes (newData) {
const self = this for (const index of Object.values(this.indexes)) {
index.reset(newData)
Object.keys(this.indexes).forEach(function (i) { }
self.indexes[i].reset(newData)
})
} }
/** /**
* Ensure an index is kept for this field. Same parameters as lib/indexes * Ensure an index is kept for this field. Same parameters as lib/indexes
* For now this function is synchronous, we need to test how much time it takes * For now this function is synchronous, we need to test how much time it takes
* We use an async API for consistency with the rest of the code * We use an async API for consistency with the rest of the code
* @param {Object} options
* @param {String} options.fieldName * @param {String} options.fieldName
* @param {Boolean} options.unique * @param {Boolean} options.unique
* @param {Boolean} options.sparse * @param {Boolean} options.sparse
* @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index (only works on Date fields, not arrays of Date) * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index (only works on Date fields, not arrays of Date)
* @param {Function} cb Optional callback, signature: err * @param {Function} callback Optional callback, signature: err
*/ */
ensureIndex (options, cb) { ensureIndex (options = {}, callback = () => {}) {
let err
const callback = cb || function () {}
options = options || {}
if (!options.fieldName) { if (!options.fieldName) {
err = new Error('Cannot create an index without a fieldName') const err = new Error('Cannot create an index without a fieldName')
err.missingFieldName = true err.missingFieldName = true
return callback(err) return callback(err)
} }
if (this.indexes[options.fieldName]) { return callback(null) } if (this.indexes[options.fieldName]) return callback(null)
this.indexes[options.fieldName] = new Index(options) this.indexes[options.fieldName] = new Index(options)
if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds } // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here if (options.expireAfterSeconds !== undefined) this.ttlIndexes[options.fieldName] = options.expireAfterSeconds // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
try { try {
this.indexes[options.fieldName].insert(this.getAllData()) this.indexes[options.fieldName].insert(this.getAllData())
@ -141,8 +135,8 @@ class Datastore extends EventEmitter {
} }
// We may want to force all options to be persisted including defaults, not just the ones passed the index creation function // We may want to force all options to be persisted including defaults, not just the ones passed the index creation function
this.persistence.persistNewState([{ $$indexCreated: options }], function (err) { this.persistence.persistNewState([{ $$indexCreated: options }], err => {
if (err) { return callback(err) } if (err) return callback(err)
return callback(null) return callback(null)
}) })
} }
@ -150,15 +144,13 @@ class Datastore extends EventEmitter {
/** /**
* Remove an index * Remove an index
* @param {String} fieldName * @param {String} fieldName
* @param {Function} cb Optional callback, signature: err * @param {Function} callback Optional callback, signature: err
*/ */
removeIndex (fieldName, cb) { removeIndex (fieldName, callback = () => {}) {
const callback = cb || function () {}
delete this.indexes[fieldName] delete this.indexes[fieldName]
this.persistence.persistNewState([{ $$indexRemoved: fieldName }], function (err) { this.persistence.persistNewState([{ $$indexRemoved: fieldName }], err => {
if (err) { return callback(err) } if (err) return callback(err)
return callback(null) return callback(null)
}) })
} }
@ -167,12 +159,11 @@ class Datastore extends EventEmitter {
* Add one or several document(s) to all indexes * Add one or several document(s) to all indexes
*/ */
addToIndexes (doc) { addToIndexes (doc) {
let i
let failingIndex let failingIndex
let error let error
const keys = Object.keys(this.indexes) const keys = Object.keys(this.indexes)
for (i = 0; i < keys.length; i += 1) { for (let i = 0; i < keys.length; i += 1) {
try { try {
this.indexes[keys[i]].insert(doc) this.indexes[keys[i]].insert(doc)
} catch (e) { } catch (e) {
@ -184,7 +175,7 @@ class Datastore extends EventEmitter {
// If an error happened, we need to rollback the insert on all other indexes // If an error happened, we need to rollback the insert on all other indexes
if (error) { if (error) {
for (i = 0; i < failingIndex; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].remove(doc) this.indexes[keys[i]].remove(doc)
} }
@ -196,11 +187,9 @@ class Datastore extends EventEmitter {
* Remove one or several document(s) from all indexes * Remove one or several document(s) from all indexes
*/ */
removeFromIndexes (doc) { removeFromIndexes (doc) {
const self = this for (const index of Object.values(this.indexes)) {
index.remove(doc)
Object.keys(this.indexes).forEach(function (i) { }
self.indexes[i].remove(doc)
})
} }
/** /**
@ -209,12 +198,11 @@ class Datastore extends EventEmitter {
* If one update violates a constraint, all changes are rolled back * If one update violates a constraint, all changes are rolled back
*/ */
updateIndexes (oldDoc, newDoc) { updateIndexes (oldDoc, newDoc) {
let i
let failingIndex let failingIndex
let error let error
const keys = Object.keys(this.indexes) const keys = Object.keys(this.indexes)
for (i = 0; i < keys.length; i += 1) { for (let i = 0; i < keys.length; i += 1) {
try { try {
this.indexes[keys[i]].update(oldDoc, newDoc) this.indexes[keys[i]].update(oldDoc, newDoc)
} catch (e) { } catch (e) {
@ -226,7 +214,7 @@ class Datastore extends EventEmitter {
// If an error happened, we need to rollback the update on all other indexes // If an error happened, we need to rollback the update on all other indexes
if (error) { if (error) {
for (i = 0; i < failingIndex; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].revertUpdate(oldDoc, newDoc) this.indexes[keys[i]].revertUpdate(oldDoc, newDoc)
} }
@ -249,7 +237,6 @@ class Datastore extends EventEmitter {
*/ */
getCandidates (query, dontExpireStaleDocs, callback) { getCandidates (query, dontExpireStaleDocs, callback) {
const indexNames = Object.keys(this.indexes) const indexNames = Object.keys(this.indexes)
const self = this
let usableQueryKeys let usableQueryKeys
if (typeof dontExpireStaleDocs === 'function') { if (typeof dontExpireStaleDocs === 'function') {
@ -259,71 +246,72 @@ class Datastore extends EventEmitter {
async.waterfall([ async.waterfall([
// STEP 1: get candidates list by checking indexes from most to least frequent usecase // STEP 1: get candidates list by checking indexes from most to least frequent usecase
function (cb) { cb => {
// For a basic match // For a basic match
usableQueryKeys = [] usableQueryKeys = []
Object.keys(query).forEach(function (k) { Object.keys(query).forEach(k => {
if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.types.isDate(query[k]) || query[k] === null) { if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.types.isDate(query[k]) || query[k] === null) {
usableQueryKeys.push(k) usableQueryKeys.push(k)
} }
}) })
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k)) usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) { if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]])) return cb(null, this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]))
} }
// For a $in match // For a $in match
usableQueryKeys = [] usableQueryKeys = []
Object.keys(query).forEach(function (k) { Object.keys(query).forEach(k => {
if (query[k] && Object.prototype.hasOwnProperty.call(query[k], '$in')) { if (query[k] && Object.prototype.hasOwnProperty.call(query[k], '$in')) {
usableQueryKeys.push(k) usableQueryKeys.push(k)
} }
}) })
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k)) usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) { if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in)) return cb(null, this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in))
} }
// For a comparison match // For a comparison match
usableQueryKeys = [] usableQueryKeys = []
Object.keys(query).forEach(function (k) { Object.keys(query).forEach(k => {
if (query[k] && (Object.prototype.hasOwnProperty.call(query[k], '$lt') || Object.prototype.hasOwnProperty.call(query[k], '$lte') || Object.prototype.hasOwnProperty.call(query[k], '$gt') || Object.prototype.hasOwnProperty.call(query[k], '$gte'))) { if (query[k] && (Object.prototype.hasOwnProperty.call(query[k], '$lt') || Object.prototype.hasOwnProperty.call(query[k], '$lte') || Object.prototype.hasOwnProperty.call(query[k], '$gt') || Object.prototype.hasOwnProperty.call(query[k], '$gte'))) {
usableQueryKeys.push(k) usableQueryKeys.push(k)
} }
}) })
usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k)) usableQueryKeys = usableQueryKeys.filter(k => indexNames.includes(k))
if (usableQueryKeys.length > 0) { if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]])) return cb(null, this.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]))
} }
// By default, return all the DB data // By default, return all the DB data
return cb(null, self.getAllData()) return cb(null, this.getAllData())
}, },
// STEP 2: remove all expired documents // STEP 2: remove all expired documents
function (docs) { docs => {
if (dontExpireStaleDocs) { return callback(null, docs) } if (dontExpireStaleDocs) return callback(null, docs)
const expiredDocsIds = [] const expiredDocsIds = []
const validDocs = [] const validDocs = []
const ttlIndexesFieldNames = Object.keys(self.ttlIndexes) const ttlIndexesFieldNames = Object.keys(this.ttlIndexes)
docs.forEach(function (doc) { docs.forEach(doc => {
let valid = true let valid = true
ttlIndexesFieldNames.forEach(function (i) { ttlIndexesFieldNames.forEach(i => {
if (doc[i] !== undefined && util.types.isDate(doc[i]) && Date.now() > doc[i].getTime() + self.ttlIndexes[i] * 1000) { if (doc[i] !== undefined && util.types.isDate(doc[i]) && Date.now() > doc[i].getTime() + this.ttlIndexes[i] * 1000) {
valid = false valid = false
} }
}) })
if (valid) { validDocs.push(doc) } else { expiredDocsIds.push(doc._id) } if (valid) validDocs.push(doc)
else expiredDocsIds.push(doc._id)
}) })
async.eachSeries(expiredDocsIds, function (_id, cb) { async.eachSeries(expiredDocsIds, (_id, cb) => {
self._remove({ _id: _id }, {}, function (err) { this._remove({ _id: _id }, {}, err => {
if (err) { return callback(err) } if (err) return callback(err)
return cb() return cb()
}) })
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
}, function (err) { }, err => {
// TODO: handle error // TODO: handle error
return callback(null, validDocs) return callback(null, validDocs)
}) })
@ -332,12 +320,12 @@ class Datastore extends EventEmitter {
/** /**
* Insert a new document * Insert a new document
* @param {Function} cb Optional callback, signature: err, insertedDoc * @param {Document} newDoc
* @param {Function} callback Optional callback, signature: err, insertedDoc
* *
* @api private Use Datastore.insert which has the same signature * @api private Use Datastore.insert which has the same signature
*/ */
_insert (newDoc, cb) { _insert (newDoc, callback = () => {}) {
const callback = cb || function () {}
let preparedDoc let preparedDoc
try { try {
@ -347,8 +335,8 @@ class Datastore extends EventEmitter {
return callback(e) return callback(e)
} }
this.persistence.persistNewState(Array.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) { this.persistence.persistNewState(Array.isArray(preparedDoc) ? preparedDoc : [preparedDoc], err => {
if (err) { return callback(err) } if (err) return callback(err)
return callback(null, model.deepCopy(preparedDoc)) return callback(null, model.deepCopy(preparedDoc))
}) })
} }
@ -357,12 +345,10 @@ class Datastore extends EventEmitter {
* Create a new _id that's not already in use * Create a new _id that's not already in use
*/ */
createNewId () { createNewId () {
let tentativeId = customUtils.uid(16) let attemptId = customUtils.uid(16)
// Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely small, so this is O(1) // Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely small, so this is O(1)
if (this.indexes._id.getMatching(tentativeId).length > 0) { if (this.indexes._id.getMatching(attemptId).length > 0) attemptId = this.createNewId()
tentativeId = this.createNewId() return attemptId
}
return tentativeId
} }
/** /**
@ -372,17 +358,16 @@ class Datastore extends EventEmitter {
*/ */
prepareDocumentForInsertion (newDoc) { prepareDocumentForInsertion (newDoc) {
let preparedDoc let preparedDoc
const self = this
if (Array.isArray(newDoc)) { if (Array.isArray(newDoc)) {
preparedDoc = [] preparedDoc = []
newDoc.forEach(function (doc) { preparedDoc.push(self.prepareDocumentForInsertion(doc)) }) newDoc.forEach(doc => { preparedDoc.push(this.prepareDocumentForInsertion(doc)) })
} else { } else {
preparedDoc = model.deepCopy(newDoc) preparedDoc = model.deepCopy(newDoc)
if (preparedDoc._id === undefined) { preparedDoc._id = this.createNewId() } if (preparedDoc._id === undefined) preparedDoc._id = this.createNewId()
const now = new Date() const now = new Date()
if (this.timestampData && preparedDoc.createdAt === undefined) { preparedDoc.createdAt = now } if (this.timestampData && preparedDoc.createdAt === undefined) preparedDoc.createdAt = now
if (this.timestampData && preparedDoc.updatedAt === undefined) { preparedDoc.updatedAt = now } if (this.timestampData && preparedDoc.updatedAt === undefined) preparedDoc.updatedAt = now
model.checkObject(preparedDoc) model.checkObject(preparedDoc)
} }
@ -394,11 +379,8 @@ class Datastore extends EventEmitter {
* @api private * @api private
*/ */
_insertInCache (preparedDoc) { _insertInCache (preparedDoc) {
if (Array.isArray(preparedDoc)) { if (Array.isArray(preparedDoc)) this._insertMultipleDocsInCache(preparedDoc)
this._insertMultipleDocsInCache(preparedDoc) else this.addToIndexes(preparedDoc)
} else {
this.addToIndexes(preparedDoc)
}
} }
/** /**
@ -407,22 +389,21 @@ class Datastore extends EventEmitter {
* @api private * @api private
*/ */
_insertMultipleDocsInCache (preparedDocs) { _insertMultipleDocsInCache (preparedDocs) {
let i let failingIndex
let failingI
let error let error
for (i = 0; i < preparedDocs.length; i += 1) { for (let i = 0; i < preparedDocs.length; i += 1) {
try { try {
this.addToIndexes(preparedDocs[i]) this.addToIndexes(preparedDocs[i])
} catch (e) { } catch (e) {
error = e error = e
failingI = i failingIndex = i
break break
} }
} }
if (error) { if (error) {
for (i = 0; i < failingI; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.removeFromIndexes(preparedDocs[i]) this.removeFromIndexes(preparedDocs[i])
} }
@ -437,6 +418,7 @@ class Datastore extends EventEmitter {
/** /**
* Count all documents matching the query * Count all documents matching the query
* @param {Object} query MongoDB-style query * @param {Object} query MongoDB-style query
* @param {Function} callback Optional callback, signature: err, count
*/ */
count (query, callback) { count (query, callback) {
const cursor = new Cursor(this, query, function (err, docs, callback) { const cursor = new Cursor(this, query, function (err, docs, callback) {
@ -444,11 +426,8 @@ class Datastore extends EventEmitter {
return callback(null, docs.length) return callback(null, docs.length)
}) })
if (typeof callback === 'function') { if (typeof callback === 'function') cursor.exec(callback)
cursor.exec(callback) else return cursor
} else {
return cursor
}
} }
/** /**
@ -456,75 +435,58 @@ class Datastore extends EventEmitter {
* If no callback is passed, we return the cursor so that user can limit, skip and finally exec * If no callback is passed, we return the cursor so that user can limit, skip and finally exec
* @param {Object} query MongoDB-style query * @param {Object} query MongoDB-style query
* @param {Object} projection MongoDB-style projection * @param {Object} projection MongoDB-style projection
* @param {Function} callback Optional callback, signature: err, docs
*/ */
find (query, projection, callback) { find (query, projection, callback) {
switch (arguments.length) { if (arguments.length === 1) {
case 1: projection = {}
// callback is undefined, will return a cursor
} else if (arguments.length === 2) {
if (typeof projection === 'function') {
callback = projection
projection = {} projection = {}
// callback is undefined, will return a cursor } // If not assume projection is an object and callback undefined
break
case 2:
if (typeof projection === 'function') {
callback = projection
projection = {}
} // If not assume projection is an object and callback undefined
break
} }
const cursor = new Cursor(this, query, function (err, docs, callback) { const cursor = new Cursor(this, query, function (err, docs, callback) {
const res = []
let i
if (err) { return callback(err) } if (err) { return callback(err) }
for (i = 0; i < docs.length; i += 1) { const res = docs.map(doc => model.deepCopy(doc))
res.push(model.deepCopy(docs[i]))
}
return callback(null, res) return callback(null, res)
}) })
cursor.projection(projection) cursor.projection(projection)
if (typeof callback === 'function') { if (typeof callback === 'function') cursor.exec(callback)
cursor.exec(callback) else return cursor
} else {
return cursor
}
} }
/** /**
* Find one document matching the query * Find one document matching the query
* @param {Object} query MongoDB-style query * @param {Object} query MongoDB-style query
* @param {Object} projection MongoDB-style projection * @param {Object} projection MongoDB-style projection
* @param {Function} callback Optional callback, signature: err, doc
*/ */
findOne (query, projection, callback) { findOne (query, projection, callback) {
switch (arguments.length) { if (arguments.length === 1) {
case 1: projection = {}
// callback is undefined, will return a cursor
} else if (arguments.length === 2) {
if (typeof projection === 'function') {
callback = projection
projection = {} projection = {}
// callback is undefined, will return a cursor } // If not assume projection is an object and callback undefined
break
case 2:
if (typeof projection === 'function') {
callback = projection
projection = {}
} // If not assume projection is an object and callback undefined
break
} }
const cursor = new Cursor(this, query, function (err, docs, callback) { const cursor = new Cursor(this, query, (err, docs, callback) => {
if (err) { return callback(err) } if (err) return callback(err)
if (docs.length === 1) { if (docs.length === 1) return callback(null, model.deepCopy(docs[0]))
return callback(null, model.deepCopy(docs[0])) else return callback(null, null)
} else {
return callback(null, null)
}
}) })
cursor.projection(projection).limit(1) cursor.projection(projection).limit(1)
if (typeof callback === 'function') { if (typeof callback === 'function') cursor.exec(callback)
cursor.exec(callback) else return cursor
} else {
return cursor
}
} }
/** /**
@ -553,29 +515,24 @@ class Datastore extends EventEmitter {
* @api private Use Datastore.update which has the same signature * @api private Use Datastore.update which has the same signature
*/ */
_update (query, updateQuery, options, cb) { _update (query, updateQuery, options, cb) {
const self = this
let numReplaced = 0
let i
if (typeof options === 'function') { if (typeof options === 'function') {
cb = options cb = options
options = {} options = {}
} }
const callback = cb || function () {} const callback = cb || (() => {})
const multi = options.multi !== undefined ? options.multi : false const multi = options.multi !== undefined ? options.multi : false
const upsert = options.upsert !== undefined ? options.upsert : false const upsert = options.upsert !== undefined ? options.upsert : false
async.waterfall([ async.waterfall([
function (cb) { // If upsert option is set, check whether we need to insert the doc cb => { // If upsert option is set, check whether we need to insert the doc
if (!upsert) { return cb() } if (!upsert) return cb()
// Need to use an internal function not tied to the executor to avoid deadlock // Need to use an internal function not tied to the executor to avoid deadlock
const cursor = new Cursor(self, query) const cursor = new Cursor(this, query)
cursor.limit(1)._exec(function (err, docs) { cursor.limit(1)._exec((err, docs) => {
if (err) { return callback(err) } if (err) return callback(err)
if (docs.length === 1) { if (docs.length === 1) return cb()
return cb() else {
} else {
let toBeInserted let toBeInserted
try { try {
@ -592,34 +549,35 @@ class Datastore extends EventEmitter {
} }
} }
return self._insert(toBeInserted, function (err, newDoc) { return this._insert(toBeInserted, (err, newDoc) => {
if (err) { return callback(err) } if (err) return callback(err)
return callback(null, 1, newDoc, true) return callback(null, 1, newDoc, true)
}) })
} }
}) })
}, },
function () { // Perform the update () => { // Perform the update
let numReplaced = 0
let modifiedDoc let modifiedDoc
const modifications = [] const modifications = []
let createdAt let createdAt
self.getCandidates(query, function (err, candidates) { this.getCandidates(query, (err, candidates) => {
if (err) { return callback(err) } if (err) return callback(err)
// Preparing update (if an error is thrown here neither the datafile nor // Preparing update (if an error is thrown here neither the datafile nor
// the in-memory indexes are affected) // the in-memory indexes are affected)
try { try {
for (i = 0; i < candidates.length; i += 1) { for (const candidate of candidates) {
if (model.match(candidates[i], query) && (multi || numReplaced === 0)) { if (model.match(candidate, query) && (multi || numReplaced === 0)) {
numReplaced += 1 numReplaced += 1
if (self.timestampData) { createdAt = candidates[i].createdAt } if (this.timestampData) { createdAt = candidate.createdAt }
modifiedDoc = model.modify(candidates[i], updateQuery) modifiedDoc = model.modify(candidate, updateQuery)
if (self.timestampData) { if (this.timestampData) {
modifiedDoc.createdAt = createdAt modifiedDoc.createdAt = createdAt
modifiedDoc.updatedAt = new Date() modifiedDoc.updatedAt = new Date()
} }
modifications.push({ oldDoc: candidates[i], newDoc: modifiedDoc }) modifications.push({ oldDoc: candidate, newDoc: modifiedDoc })
} }
} }
} catch (err) { } catch (err) {
@ -628,21 +586,21 @@ class Datastore extends EventEmitter {
// Change the docs in memory // Change the docs in memory
try { try {
self.updateIndexes(modifications) this.updateIndexes(modifications)
} catch (err) { } catch (err) {
return callback(err) return callback(err)
} }
// Update the datafile // Update the datafile
const updatedDocs = modifications.map(x => x.newDoc) const updatedDocs = modifications.map(x => x.newDoc)
self.persistence.persistNewState(updatedDocs, function (err) { this.persistence.persistNewState(updatedDocs, err => {
if (err) { return callback(err) } if (err) return callback(err)
if (!options.returnUpdatedDocs) { if (!options.returnUpdatedDocs) {
return callback(null, numReplaced) return callback(null, numReplaced)
} else { } else {
let updatedDocsDC = [] let updatedDocsDC = []
updatedDocs.forEach(function (doc) { updatedDocsDC.push(model.deepCopy(doc)) }) updatedDocs.forEach(doc => { updatedDocsDC.push(model.deepCopy(doc)) })
if (!multi) { updatedDocsDC = updatedDocsDC[0] } if (!multi) updatedDocsDC = updatedDocsDC[0]
return callback(null, numReplaced, updatedDocsDC) return callback(null, numReplaced, updatedDocsDC)
} }
}) })
@ -665,32 +623,32 @@ class Datastore extends EventEmitter {
* @api private Use Datastore.remove which has the same signature * @api private Use Datastore.remove which has the same signature
*/ */
_remove (query, options, cb) { _remove (query, options, cb) {
const self = this
let numRemoved = 0
const removedDocs = []
if (typeof options === 'function') { if (typeof options === 'function') {
cb = options cb = options
options = {} options = {}
} }
const callback = cb || function () {} const callback = cb || (() => {})
const multi = options.multi !== undefined ? options.multi : false const multi = options.multi !== undefined ? options.multi : false
this.getCandidates(query, true, function (err, candidates) { this.getCandidates(query, true, (err, candidates) => {
if (err) { return callback(err) } if (err) return callback(err)
const removedDocs = []
let numRemoved = 0
try { try {
candidates.forEach(function (d) { candidates.forEach(d => {
if (model.match(d, query) && (multi || numRemoved === 0)) { if (model.match(d, query) && (multi || numRemoved === 0)) {
numRemoved += 1 numRemoved += 1
removedDocs.push({ $$deleted: true, _id: d._id }) removedDocs.push({ $$deleted: true, _id: d._id })
self.removeFromIndexes(d) this.removeFromIndexes(d)
} }
}) })
} catch (err) { return callback(err) } } catch (err) {
return callback(err)
}
self.persistence.persistNewState(removedDocs, function (err) { this.persistence.persistNewState(removedDocs, err => {
if (err) { return callback(err) } if (err) return callback(err)
return callback(null, numRemoved) return callback(null, numRemoved)
}) })
}) })

@ -9,12 +9,11 @@ class Executor {
this.ready = false this.ready = false
// This queue will execute all commands, one-by-one in order // This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) { this.queue = async.queue((task, cb) => {
const newArguments = []
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array // task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (let i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]) } const newArguments = Array.from(task.arguments)
const lastArg = task.arguments[task.arguments.length - 1]
const lastArg = newArguments[newArguments.length - 1]
// Always tell the queue task is complete. Execute callback if any was given. // Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') { if (typeof lastArg === 'function') {
@ -29,10 +28,10 @@ class Executor {
} }
} else if (!lastArg && task.arguments.length !== 0) { } else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callback // false/undefined/null supplied as callback
newArguments[newArguments.length - 1] = function () { cb() } newArguments[newArguments.length - 1] = () => { cb() }
} else { } else {
// Nothing supplied as callback // Nothing supplied as callback
newArguments.push(function () { cb() }) newArguments.push(() => { cb() })
} }
task.fn.apply(task.this, newArguments) task.fn.apply(task.this, newArguments)
@ -50,11 +49,8 @@ class Executor {
* @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready * @param {Boolean} forceQueuing Optional (defaults to false) force executor to queue task even if it is not ready
*/ */
push (task, forceQueuing) { push (task, forceQueuing) {
if (this.ready || forceQueuing) { if (this.ready || forceQueuing) this.queue.push(task)
this.queue.push(task) else this.buffer.push(task)
} else {
this.buffer.push(task)
}
} }
/** /**
@ -62,9 +58,8 @@ class Executor {
* Automatically sets executor as ready * Automatically sets executor as ready
*/ */
processBuffer () { processBuffer () {
let i
this.ready = true this.ready = true
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]) } this.buffer.forEach(task => { this.queue.push(task) })
this.buffer = [] this.buffer = []
} }
} }

@ -6,19 +6,17 @@ const { uniq } = require('./utils.js')
/** /**
* Two indexed pointers are equal iif they point to the same place * Two indexed pointers are equal iif they point to the same place
*/ */
function checkValueEquality (a, b) { const checkValueEquality = (a, b) => a === b
return a === b
}
/** /**
* Type-aware projection * Type-aware projection
*/ */
function projectForUnique (elt) { function projectForUnique (elt) {
if (elt === null) { return '$null' } if (elt === null) return '$null'
if (typeof elt === 'string') { return '$string' + elt } if (typeof elt === 'string') return '$string' + elt
if (typeof elt === 'boolean') { return '$boolean' + elt } if (typeof elt === 'boolean') return '$boolean' + elt
if (typeof elt === 'number') { return '$number' + elt } if (typeof elt === 'number') return '$number' + elt
if (util.types.isDate(elt)) { return '$date' + elt.getTime() } // TODO: there is an obvious error here, if (util.types.isDate(elt)) return '$date' + elt.getTime()
return elt // Arrays and objects, will check for pointer equality return elt // Arrays and objects, will check for pointer equality
} }
@ -50,7 +48,7 @@ class Index {
reset (newData) { reset (newData) {
this.tree = new BinarySearchTree(this.treeOptions) this.tree = new BinarySearchTree(this.treeOptions)
if (newData) { this.insert(newData) } if (newData) this.insert(newData)
} }
/** /**
@ -60,8 +58,7 @@ class Index {
*/ */
insert (doc) { insert (doc) {
let keys let keys
let i let failingIndex
let failingI
let error let error
if (Array.isArray(doc)) { if (Array.isArray(doc)) {
@ -72,26 +69,25 @@ class Index {
const key = model.getDotValue(doc, this.fieldName) const key = model.getDotValue(doc, this.fieldName)
// We don't index documents that don't contain the field if the index is sparse // We don't index documents that don't contain the field if the index is sparse
if (key === undefined && this.sparse) { return } if (key === undefined && this.sparse) return
if (!Array.isArray(key)) { if (!Array.isArray(key)) this.tree.insert(key, doc)
this.tree.insert(key, doc) else {
} else {
// If an insert fails due to a unique constraint, roll back all inserts before it // If an insert fails due to a unique constraint, roll back all inserts before it
keys = uniq(key, projectForUnique) keys = uniq(key, projectForUnique)
for (i = 0; i < keys.length; i += 1) { for (let i = 0; i < keys.length; i += 1) {
try { try {
this.tree.insert(keys[i], doc) this.tree.insert(keys[i], doc)
} catch (e) { } catch (e) {
error = e error = e
failingI = i failingIndex = i
break break
} }
} }
if (error) { if (error) {
for (i = 0; i < failingI; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.tree.delete(keys[i], doc) this.tree.delete(keys[i], doc)
} }
@ -107,22 +103,21 @@ class Index {
* @API private * @API private
*/ */
insertMultipleDocs (docs) { insertMultipleDocs (docs) {
let i
let error let error
let failingI let failingIndex
for (i = 0; i < docs.length; i += 1) { for (let i = 0; i < docs.length; i += 1) {
try { try {
this.insert(docs[i]) this.insert(docs[i])
} catch (e) { } catch (e) {
error = e error = e
failingI = i failingIndex = i
break break
} }
} }
if (error) { if (error) {
for (i = 0; i < failingI; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.remove(docs[i]) this.remove(docs[i])
} }
@ -137,22 +132,20 @@ class Index {
* O(log(n)) * O(log(n))
*/ */
remove (doc) { remove (doc) {
const self = this
if (Array.isArray(doc)) { if (Array.isArray(doc)) {
doc.forEach(function (d) { self.remove(d) }) doc.forEach(d => { this.remove(d) })
return return
} }
const key = model.getDotValue(doc, this.fieldName) const key = model.getDotValue(doc, this.fieldName)
if (key === undefined && this.sparse) { return } if (key === undefined && this.sparse) return
if (!Array.isArray(key)) { if (!Array.isArray(key)) {
this.tree.delete(key, doc) this.tree.delete(key, doc)
} else { } else {
uniq(key, projectForUnique).forEach(function (_key) { uniq(key, projectForUnique).forEach(_key => {
self.tree.delete(_key, doc) this.tree.delete(_key, doc)
}) })
} }
} }
@ -187,31 +180,30 @@ class Index {
* @API private * @API private
*/ */
updateMultipleDocs (pairs) { updateMultipleDocs (pairs) {
let i let failingIndex
let failingI
let error let error
for (i = 0; i < pairs.length; i += 1) { for (let i = 0; i < pairs.length; i += 1) {
this.remove(pairs[i].oldDoc) this.remove(pairs[i].oldDoc)
} }
for (i = 0; i < pairs.length; i += 1) { for (let i = 0; i < pairs.length; i += 1) {
try { try {
this.insert(pairs[i].newDoc) this.insert(pairs[i].newDoc)
} catch (e) { } catch (e) {
error = e error = e
failingI = i failingIndex = i
break break
} }
} }
// If an error was raised, roll back changes in the inverse order // If an error was raised, roll back changes in the inverse order
if (error) { if (error) {
for (i = 0; i < failingI; i += 1) { for (let i = 0; i < failingIndex; i += 1) {
this.remove(pairs[i].newDoc) this.remove(pairs[i].newDoc)
} }
for (i = 0; i < pairs.length; i += 1) { for (let i = 0; i < pairs.length; i += 1) {
this.insert(pairs[i].oldDoc) this.insert(pairs[i].oldDoc)
} }
@ -225,10 +217,9 @@ class Index {
revertUpdate (oldDoc, newDoc) { revertUpdate (oldDoc, newDoc) {
const revert = [] const revert = []
if (!Array.isArray(oldDoc)) { if (!Array.isArray(oldDoc)) this.update(newDoc, oldDoc)
this.update(newDoc, oldDoc) else {
} else { oldDoc.forEach(pair => {
oldDoc.forEach(function (pair) {
revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc }) revert.push({ oldDoc: pair.newDoc, newDoc: pair.oldDoc })
}) })
this.update(revert) this.update(revert)
@ -241,21 +232,18 @@ class Index {
* @return {Array of documents} * @return {Array of documents}
*/ */
getMatching (value) { getMatching (value) {
const self = this if (!Array.isArray(value)) return this.tree.search(value)
else {
if (!Array.isArray(value)) {
return self.tree.search(value)
} else {
const _res = {} const _res = {}
const res = [] const res = []
value.forEach(function (v) { value.forEach(v => {
self.getMatching(v).forEach(function (doc) { this.getMatching(v).forEach(doc => {
_res[doc._id] = doc _res[doc._id] = doc
}) })
}) })
Object.keys(_res).forEach(function (_id) { Object.keys(_res).forEach(_id => {
res.push(_res[_id]) res.push(_res[_id])
}) })
@ -280,12 +268,8 @@ class Index {
getAll () { getAll () {
const res = [] const res = []
this.tree.executeOnEveryNode(function (node) { this.tree.executeOnEveryNode(node => {
let i res.push(...node.data)
for (i = 0; i < node.data.length; i += 1) {
res.push(node.data[i])
}
}) })
return res return res

@ -20,36 +20,38 @@ const arrayComparisonFunctions = {}
* Its serialized-then-deserialized version it will transformed into a Date object * Its serialized-then-deserialized version it will transformed into a Date object
* But you really need to want it to trigger such behaviour, even when warned not to use '$' at the beginning of the field names... * But you really need to want it to trigger such behaviour, even when warned not to use '$' at the beginning of the field names...
*/ */
function checkKey (k, v) { const checkKey = (k, v) => {
if (typeof k === 'number') { if (typeof k === 'number') k = k.toString()
k = k.toString()
}
if (k[0] === '$' && !(k === '$$date' && typeof v === 'number') && !(k === '$$deleted' && v === true) && !(k === '$$indexCreated') && !(k === '$$indexRemoved')) { if (
throw new Error('Field names cannot begin with the $ character') k[0] === '$' &&
} !(k === '$$date' && typeof v === 'number') &&
!(k === '$$deleted' && v === true) &&
!(k === '$$indexCreated') &&
!(k === '$$indexRemoved')
) throw new Error('Field names cannot begin with the $ character')
if (k.indexOf('.') !== -1) { if (k.indexOf('.') !== -1) throw new Error('Field names cannot contain a .')
throw new Error('Field names cannot contain a .')
}
} }
/** /**
* Check a DB object and throw an error if it's not valid * Check a DB object and throw an error if it's not valid
* Works by applying the above checkKey function to all fields recursively * Works by applying the above checkKey function to all fields recursively
*/ */
function checkObject (obj) { const checkObject = obj => {
if (Array.isArray(obj)) { if (Array.isArray(obj)) {
obj.forEach(function (o) { obj.forEach(o => {
checkObject(o) checkObject(o)
}) })
} }
if (typeof obj === 'object' && obj !== null) { if (typeof obj === 'object' && obj !== null) {
Object.keys(obj).forEach(function (k) { for (const k in obj) {
checkKey(k, obj[k]) if (Object.prototype.hasOwnProperty.call(obj, k)) {
checkObject(obj[k]) checkKey(k, obj[k])
}) checkObject(obj[k])
}
}
} }
} }
@ -61,36 +63,37 @@ function checkObject (obj) {
* Accepted primitive types: Number, String, Boolean, Date, null * Accepted primitive types: Number, String, Boolean, Date, null
* Accepted secondary types: Objects, Arrays * Accepted secondary types: Objects, Arrays
*/ */
function serialize (obj) { const serialize = obj => {
const res = JSON.stringify(obj, function (k, v) { return JSON.stringify(obj, function (k, v) {
checkKey(k, v) checkKey(k, v)
if (v === undefined) { return undefined } if (v === undefined) return undefined
if (v === null) { return null } if (v === null) return null
// Hackish way of checking if object is Date (this way it works between execution contexts in node-webkit). // Hackish way of checking if object is Date (this way it works between execution contexts in node-webkit).
// We can't use value directly because for dates it is already string in this function (date.toJSON was already called), so we use this // We can't use value directly because for dates it is already string in this function (date.toJSON was already called), so we use this
if (typeof this[k].getTime === 'function') { return { $$date: this[k].getTime() } } if (typeof this[k].getTime === 'function') return { $$date: this[k].getTime() }
return v return v
}) })
return res
} }
/** /**
* From a one-line representation of an object generate by the serialize function * From a one-line representation of an object generate by the serialize function
* Return the object itself * Return the object itself
*/ */
function deserialize (rawData) { const deserialize = rawData => JSON.parse(rawData, function (k, v) {
return JSON.parse(rawData, function (k, v) { if (k === '$$date') return new Date(v)
if (k === '$$date') { return new Date(v) } if (
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null) { return v } typeof v === 'string' ||
if (v && v.$$date) { return v.$$date } typeof v === 'number' ||
typeof v === 'boolean' ||
return v v === null
}) ) return v
} if (v && v.$$date) return v.$$date
return v
})
/** /**
* Deep copy a DB object * Deep copy a DB object
@ -98,29 +101,26 @@ function deserialize (rawData) {
* where the keys are valid, i.e. don't begin with $ and don't contain a . * where the keys are valid, i.e. don't begin with $ and don't contain a .
*/ */
function deepCopy (obj, strictKeys) { function deepCopy (obj, strictKeys) {
let res if (
typeof obj === 'boolean' ||
if (typeof obj === 'boolean' ||
typeof obj === 'number' || typeof obj === 'number' ||
typeof obj === 'string' || typeof obj === 'string' ||
obj === null || obj === null ||
(util.types.isDate(obj))) { (util.types.isDate(obj))
return obj ) return obj
}
if (Array.isArray(obj)) { if (Array.isArray(obj)) return obj.map(o => deepCopy(o, strictKeys))
res = []
obj.forEach(function (o) { res.push(deepCopy(o, strictKeys)) })
return res
}
if (typeof obj === 'object') { if (typeof obj === 'object') {
res = {} const res = {}
Object.keys(obj).forEach(function (k) { for (const k in obj) {
if (!strictKeys || (k[0] !== '$' && k.indexOf('.') === -1)) { if (
Object.prototype.hasOwnProperty.call(obj, k) &&
(!strictKeys || (k[0] !== '$' && k.indexOf('.') === -1))
) {
res[k] = deepCopy(obj[k], strictKeys) res[k] = deepCopy(obj[k], strictKeys)
} }
}) }
return res return res
} }
@ -131,34 +131,32 @@ function deepCopy (obj, strictKeys) {
* Tells if an object is a primitive type or a "real" object * Tells if an object is a primitive type or a "real" object
* Arrays are considered primitive * Arrays are considered primitive
*/ */
function isPrimitiveType (obj) { const isPrimitiveType = obj => (
return (typeof obj === 'boolean' || typeof obj === 'boolean' ||
typeof obj === 'number' || typeof obj === 'number' ||
typeof obj === 'string' || typeof obj === 'string' ||
obj === null || obj === null ||
util.types.isDate(obj) || util.types.isDate(obj) ||
Array.isArray(obj)) Array.isArray(obj)
} )
/** /**
* Utility functions for comparing things * Utility functions for comparing things
* Assumes type checking was already done (a and b already have the same type) * Assumes type checking was already done (a and b already have the same type)
* compareNSB works for numbers, strings and booleans * compareNSB works for numbers, strings and booleans
*/ */
function compareNSB (a, b) { const compareNSB = (a, b) => {
if (a < b) { return -1 } if (a < b) return -1
if (a > b) { return 1 } if (a > b) return 1
return 0 return 0
} }
function compareArrays (a, b) { const compareArrays = (a, b) => {
let i const minLength = Math.min(a.length, b.length)
let comp for (let i = 0; i < minLength; i += 1) {
const comp = compareThings(a[i], b[i])
for (i = 0; i < Math.min(a.length, b.length); i += 1) { if (comp !== 0) return comp
comp = compareThings(a[i], b[i])
if (comp !== 0) { return comp }
} }
// Common section was identical, longest one wins // Common section was identical, longest one wins
@ -175,47 +173,45 @@ function compareArrays (a, b) {
* *
* @param {Function} _compareStrings String comparing function, returning -1, 0 or 1, overriding default string comparison (useful for languages with accented letters) * @param {Function} _compareStrings String comparing function, returning -1, 0 or 1, overriding default string comparison (useful for languages with accented letters)
*/ */
function compareThings (a, b, _compareStrings) { const compareThings = (a, b, _compareStrings) => {
let comp
let i
const compareStrings = _compareStrings || compareNSB const compareStrings = _compareStrings || compareNSB
// undefined // undefined
if (a === undefined) { return b === undefined ? 0 : -1 } if (a === undefined) return b === undefined ? 0 : -1
if (b === undefined) { return a === undefined ? 0 : 1 } if (b === undefined) return 1 // no need to test if a === undefined
// null // null
if (a === null) { return b === null ? 0 : -1 } if (a === null) return b === null ? 0 : -1
if (b === null) { return a === null ? 0 : 1 } if (b === null) return 1 // no need to test if a === null
// Numbers // Numbers
if (typeof a === 'number') { return typeof b === 'number' ? compareNSB(a, b) : -1 } if (typeof a === 'number') return typeof b === 'number' ? compareNSB(a, b) : -1
if (typeof b === 'number') { return typeof a === 'number' ? compareNSB(a, b) : 1 } if (typeof b === 'number') return typeof a === 'number' ? compareNSB(a, b) : 1
// Strings // Strings
if (typeof a === 'string') { return typeof b === 'string' ? compareStrings(a, b) : -1 } if (typeof a === 'string') return typeof b === 'string' ? compareStrings(a, b) : -1
if (typeof b === 'string') { return typeof a === 'string' ? compareStrings(a, b) : 1 } if (typeof b === 'string') return typeof a === 'string' ? compareStrings(a, b) : 1
// Booleans // Booleans
if (typeof a === 'boolean') { return typeof b === 'boolean' ? compareNSB(a, b) : -1 } if (typeof a === 'boolean') return typeof b === 'boolean' ? compareNSB(a, b) : -1
if (typeof b === 'boolean') { return typeof a === 'boolean' ? compareNSB(a, b) : 1 } if (typeof b === 'boolean') return typeof a === 'boolean' ? compareNSB(a, b) : 1
// Dates // Dates
if (util.types.isDate(a)) { return util.types.isDate(b) ? compareNSB(a.getTime(), b.getTime()) : -1 } if (util.types.isDate(a)) return util.types.isDate(b) ? compareNSB(a.getTime(), b.getTime()) : -1
if (util.types.isDate(b)) { return util.types.isDate(a) ? compareNSB(a.getTime(), b.getTime()) : 1 } if (util.types.isDate(b)) return util.types.isDate(a) ? compareNSB(a.getTime(), b.getTime()) : 1
// Arrays (first element is most significant and so on) // Arrays (first element is most significant and so on)
if (Array.isArray(a)) { return Array.isArray(b) ? compareArrays(a, b) : -1 } if (Array.isArray(a)) return Array.isArray(b) ? compareArrays(a, b) : -1
if (Array.isArray(b)) { return Array.isArray(a) ? compareArrays(a, b) : 1 } if (Array.isArray(b)) return Array.isArray(a) ? compareArrays(a, b) : 1
// Objects // Objects
const aKeys = Object.keys(a).sort() const aKeys = Object.keys(a).sort()
const bKeys = Object.keys(b).sort() const bKeys = Object.keys(b).sort()
for (i = 0; i < Math.min(aKeys.length, bKeys.length); i += 1) { for (let i = 0; i < Math.min(aKeys.length, bKeys.length); i += 1) {
comp = compareThings(a[aKeys[i]], b[bKeys[i]]) const comp = compareThings(a[aKeys[i]], b[bKeys[i]])
if (comp !== 0) { return comp } if (comp !== 0) return comp
} }
return compareNSB(aKeys.length, bKeys.length) return compareNSB(aKeys.length, bKeys.length)
@ -237,14 +233,14 @@ function compareThings (a, b, _compareStrings) {
/** /**
* Set a field to a new value * Set a field to a new value
*/ */
lastStepModifierFunctions.$set = function (obj, field, value) { lastStepModifierFunctions.$set = (obj, field, value) => {
obj[field] = value obj[field] = value
} }
/** /**
* Unset a field * Unset a field
*/ */
lastStepModifierFunctions.$unset = function (obj, field, value) { lastStepModifierFunctions.$unset = (obj, field, value) => {
delete obj[field] delete obj[field]
} }
@ -254,29 +250,34 @@ lastStepModifierFunctions.$unset = function (obj, field, value) {
* Optional modifier $slice to slice the resulting array, see https://docs.mongodb.org/manual/reference/operator/update/slice/ * Optional modifier $slice to slice the resulting array, see https://docs.mongodb.org/manual/reference/operator/update/slice/
* Différeence with MongoDB: if $slice is specified and not $each, we act as if value is an empty array * Différeence with MongoDB: if $slice is specified and not $each, we act as if value is an empty array
*/ */
lastStepModifierFunctions.$push = function (obj, field, value) { lastStepModifierFunctions.$push = (obj, field, value) => {
// Create the array if it doesn't exist // Create the array if it doesn't exist
if (!Object.prototype.hasOwnProperty.call(obj, field)) { obj[field] = [] } if (!Object.prototype.hasOwnProperty.call(obj, field)) obj[field] = []
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $push an element on non-array values') } if (!Array.isArray(obj[field])) throw new Error('Can\'t $push an element on non-array values')
if (value !== null && typeof value === 'object' && value.$slice && value.$each === undefined) { if (
value.$each = [] value !== null &&
} typeof value === 'object' &&
value.$slice &&
value.$each === undefined
) value.$each = []
if (value !== null && typeof value === 'object' && value.$each) { if (value !== null && typeof value === 'object' && value.$each) {
if (Object.keys(value).length >= 3 || (Object.keys(value).length === 2 && value.$slice === undefined)) { throw new Error('Can only use $slice in cunjunction with $each when $push to array') } if (
if (!Array.isArray(value.$each)) { throw new Error('$each requires an array value') } Object.keys(value).length >= 3 ||
(Object.keys(value).length === 2 && value.$slice === undefined)
) throw new Error('Can only use $slice in cunjunction with $each when $push to array')
if (!Array.isArray(value.$each)) throw new Error('$each requires an array value')
value.$each.forEach(function (v) { value.$each.forEach(v => {
obj[field].push(v) obj[field].push(v)
}) })
if (value.$slice === undefined || typeof value.$slice !== 'number') { return } if (value.$slice === undefined || typeof value.$slice !== 'number') return
if (value.$slice === 0) { if (value.$slice === 0) obj[field] = []
obj[field] = [] else {
} else {
let start let start
let end let end
const n = obj[field].length const n = obj[field].length
@ -299,134 +300,112 @@ lastStepModifierFunctions.$push = function (obj, field, value) {
* No modification if the element is already in the array * No modification if the element is already in the array
* Note that it doesn't check whether the original array contains duplicates * Note that it doesn't check whether the original array contains duplicates
*/ */
lastStepModifierFunctions.$addToSet = function (obj, field, value) { lastStepModifierFunctions.$addToSet = (obj, field, value) => {
let addToSet = true
// Create the array if it doesn't exist // Create the array if it doesn't exist
if (!Object.prototype.hasOwnProperty.call(obj, field)) { obj[field] = [] } if (!Object.prototype.hasOwnProperty.call(obj, field)) { obj[field] = [] }
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $addToSet an element on non-array values') } if (!Array.isArray(obj[field])) throw new Error('Can\'t $addToSet an element on non-array values')
if (value !== null && typeof value === 'object' && value.$each) { if (value !== null && typeof value === 'object' && value.$each) {
if (Object.keys(value).length > 1) { throw new Error('Can\'t use another field in conjunction with $each') } if (Object.keys(value).length > 1) throw new Error('Can\'t use another field in conjunction with $each')
if (!Array.isArray(value.$each)) { throw new Error('$each requires an array value') } if (!Array.isArray(value.$each)) throw new Error('$each requires an array value')
value.$each.forEach(function (v) { value.$each.forEach(v => {
lastStepModifierFunctions.$addToSet(obj, field, v) lastStepModifierFunctions.$addToSet(obj, field, v)
}) })
} else { } else {
obj[field].forEach(function (v) { let addToSet = true
if (compareThings(v, value) === 0) { addToSet = false } obj[field].forEach(v => {
if (compareThings(v, value) === 0) addToSet = false
}) })
if (addToSet) { obj[field].push(value) } if (addToSet) obj[field].push(value)
} }
} }
/** /**
* Remove the first or last element of an array * Remove the first or last element of an array
*/ */
lastStepModifierFunctions.$pop = function (obj, field, value) { lastStepModifierFunctions.$pop = (obj, field, value) => {
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $pop an element from non-array values') } if (!Array.isArray(obj[field])) throw new Error('Can\'t $pop an element from non-array values')
if (typeof value !== 'number') { throw new Error(value + ' isn\'t an integer, can\'t use it with $pop') } if (typeof value !== 'number') throw new Error(`${value} isn't an integer, can't use it with $pop`)
if (value === 0) { return } if (value === 0) return
if (value > 0) { if (value > 0) obj[field] = obj[field].slice(0, obj[field].length - 1)
obj[field] = obj[field].slice(0, obj[field].length - 1) else obj[field] = obj[field].slice(1)
} else {
obj[field] = obj[field].slice(1)
}
} }
/** /**
* Removes all instances of a value from an existing array * Removes all instances of a value from an existing array
*/ */
lastStepModifierFunctions.$pull = function (obj, field, value) { lastStepModifierFunctions.$pull = (obj, field, value) => {
if (!Array.isArray(obj[field])) { throw new Error('Can\'t $pull an element from non-array values') } if (!Array.isArray(obj[field])) throw new Error('Can\'t $pull an element from non-array values')
const arr = obj[field] const arr = obj[field]
for (let i = arr.length - 1; i >= 0; i -= 1) { for (let i = arr.length - 1; i >= 0; i -= 1) {
if (match(arr[i], value)) { if (match(arr[i], value)) arr.splice(i, 1)
arr.splice(i, 1)
}
} }
} }
/** /**
* Increment a numeric field's value * Increment a numeric field's value
*/ */
lastStepModifierFunctions.$inc = function (obj, field, value) { lastStepModifierFunctions.$inc = (obj, field, value) => {
if (typeof value !== 'number') { throw new Error(value + ' must be a number') } if (typeof value !== 'number') throw new Error(`${value} must be a number`)
if (typeof obj[field] !== 'number') { if (typeof obj[field] !== 'number') {
if (!Object.prototype.hasOwnProperty.call(obj, field)) { if (!Object.prototype.hasOwnProperty.call(obj, field)) obj[field] = value
obj[field] = value else throw new Error('Don\'t use the $inc modifier on non-number fields')
} else { } else obj[field] += value
throw new Error('Don\'t use the $inc modifier on non-number fields')
}
} else {
obj[field] += value
}
} }
/** /**
* Updates the value of the field, only if specified field is greater than the current value of the field * Updates the value of the field, only if specified field is greater than the current value of the field
*/ */
lastStepModifierFunctions.$max = function (obj, field, value) { lastStepModifierFunctions.$max = (obj, field, value) => {
if (typeof obj[field] === 'undefined') { if (typeof obj[field] === 'undefined') obj[field] = value
obj[field] = value else if (value > obj[field]) obj[field] = value
} else if (value > obj[field]) {
obj[field] = value
}
} }
/** /**
* Updates the value of the field, only if specified field is smaller than the current value of the field * Updates the value of the field, only if specified field is smaller than the current value of the field
*/ */
lastStepModifierFunctions.$min = function (obj, field, value) { lastStepModifierFunctions.$min = (obj, field, value) => {
if (typeof obj[field] === 'undefined') { if (typeof obj[field] === 'undefined') obj[field] = value
obj[field] = value else if (value < obj[field]) obj[field] = value
} else if (value < obj[field]) {
obj[field] = value
}
} }
// Given its name, create the complete modifier function // Given its name, create the complete modifier function
function createModifierFunction (modifier) { const createModifierFunction = modifier => (obj, field, value) => {
return function (obj, field, value) { const fieldParts = typeof field === 'string' ? field.split('.') : field
const fieldParts = typeof field === 'string' ? field.split('.') : field
if (fieldParts.length === 1) lastStepModifierFunctions[modifier](obj, field, value)
if (fieldParts.length === 1) { else {
lastStepModifierFunctions[modifier](obj, field, value) if (obj[fieldParts[0]] === undefined) {
} else { if (modifier === '$unset') return // Bad looking specific fix, needs to be generalized modifiers that behave like $unset are implemented
if (obj[fieldParts[0]] === undefined) { obj[fieldParts[0]] = {}
if (modifier === '$unset') { return } // Bad looking specific fix, needs to be generalized modifiers that behave like $unset are implemented
obj[fieldParts[0]] = {}
}
modifierFunctions[modifier](obj[fieldParts[0]], fieldParts.slice(1), value)
} }
modifierFunctions[modifier](obj[fieldParts[0]], fieldParts.slice(1), value)
} }
} }
// Actually create all modifier functions // Actually create all modifier functions
Object.keys(lastStepModifierFunctions).forEach(function (modifier) { Object.keys(lastStepModifierFunctions).forEach(modifier => {
modifierFunctions[modifier] = createModifierFunction(modifier) modifierFunctions[modifier] = createModifierFunction(modifier)
}) })
/** /**
* Modify a DB object according to an update query * Modify a DB object according to an update query
*/ */
function modify (obj, updateQuery) { const modify = (obj, updateQuery) => {
const keys = Object.keys(updateQuery) const keys = Object.keys(updateQuery)
const firstChars = keys.map(item => item[0]) const firstChars = keys.map(item => item[0])
const dollarFirstChars = firstChars.filter(c => c === '$') const dollarFirstChars = firstChars.filter(c => c === '$')
let newDoc let newDoc
let modifiers let modifiers
if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) { throw new Error('You cannot change a document\'s _id') } if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) throw new Error('You cannot change a document\'s _id')
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) { if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) throw new Error('You cannot mix modifiers and normal fields')
throw new Error('You cannot mix modifiers and normal fields')
}
if (dollarFirstChars.length === 0) { if (dollarFirstChars.length === 0) {
// Simply replace the object with the update query contents // Simply replace the object with the update query contents
@ -436,17 +415,15 @@ function modify (obj, updateQuery) {
// Apply modifiers // Apply modifiers
modifiers = uniq(keys) modifiers = uniq(keys)
newDoc = deepCopy(obj) newDoc = deepCopy(obj)
modifiers.forEach(function (m) { modifiers.forEach(m => {
if (!modifierFunctions[m]) { throw new Error('Unknown modifier ' + m) } if (!modifierFunctions[m]) throw new Error(`Unknown modifier ${m}`)
// Can't rely on Object.keys throwing on non objects since ES6 // Can't rely on Object.keys throwing on non objects since ES6
// Not 100% satisfying as non objects can be interpreted as objects but no false negatives so we can live with it // Not 100% satisfying as non objects can be interpreted as objects but no false negatives so we can live with it
if (typeof updateQuery[m] !== 'object') { if (typeof updateQuery[m] !== 'object') throw new Error(`Modifier ${m}'s argument must be an object`)
throw new Error('Modifier ' + m + '\'s argument must be an object')
}
const keys = Object.keys(updateQuery[m]) const keys = Object.keys(updateQuery[m])
keys.forEach(function (k) { keys.forEach(k => {
modifierFunctions[m](newDoc, k, updateQuery[m][k]) modifierFunctions[m](newDoc, k, updateQuery[m][k])
}) })
}) })
@ -455,7 +432,7 @@ function modify (obj, updateQuery) {
// Check result is valid and return it // Check result is valid and return it
checkObject(newDoc) checkObject(newDoc)
if (obj._id !== newDoc._id) { throw new Error('You can\'t change a document\'s _id') } if (obj._id !== newDoc._id) throw new Error('You can\'t change a document\'s _id')
return newDoc return newDoc
} }
@ -468,33 +445,23 @@ function modify (obj, updateQuery) {
* @param {Object} obj * @param {Object} obj
* @param {String} field * @param {String} field
*/ */
function getDotValue (obj, field) { const getDotValue = (obj, field) => {
const fieldParts = typeof field === 'string' ? field.split('.') : field const fieldParts = typeof field === 'string' ? field.split('.') : field
let i
let objs
if (!obj) { return undefined } // field cannot be empty so that means we should return undefined so that nothing can match if (!obj) return undefined // field cannot be empty so that means we should return undefined so that nothing can match
if (fieldParts.length === 0) { return obj } if (fieldParts.length === 0) return obj
if (fieldParts.length === 1) { return obj[fieldParts[0]] } if (fieldParts.length === 1) return obj[fieldParts[0]]
if (Array.isArray(obj[fieldParts[0]])) { if (Array.isArray(obj[fieldParts[0]])) {
// If the next field is an integer, return only this item of the array // If the next field is an integer, return only this item of the array
i = parseInt(fieldParts[1], 10) const i = parseInt(fieldParts[1], 10)
if (typeof i === 'number' && !isNaN(i)) { if (typeof i === 'number' && !isNaN(i)) return getDotValue(obj[fieldParts[0]][i], fieldParts.slice(2))
return getDotValue(obj[fieldParts[0]][i], fieldParts.slice(2))
}
// Return the array of values // Return the array of values
objs = [] return obj[fieldParts[0]].map(el => getDotValue(el, fieldParts.slice(1)))
for (i = 0; i < obj[fieldParts[0]].length; i += 1) { } else return getDotValue(obj[fieldParts[0]], fieldParts.slice(1))
objs.push(getDotValue(obj[fieldParts[0]][i], fieldParts.slice(1)))
}
return objs
} else {
return getDotValue(obj[fieldParts[0]], fieldParts.slice(1))
}
} }
/** /**
@ -503,24 +470,33 @@ function getDotValue (obj, field) {
* In the case of object, we check deep equality * In the case of object, we check deep equality
* Returns true if they are, false otherwise * Returns true if they are, false otherwise
*/ */
function areThingsEqual (a, b) { const areThingsEqual = (a, b) => {
let aKeys
let bKeys
let i
// Strings, booleans, numbers, null // Strings, booleans, numbers, null
if (a === null || typeof a === 'string' || typeof a === 'boolean' || typeof a === 'number' || if (
b === null || typeof b === 'string' || typeof b === 'boolean' || typeof b === 'number') { return a === b } a === null ||
typeof a === 'string' ||
typeof a === 'boolean' ||
typeof a === 'number' ||
b === null ||
typeof b === 'string' ||
typeof b === 'boolean' ||
typeof b === 'number'
) return a === b
// Dates // Dates
if (util.types.isDate(a) || util.types.isDate(b)) { return util.types.isDate(a) && util.types.isDate(b) && a.getTime() === b.getTime() } if (util.types.isDate(a) || util.types.isDate(b)) return util.types.isDate(a) && util.types.isDate(b) && a.getTime() === b.getTime()
// Arrays (no match since arrays are used as a $in) // Arrays (no match since arrays are used as a $in)
// undefined (no match since they mean field doesn't exist and can't be serialized) // undefined (no match since they mean field doesn't exist and can't be serialized)
if ((!(Array.isArray(a) && Array.isArray(b)) && (Array.isArray(a) || Array.isArray(b))) || a === undefined || b === undefined) { return false } if (
(!(Array.isArray(a) && Array.isArray(b)) && (Array.isArray(a) || Array.isArray(b))) ||
a === undefined || b === undefined
) return false
// General objects (check for deep equality) // General objects (check for deep equality)
// a and b should be objects at this point // a and b should be objects at this point
let aKeys
let bKeys
try { try {
aKeys = Object.keys(a) aKeys = Object.keys(a)
bKeys = Object.keys(b) bKeys = Object.keys(b)
@ -528,10 +504,10 @@ function areThingsEqual (a, b) {
return false return false
} }
if (aKeys.length !== bKeys.length) { return false } if (aKeys.length !== bKeys.length) return false
for (i = 0; i < aKeys.length; i += 1) { for (const el of aKeys) {
if (bKeys.indexOf(aKeys[i]) === -1) { return false } if (bKeys.indexOf(el) === -1) return false
if (!areThingsEqual(a[aKeys[i]], b[aKeys[i]])) { return false } if (!areThingsEqual(a[el], b[el])) return false
} }
return true return true
} }
@ -539,13 +515,17 @@ function areThingsEqual (a, b) {
/** /**
* Check that two values are comparable * Check that two values are comparable
*/ */
function areComparable (a, b) { const areComparable = (a, b) => {
if (typeof a !== 'string' && typeof a !== 'number' && !util.types.isDate(a) && if (
typeof b !== 'string' && typeof b !== 'number' && !util.types.isDate(b)) { typeof a !== 'string' &&
return false typeof a !== 'number' &&
} !util.types.isDate(a) &&
typeof b !== 'string' &&
if (typeof a !== typeof b) { return false } typeof b !== 'number' &&
!util.types.isDate(b)
) return false
if (typeof a !== typeof b) return false
return true return true
} }
@ -555,88 +535,62 @@ function areComparable (a, b) {
* @param {Native value} a Value in the object * @param {Native value} a Value in the object
* @param {Native value} b Value in the query * @param {Native value} b Value in the query
*/ */
comparisonFunctions.$lt = function (a, b) { comparisonFunctions.$lt = (a, b) => areComparable(a, b) && a < b
return areComparable(a, b) && a < b
}
comparisonFunctions.$lte = function (a, b) {
return areComparable(a, b) && a <= b
}
comparisonFunctions.$gt = function (a, b) { comparisonFunctions.$lte = (a, b) => areComparable(a, b) && a <= b
return areComparable(a, b) && a > b
}
comparisonFunctions.$gte = function (a, b) { comparisonFunctions.$gt = (a, b) => areComparable(a, b) && a > b
return areComparable(a, b) && a >= b
}
comparisonFunctions.$ne = function (a, b) { comparisonFunctions.$gte = (a, b) => areComparable(a, b) && a >= b
if (a === undefined) { return true }
return !areThingsEqual(a, b)
}
comparisonFunctions.$in = function (a, b) { comparisonFunctions.$ne = (a, b) => a === undefined || !areThingsEqual(a, b)
let i
if (!Array.isArray(b)) { throw new Error('$in operator called with a non-array') } comparisonFunctions.$in = (a, b) => {
if (!Array.isArray(b)) throw new Error('$in operator called with a non-array')
for (i = 0; i < b.length; i += 1) { for (const el of b) {
if (areThingsEqual(a, b[i])) { return true } if (areThingsEqual(a, el)) return true
} }
return false return false
} }
comparisonFunctions.$nin = function (a, b) { comparisonFunctions.$nin = (a, b) => {
if (!Array.isArray(b)) { throw new Error('$nin operator called with a non-array') } if (!Array.isArray(b)) throw new Error('$nin operator called with a non-array')
return !comparisonFunctions.$in(a, b) return !comparisonFunctions.$in(a, b)
} }
comparisonFunctions.$regex = function (a, b) { comparisonFunctions.$regex = (a, b) => {
if (!util.types.isRegExp(b)) { throw new Error('$regex operator called with non regular expression') } if (!util.types.isRegExp(b)) throw new Error('$regex operator called with non regular expression')
if (typeof a !== 'string') { if (typeof a !== 'string') return false
return false else return b.test(a)
} else {
return b.test(a)
}
} }
comparisonFunctions.$exists = function (value, exists) { comparisonFunctions.$exists = (value, exists) => {
if (exists || exists === '') { // This will be true for all values of stat except false, null, undefined and 0 // This will be true for all values of stat except false, null, undefined and 0
exists = true // That's strange behaviour (we should only use true/false) but that's the way Mongo does it... // That's strange behaviour (we should only use true/false) but that's the way Mongo does it...
} else { if (exists || exists === '') exists = true
exists = false else exists = false
}
if (value === undefined) { if (value === undefined) return !exists
return !exists else return exists
} else {
return exists
}
} }
// Specific to arrays // Specific to arrays
comparisonFunctions.$size = function (obj, value) { comparisonFunctions.$size = (obj, value) => {
if (!Array.isArray(obj)) { return false } if (!Array.isArray(obj)) return false
if (value % 1 !== 0) { throw new Error('$size operator called without an integer') } if (value % 1 !== 0) throw new Error('$size operator called without an integer')
return obj.length === value return obj.length === value
} }
comparisonFunctions.$elemMatch = function (obj, value) {
if (!Array.isArray(obj)) { return false } comparisonFunctions.$elemMatch = (obj, value) => {
let i = obj.length if (!Array.isArray(obj)) return false
let result = false // Initialize result return obj.some(el => match(el, value))
while (i--) {
if (match(obj[i], value)) { // If match for array element, return true
result = true
break
}
}
return result
} }
arrayComparisonFunctions.$size = true arrayComparisonFunctions.$size = true
arrayComparisonFunctions.$elemMatch = true arrayComparisonFunctions.$elemMatch = true
@ -645,13 +599,11 @@ arrayComparisonFunctions.$elemMatch = true
* @param {Model} obj * @param {Model} obj
* @param {Array of Queries} query * @param {Array of Queries} query
*/ */
logicalOperators.$or = function (obj, query) { logicalOperators.$or = (obj, query) => {
let i if (!Array.isArray(query)) throw new Error('$or operator used without an array')
if (!Array.isArray(query)) { throw new Error('$or operator used without an array') } for (let i = 0; i < query.length; i += 1) {
if (match(obj, query[i])) return true
for (i = 0; i < query.length; i += 1) {
if (match(obj, query[i])) { return true }
} }
return false return false
@ -662,13 +614,11 @@ logicalOperators.$or = function (obj, query) {
* @param {Model} obj * @param {Model} obj
* @param {Array of Queries} query * @param {Array of Queries} query
*/ */
logicalOperators.$and = function (obj, query) { logicalOperators.$and = (obj, query) => {
let i if (!Array.isArray(query)) throw new Error('$and operator used without an array')
if (!Array.isArray(query)) { throw new Error('$and operator used without an array') }
for (i = 0; i < query.length; i += 1) { for (let i = 0; i < query.length; i += 1) {
if (!match(obj, query[i])) { return false } if (!match(obj, query[i])) return false
} }
return true return true
@ -679,20 +629,18 @@ logicalOperators.$and = function (obj, query) {
* @param {Model} obj * @param {Model} obj
* @param {Query} query * @param {Query} query
*/ */
logicalOperators.$not = function (obj, query) { logicalOperators.$not = (obj, query) => !match(obj, query)
return !match(obj, query)
}
/** /**
* Use a function to match * Use a function to match
* @param {Model} obj * @param {Model} obj
* @param {Query} query * @param {Query} query
*/ */
logicalOperators.$where = function (obj, fn) { logicalOperators.$where = (obj, fn) => {
if (typeof fn !== 'function') { throw new Error('$where operator used without a function') } if (typeof fn !== 'function') throw new Error('$where operator used without a function')
const result = fn.call(obj) const result = fn.call(obj)
if (typeof result !== 'boolean') { throw new Error('$where function must return boolean') } if (typeof result !== 'boolean') throw new Error('$where function must return boolean')
return result return result
} }
@ -702,29 +650,20 @@ logicalOperators.$where = function (obj, fn) {
* @param {Object} obj Document to check * @param {Object} obj Document to check
* @param {Object} query * @param {Object} query
*/ */
function match (obj, query) { const match = (obj, query) => {
let queryKey
let queryValue
let i
// Primitive query against a primitive type // Primitive query against a primitive type
// This is a bit of a hack since we construct an object with an arbitrary key only to dereference it later // This is a bit of a hack since we construct an object with an arbitrary key only to dereference it later
// But I don't have time for a cleaner implementation now // But I don't have time for a cleaner implementation now
if (isPrimitiveType(obj) || isPrimitiveType(query)) { if (isPrimitiveType(obj) || isPrimitiveType(query)) return matchQueryPart({ needAKey: obj }, 'needAKey', query)
return matchQueryPart({ needAKey: obj }, 'needAKey', query)
}
// Normal query // Normal query
const queryKeys = Object.keys(query) for (const queryKey in query) {
for (i = 0; i < queryKeys.length; i += 1) { if (Object.prototype.hasOwnProperty.call(query, queryKey)) {
queryKey = queryKeys[i] const queryValue = query[queryKey]
queryValue = query[queryKey] if (queryKey[0] === '$') {
if (!logicalOperators[queryKey]) throw new Error(`Unknown logical operator ${queryKey}`)
if (queryKey[0] === '$') { if (!logicalOperators[queryKey](obj, queryValue)) return false
if (!logicalOperators[queryKey]) { throw new Error('Unknown logical operator ' + queryKey) } } else if (!matchQueryPart(obj, queryKey, queryValue)) return false
if (!logicalOperators[queryKey](obj, queryValue)) { return false }
} else {
if (!matchQueryPart(obj, queryKey, queryValue)) { return false }
} }
} }
@ -737,29 +676,22 @@ function match (obj, query) {
*/ */
function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) { function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) {
const objValue = getDotValue(obj, queryKey) const objValue = getDotValue(obj, queryKey)
let i
let keys
let firstChars
let dollarFirstChars
// Check if the value is an array if we don't force a treatment as value // Check if the value is an array if we don't force a treatment as value
if (Array.isArray(objValue) && !treatObjAsValue) { if (Array.isArray(objValue) && !treatObjAsValue) {
// If the queryValue is an array, try to perform an exact match // If the queryValue is an array, try to perform an exact match
if (Array.isArray(queryValue)) { if (Array.isArray(queryValue)) return matchQueryPart(obj, queryKey, queryValue, true)
return matchQueryPart(obj, queryKey, queryValue, true)
}
// Check if we are using an array-specific comparison function // Check if we are using an array-specific comparison function
if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue)) { if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue)) {
keys = Object.keys(queryValue) for (const key in queryValue) {
for (i = 0; i < keys.length; i += 1) { if (Object.prototype.hasOwnProperty.call(queryValue, key) && arrayComparisonFunctions[key]) { return matchQueryPart(obj, queryKey, queryValue, true) }
if (arrayComparisonFunctions[keys[i]]) { return matchQueryPart(obj, queryKey, queryValue, true) }
} }
} }
// If not, treat it as an array of { obj, query } where there needs to be at least one match // If not, treat it as an array of { obj, query } where there needs to be at least one match
for (i = 0; i < objValue.length; i += 1) { for (const el of objValue) {
if (matchQueryPart({ k: objValue[i] }, 'k', queryValue)) { return true } // k here could be any string if (matchQueryPart({ k: el }, 'k', queryValue)) return true // k here could be any string
} }
return false return false
} }
@ -767,33 +699,29 @@ function matchQueryPart (obj, queryKey, queryValue, treatObjAsValue) {
// queryValue is an actual object. Determine whether it contains comparison operators // queryValue is an actual object. Determine whether it contains comparison operators
// or only normal fields. Mixed objects are not allowed // or only normal fields. Mixed objects are not allowed
if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue) && !Array.isArray(queryValue)) { if (queryValue !== null && typeof queryValue === 'object' && !util.types.isRegExp(queryValue) && !Array.isArray(queryValue)) {
keys = Object.keys(queryValue) const keys = Object.keys(queryValue)
firstChars = keys.map(item => item[0]) const firstChars = keys.map(item => item[0])
dollarFirstChars = firstChars.filter(c => c === '$') const dollarFirstChars = firstChars.filter(c => c === '$')
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) { if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) throw new Error('You cannot mix operators and normal fields')
throw new Error('You cannot mix operators and normal fields')
}
// queryValue is an object of this form: { $comparisonOperator1: value1, ... } // queryValue is an object of this form: { $comparisonOperator1: value1, ... }
if (dollarFirstChars.length > 0) { if (dollarFirstChars.length > 0) {
for (i = 0; i < keys.length; i += 1) { for (const key of keys) {
if (!comparisonFunctions[keys[i]]) { throw new Error('Unknown comparison function ' + keys[i]) } if (!comparisonFunctions[key]) throw new Error(`Unknown comparison function ${key}`)
if (!comparisonFunctions[keys[i]](objValue, queryValue[keys[i]])) { return false } if (!comparisonFunctions[key](objValue, queryValue[key])) return false
} }
return true return true
} }
} }
// Using regular expressions with basic querying // Using regular expressions with basic querying
if (util.types.isRegExp(queryValue)) { return comparisonFunctions.$regex(objValue, queryValue) } if (util.types.isRegExp(queryValue)) return comparisonFunctions.$regex(objValue, queryValue)
// queryValue is either a native value or a normal object // queryValue is either a native value or a normal object
// Basic matching is possible // Basic matching is possible
if (!areThingsEqual(objValue, queryValue)) { return false } return areThingsEqual(objValue, queryValue)
return true
} }
// Interface // Interface

@ -19,31 +19,33 @@ class Persistence {
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion) * Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
*/ */
constructor (options) { constructor (options) {
let i
let j
let randomString
this.db = options.db this.db = options.db
this.inMemoryOnly = this.db.inMemoryOnly this.inMemoryOnly = this.db.inMemoryOnly
this.filename = this.db.filename this.filename = this.db.filename
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1 this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') { if (
throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files') !this.inMemoryOnly &&
} this.filename &&
this.filename.charAt(this.filename.length - 1) === '~'
) throw new Error('The datafile name can\'t end with a ~, which is reserved for crash safe backup files')
// After serialization and before deserialization hooks with some basic sanity checks // After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) { if (
throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss') options.afterSerialization &&
} !options.beforeDeserialization
if (!options.afterSerialization && options.beforeDeserialization) { ) throw new Error('Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss')
throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss') if (
} !options.afterSerialization &&
this.afterSerialization = options.afterSerialization || function (s) { return s } options.beforeDeserialization
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s } ) throw new Error('Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss')
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) { this.afterSerialization = options.afterSerialization || (s => s)
randomString = customUtils.uid(i) this.beforeDeserialization = options.beforeDeserialization || (s => s)
for (let i = 1; i < 30; i += 1) {
for (let j = 0; j < 10; j += 1) {
const randomString = customUtils.uid(i)
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) { if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss') throw new Error('beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss')
} }
@ -67,33 +69,31 @@ class Persistence {
* Persist cached database * Persist cached database
* This serves as a compaction function since the cache always contains only the number of documents in the collection * This serves as a compaction function since the cache always contains only the number of documents in the collection
* while the data file is append-only so it may grow larger * while the data file is append-only so it may grow larger
* @param {Function} cb Optional callback, signature: err * @param {Function} callback Optional callback, signature: err
*/ */
persistCachedDatabase (cb) { persistCachedDatabase (callback = () => {}) {
const callback = cb || function () {}
let toPersist = '' let toPersist = ''
const self = this
if (this.inMemoryOnly) { return callback(null) } if (this.inMemoryOnly) return callback(null)
this.db.getAllData().forEach(function (doc) { this.db.getAllData().forEach(doc => {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n' toPersist += this.afterSerialization(model.serialize(doc)) + '\n'
}) })
Object.keys(this.db.indexes).forEach(function (fieldName) { Object.keys(this.db.indexes).forEach(fieldName => {
if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ toPersist += this.afterSerialization(model.serialize({
$$indexCreated: { $$indexCreated: {
fieldName: fieldName, fieldName: fieldName,
unique: self.db.indexes[fieldName].unique, unique: this.db.indexes[fieldName].unique,
sparse: self.db.indexes[fieldName].sparse sparse: this.db.indexes[fieldName].sparse
} }
})) + '\n' })) + '\n'
} }
}) })
storage.crashSafeWriteFile(this.filename, toPersist, function (err) { storage.crashSafeWriteFile(this.filename, toPersist, err => {
if (err) { return callback(err) } if (err) return callback(err)
self.db.emit('compaction.done') this.db.emit('compaction.done')
return callback(null) return callback(null)
}) })
} }
@ -110,14 +110,13 @@ class Persistence {
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds * @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
*/ */
setAutocompactionInterval (interval) { setAutocompactionInterval (interval) {
const self = this
const minInterval = 5000 const minInterval = 5000
const realInterval = Math.max(interval || 0, minInterval) const realInterval = Math.max(interval || 0, minInterval)
this.stopAutocompaction() this.stopAutocompaction()
this.autocompactionIntervalId = setInterval(function () { this.autocompactionIntervalId = setInterval(() => {
self.compactDatafile() this.compactDatafile()
}, realInterval) }, realInterval)
} }
@ -125,32 +124,28 @@ class Persistence {
* Stop autocompaction (do nothing if autocompaction was not running) * Stop autocompaction (do nothing if autocompaction was not running)
*/ */
stopAutocompaction () { stopAutocompaction () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId) } if (this.autocompactionIntervalId) clearInterval(this.autocompactionIntervalId)
} }
/** /**
* Persist new state for the given newDocs (can be insertion, update or removal) * Persist new state for the given newDocs (can be insertion, update or removal)
* Use an append-only format * Use an append-only format
* @param {Array} newDocs Can be empty if no doc was updated/removed * @param {Array} newDocs Can be empty if no doc was updated/removed
* @param {Function} cb Optional, signature: err * @param {Function} callback Optional, signature: err
*/ */
persistNewState (newDocs, cb) { persistNewState (newDocs, callback = () => {}) {
const self = this
let toPersist = '' let toPersist = ''
const callback = cb || function () {}
// In-memory only datastore // In-memory only datastore
if (self.inMemoryOnly) { return callback(null) } if (this.inMemoryOnly) return callback(null)
newDocs.forEach(function (doc) { newDocs.forEach(doc => {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n' toPersist += this.afterSerialization(model.serialize(doc)) + '\n'
}) })
if (toPersist.length === 0) { return callback(null) } if (toPersist.length === 0) return callback(null)
storage.appendFile(self.filename, toPersist, 'utf8', function (err) { storage.appendFile(this.filename, toPersist, 'utf8', err => callback(err))
return callback(err)
})
} }
/** /**
@ -161,39 +156,29 @@ class Persistence {
const data = rawData.split('\n') const data = rawData.split('\n')
const dataById = {} const dataById = {}
const tdata = [] const tdata = []
let i
const indexes = {} const indexes = {}
let corruptItems = -1 let corruptItems = -1
for (i = 0; i < data.length; i += 1) { for (const datum of data) {
let doc
try { try {
doc = model.deserialize(this.beforeDeserialization(data[i])) const doc = model.deserialize(this.beforeDeserialization(datum))
if (doc._id) { if (doc._id) {
if (doc.$$deleted === true) { if (doc.$$deleted === true) delete dataById[doc._id]
delete dataById[doc._id] else dataById[doc._id] = doc
} else { } else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
dataById[doc._id] = doc else if (typeof doc.$$indexRemoved === 'string') delete indexes[doc.$$indexRemoved]
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated
} else if (typeof doc.$$indexRemoved === 'string') {
delete indexes[doc.$$indexRemoved]
}
} catch (e) { } catch (e) {
corruptItems += 1 corruptItems += 1
} }
} }
// A bit lenient on corruption // A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) { if (
throw new Error('More than ' + Math.floor(100 * this.corruptAlertThreshold) + '% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss') data.length > 0 &&
} corruptItems / data.length > this.corruptAlertThreshold
) throw new Error(`More than ${Math.floor(100 * this.corruptAlertThreshold)}% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss`)
Object.keys(dataById).forEach(function (k) { tdata.push(...Object.values(dataById))
tdata.push(dataById[k])
})
return { data: tdata, indexes: indexes } return { data: tdata, indexes: indexes }
} }
@ -206,56 +191,53 @@ class Persistence {
* This means pulling data out of the data file or creating it if it doesn't exist * This means pulling data out of the data file or creating it if it doesn't exist
* Also, all data is persisted right away, which has the effect of compacting the database file * Also, all data is persisted right away, which has the effect of compacting the database file
* This operation is very quick at startup for a big collection (60ms for ~10k docs) * This operation is very quick at startup for a big collection (60ms for ~10k docs)
* @param {Function} cb Optional callback, signature: err * @param {Function} callback Optional callback, signature: err
*/ */
loadDatabase (cb) { loadDatabase (callback = () => {}) {
const callback = cb || function () {} this.db.resetIndexes()
const self = this
self.db.resetIndexes()
// In-memory only datastore // In-memory only datastore
if (self.inMemoryOnly) { return callback(null) } if (this.inMemoryOnly) return callback(null)
async.waterfall([ async.waterfall([
function (cb) { cb => {
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) { Persistence.ensureDirectoryExists(path.dirname(this.filename), err => {
// TODO: handle error // TODO: handle error
// eslint-disable-next-line node/handle-callback-err // eslint-disable-next-line node/handle-callback-err
storage.ensureDatafileIntegrity(self.filename, function (err) { storage.ensureDatafileIntegrity(this.filename, err => {
// TODO: handle error // TODO: handle error
storage.readFile(self.filename, 'utf8', function (err, rawData) { storage.readFile(this.filename, 'utf8', (err, rawData) => {
if (err) { return cb(err) } if (err) return cb(err)
let treatedData let treatedData
try { try {
treatedData = self.treatRawData(rawData) treatedData = this.treatRawData(rawData)
} catch (e) { } catch (e) {
return cb(e) return cb(e)
} }
// Recreate all indexes in the datafile // Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) { Object.keys(treatedData.indexes).forEach(key => {
self.db.indexes[key] = new Index(treatedData.indexes[key]) this.db.indexes[key] = new Index(treatedData.indexes[key])
}) })
// Fill cached database (i.e. all indexes) with data // Fill cached database (i.e. all indexes) with data
try { try {
self.db.resetIndexes(treatedData.data) this.db.resetIndexes(treatedData.data)
} catch (e) { } catch (e) {
self.db.resetIndexes() // Rollback any index which didn't fail this.db.resetIndexes() // Rollback any index which didn't fail
return cb(e) return cb(e)
} }
self.db.persistence.persistCachedDatabase(cb) this.db.persistence.persistCachedDatabase(cb)
}) })
}) })
}) })
} }
], function (err) { ], err => {
if (err) { return callback(err) } if (err) return callback(err)
self.db.executor.processBuffer() this.db.executor.processBuffer()
return callback(null) return callback(null)
}) })
} }
@ -264,9 +246,7 @@ class Persistence {
* Check if a directory stat and create it on the fly if it is not the case * Check if a directory stat and create it on the fly if it is not the case
* cb is optional, signature: err * cb is optional, signature: err
*/ */
static ensureDirectoryExists (dir, cb) { static ensureDirectoryExists (dir, callback = () => {}) {
const callback = cb || function () {}
storage.mkdir(dir, { recursive: true }, err => { callback(err) }) storage.mkdir(dir, { recursive: true }, err => { callback(err) })
} }
@ -277,26 +257,19 @@ class Persistence {
static getNWAppFilename (appName, relativeFilename) { static getNWAppFilename (appName, relativeFilename) {
let home let home
switch (process.platform) { if (process.platform === 'win32' || process.platform === 'win64') {
case 'win32': home = process.env.LOCALAPPDATA || process.env.APPDATA
case 'win64': if (!home) throw new Error('Couldn\'t find the base application data folder')
home = process.env.LOCALAPPDATA || process.env.APPDATA home = path.join(home, appName)
if (!home) { throw new Error('Couldn\'t find the base application data folder') } } else if (process.platform === 'darwin') {
home = path.join(home, appName) home = process.env.HOME
break if (!home) throw new Error('Couldn\'t find the base application data directory')
case 'darwin': home = path.join(home, 'Library', 'Application Support', appName)
home = process.env.HOME } else if (process.platform === 'linux') {
if (!home) { throw new Error('Couldn\'t find the base application data directory') } home = process.env.HOME
home = path.join(home, 'Library', 'Application Support', appName) if (!home) throw new Error('Couldn\'t find the base application data directory')
break home = path.join(home, '.config', appName)
case 'linux': } else throw new Error(`Can't use the Node Webkit relative path for platform ${process.platform}`)
home = process.env.HOME
if (!home) { throw new Error('Couldn\'t find the base application data directory') }
home = path.join(home, '.config', appName)
break
default:
throw new Error('Can\'t use the Node Webkit relative path for platform ' + process.platform)
}
return path.join(home, 'nedb-data', relativeFilename) return path.join(home, 'nedb-data', relativeFilename)
} }

@ -23,11 +23,11 @@ storage.mkdir = fs.mkdir
/** /**
* Explicit name ... * Explicit name ...
*/ */
storage.ensureFileDoesntExist = function (file, callback) { storage.ensureFileDoesntExist = (file, callback) => {
storage.exists(file, function (exists) { storage.exists(file, exists => {
if (!exists) { return callback(null) } if (!exists) return callback(null)
storage.unlink(file, function (err) { return callback(err) }) storage.unlink(file, err => callback(err))
}) })
} }
@ -37,7 +37,7 @@ storage.ensureFileDoesntExist = function (file, callback) {
* @param {Boolean} options.isDir Optional, defaults to false * @param {Boolean} options.isDir Optional, defaults to false
* If options is a string, it is assumed that the flush of the file (not dir) called options was requested * If options is a string, it is assumed that the flush of the file (not dir) called options was requested
*/ */
storage.flushToStorage = function (options, callback) { storage.flushToStorage = (options, callback) => {
let filename let filename
let flags let flags
if (typeof options === 'string') { if (typeof options === 'string') {
@ -50,12 +50,12 @@ storage.flushToStorage = function (options, callback) {
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss // Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens // except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null) } if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) return callback(null)
fs.open(filename, flags, function (err, fd) { fs.open(filename, flags, (err, fd) => {
if (err) { return callback(err) } if (err) return callback(err)
fs.fsync(fd, function (errFS) { fs.fsync(fd, errFS => {
fs.close(fd, function (errC) { fs.close(fd, errC => {
if (errFS || errC) { if (errFS || errC) {
const e = new Error('Failed to flush to storage') const e = new Error('Failed to flush to storage')
e.errorOnFsync = errFS e.errorOnFsync = errFS
@ -73,32 +73,28 @@ storage.flushToStorage = function (options, callback) {
* Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost) * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
* @param {String} filename * @param {String} filename
* @param {String} data * @param {String} data
* @param {Function} cb Optional callback, signature: err * @param {Function} callback Optional callback, signature: err
*/ */
storage.crashSafeWriteFile = function (filename, data, cb) { storage.crashSafeWriteFile = (filename, data, callback = () => {}) => {
const callback = cb || function () {}
const tempFilename = filename + '~' const tempFilename = filename + '~'
async.waterfall([ async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }), async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }),
function (cb) { cb => {
storage.exists(filename, function (exists) { storage.exists(filename, exists => {
if (exists) { if (exists) storage.flushToStorage(filename, err => cb(err))
storage.flushToStorage(filename, function (err) { return cb(err) }) else return cb()
} else {
return cb()
}
}) })
}, },
function (cb) { cb => {
storage.writeFile(tempFilename, data, function (err) { return cb(err) }) storage.writeFile(tempFilename, data, err => cb(err))
}, },
async.apply(storage.flushToStorage, tempFilename), async.apply(storage.flushToStorage, tempFilename),
function (cb) { cb => {
storage.rename(tempFilename, filename, function (err) { return cb(err) }) storage.rename(tempFilename, filename, err => cb(err))
}, },
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true }) async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err) }) ], err => callback(err))
} }
/** /**
@ -106,21 +102,19 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
* @param {String} filename * @param {String} filename
* @param {Function} callback signature: err * @param {Function} callback signature: err
*/ */
storage.ensureDatafileIntegrity = function (filename, callback) { storage.ensureDatafileIntegrity = (filename, callback) => {
const tempFilename = filename + '~' const tempFilename = filename + '~'
storage.exists(filename, function (filenameExists) { storage.exists(filename, filenameExists => {
// Write was successful // Write was successful
if (filenameExists) { return callback(null) } if (filenameExists) return callback(null)
storage.exists(tempFilename, function (oldFilenameExists) { storage.exists(tempFilename, oldFilenameExists => {
// New database // New database
if (!oldFilenameExists) { if (!oldFilenameExists) return storage.writeFile(filename, '', 'utf8', err => { callback(err) })
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err) })
}
// Write failed, use old version // Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err) }) storage.rename(tempFilename, filename, err => callback(err))
}) })
}) })
} }

Loading…
Cancel
Save