getCandidates is now async

lib/cursor.js

@@ -103,24 +103,34 @@ Cursor.prototype.project = function (candidates) {
  *
  * @param {Function} callback - Signature: err, results
  */
-Cursor.prototype._exec = function(callback) {
+Cursor.prototype._exec = function(_callback) {
-  var candidates = this.db.getCandidates(this.query)
+  var res = [], added = 0, skipped = 0, self = this
-    , res = [], added = 0, skipped = 0, self = this
     , error = null
     , i, keys, key
     ;
 
+  function callback (error, res) {
+    if (self.execFn) {
+      return self.execFn(error, res, _callback);
+    } else {
+      return _callback(error, res);
+    }
+  }
+
+  this.db.getCandidates(this.query, function (err, candidates) {
+    if (err) { return callback(err); }
+
     try {
       for (i = 0; i < candidates.length; i += 1) {
-      if (model.match(candidates[i], this.query)) {
+        if (model.match(candidates[i], self.query)) {
           // If a sort is defined, wait for the results to be sorted before applying limit and skip
-        if (!this._sort) {
+          if (!self._sort) {
-          if (this._skip && this._skip > skipped) {
+            if (self._skip && self._skip > skipped) {
               skipped += 1;
             } else {
               res.push(candidates[i]);
               added += 1;
-            if (this._limit && this._limit <= added) { break; }
+              if (self._limit && self._limit <= added) { break; }
             }
           } else {
             res.push(candidates[i]);
@@ -132,8 +142,8 @@ Cursor.prototype._exec = function(callback) {
     }
 
     // Apply all sorts
-  if (this._sort) {
+    if (self._sort) {
-    keys = Object.keys(this._sort);
+      keys = Object.keys(self._sort);
 
       // Sorting
       var criteria = [];
@@ -154,25 +164,22 @@ Cursor.prototype._exec = function(callback) {
       });
 
       // Applying limit and skip
-    var limit = this._limit || res.length
+      var limit = self._limit || res.length
-      , skip = this._skip || 0;
+        , skip = self._skip || 0;
 
       res = res.slice(skip, skip + limit);
     }
 
     // Apply projection
     try {
-    res = this.project(res);
+      res = self.project(res);
     } catch (e) {
       error = e;
       res = undefined;
     }
 
-  if (this.execFn) {
-    return this.execFn(error, res, callback);
-  } else {
     return callback(error, res);
-  }
+  });
 };
 
 Cursor.prototype.exec = function () {

lib/datastore.js

@@ -115,6 +115,7 @@ Datastore.prototype.resetIndexes = function (newData) {
  * @param {String} options.fieldName
  * @param {Boolean} options.unique
  * @param {Boolean} options.sparse
+ * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index
  * @param {Function} cb Optional callback, signature: err
  */
 Datastore.prototype.ensureIndex = function (options, cb) {
@@ -243,12 +244,20 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) {
  * One way to make it better would be to enable the use of multiple indexes if the first usable index
  * returns too much data. I may do it in the future.
  *
- * TODO: needs to be moved to the Cursor module
+ * Returned candidates will be scanned to find and remove all expired documents
+ *
+ * @param {Query} query
+ * @param {Function} callback Signature err, docs
  */
-Datastore.prototype.getCandidates = function (query) {
+Datastore.prototype.getCandidates = function (query, callback) {
   var indexNames = Object.keys(this.indexes)
+    , self = this
     , usableQueryKeys;
 
+
+  async.waterfall([
+  // STEP 1: get candidates list by checking indexes from most to least frequent usecase
+  function (cb) {
     // For a basic match
     usableQueryKeys = [];
     Object.keys(query).forEach(function (k) {
@@ -258,7 +267,7 @@ Datastore.prototype.getCandidates = function (query) {
     });
     usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
     if (usableQueryKeys.length > 0) {
-    return this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]);
+      return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]));
     }
 
     // For a $in match
@@ -270,7 +279,7 @@ Datastore.prototype.getCandidates = function (query) {
     });
     usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
     if (usableQueryKeys.length > 0) {
-    return this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in);
+      return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in));
     }
 
     // For a comparison match
@@ -282,11 +291,16 @@ Datastore.prototype.getCandidates = function (query) {
     });
     usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
     if (usableQueryKeys.length > 0) {
-    return this.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]);
+      return cb(null, self.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]));
     }
 
     // By default, return all the DB data
-  return this.getAllData();
+    return cb(null, self.getAllData());
+  }
+  // STEP 2: remove all expired documents
+  , function (docs) {
+    return callback(null, docs);
+  }]);
 };
 
 
@@ -548,10 +562,10 @@ Datastore.prototype._update = function (query, updateQuery, options, cb) {
     });
   }
   , function () {   // Perform the update
-    var modifiedDoc
+    var modifiedDoc , modifications = [];
-      , candidates = self.getCandidates(query)
+
-      , modifications = []
+    self.getCandidates(query, function (err, candidates) {
-      ;
+      if (err) { return callback(err); }
 
       // Preparing update (if an error is thrown here neither the datafile nor
       // the in-memory indexes are affected)
@@ -587,9 +601,10 @@ Datastore.prototype._update = function (query, updateQuery, options, cb) {
           return callback(null, numReplaced, updatedDocsDC);
         }
       });
-  }
+    });
-  ]);
+  }]);
 };
+
 Datastore.prototype.update = function () {
   this.executor.push({ this: this, fn: this._update, arguments: arguments });
 };
@@ -607,17 +622,16 @@ Datastore.prototype.update = function () {
  */
 Datastore.prototype._remove = function (query, options, cb) {
   var callback
-    , self = this
+    , self = this, numRemoved = 0, removedDocs = [], multi
-    , numRemoved = 0
-    , multi
-    , removedDocs = []
-    , candidates = this.getCandidates(query)
     ;
 
   if (typeof options === 'function') { cb = options; options = {}; }
   callback = cb || function () {};
   multi = options.multi !== undefined ? options.multi : false;
 
+  this.getCandidates(query, function (err, candidates) {
+    if (err) { return callback(err); }
+
     try {
       candidates.forEach(function (d) {
         if (model.match(d, query) && (multi || numRemoved === 0)) {
@@ -632,7 +646,9 @@ Datastore.prototype._remove = function (query, options, cb) {
       if (err) { return callback(err); }
       return callback(null, numRemoved);
     });
+  });
 };
+
 Datastore.prototype.remove = function () {
   this.executor.push({ this: this, fn: this._remove, arguments: arguments });
 };

lib/indexes.js

@@ -32,11 +32,13 @@ function projectForUnique (elt) {
  * @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields)
  * @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
  * @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
+ * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index
  */
 function Index (options) {
   this.fieldName = options.fieldName;
   this.unique = options.unique || false;
   this.sparse = options.sparse || false;
+  if (options.expireAfterSeconds !== undefined) { this.expireAfterSeconds = options.expireAfterSeconds; }
 
   this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
 

test/db.test.js

@@ -432,8 +432,8 @@ describe('Database', function () {
           d.insert({ tf: 6 }, function () {
             d.insert({ tf: 4, an: 'other' }, function (err, _doc2) {
               d.insert({ tf: 9 }, function () {
-                var data = d.getCandidates({ r: 6, tf: 4 })
+                d.getCandidates({ r: 6, tf: 4 }, function (err, data) {
-                  , doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
+                  var doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
                     , doc2 = _.find(data, function (d) { return d._id === _doc2._id; })
                     ;
 
@@ -448,6 +448,7 @@ describe('Database', function () {
           });
         });
       });
+    });
 
     it('Can use an index to get docs with a $in match', function (done) {
       d.ensureIndex({ fieldName: 'tf' }, function (err) {
@@ -455,8 +456,8 @@ describe('Database', function () {
           d.insert({ tf: 6 }, function (err, _doc1) {
             d.insert({ tf: 4, an: 'other' }, function (err) {
               d.insert({ tf: 9 }, function (err, _doc2) {
-                var data = d.getCandidates({ r: 6, tf: { $in: [6, 9, 5] } })
+                d.getCandidates({ r: 6, tf: { $in: [6, 9, 5] } }, function (err, data) {
-                  , doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
+                  var doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
                     , doc2 = _.find(data, function (d) { return d._id === _doc2._id; })
                     ;
 
@@ -471,6 +472,7 @@ describe('Database', function () {
           });
         });
       });
+    });
 
     it('If no index can be used, return the whole database', function (done) {
       d.ensureIndex({ fieldName: 'tf' }, function (err) {
@@ -478,8 +480,8 @@ describe('Database', function () {
           d.insert({ tf: 6 }, function (err, _doc2) {
             d.insert({ tf: 4, an: 'other' }, function (err, _doc3) {
               d.insert({ tf: 9 }, function (err, _doc4) {
-                var data = d.getCandidates({ r: 6, notf: { $in: [6, 9, 5] } })
+                d.getCandidates({ r: 6, notf: { $in: [6, 9, 5] } }, function (err, data) {
-                  , doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
+                  var doc1 = _.find(data, function (d) { return d._id === _doc1._id; })
                     , doc2 = _.find(data, function (d) { return d._id === _doc2._id; })
                     , doc3 = _.find(data, function (d) { return d._id === _doc3._id; })
                     , doc4 = _.find(data, function (d) { return d._id === _doc4._id; })
@@ -498,6 +500,7 @@ describe('Database', function () {
           });
         });
       });
+    });
 
     it('Can use indexes for comparison matches', function (done) {
       d.ensureIndex({ fieldName: 'tf' }, function (err) {
@@ -505,8 +508,8 @@ describe('Database', function () {
           d.insert({ tf: 6 }, function (err, _doc2) {
             d.insert({ tf: 4, an: 'other' }, function (err, _doc3) {
               d.insert({ tf: 9 }, function (err, _doc4) {
-                var data = d.getCandidates({ r: 6, tf: { $lte: 9, $gte: 6 } })
+                d.getCandidates({ r: 6, tf: { $lte: 9, $gte: 6 } }, function (err, data) {
-                  , doc2 = _.find(data, function (d) { return d._id === _doc2._id; })
+                  var doc2 = _.find(data, function (d) { return d._id === _doc2._id; })
                     , doc4 = _.find(data, function (d) { return d._id === _doc4._id; })
                     ;
 
@@ -521,6 +524,7 @@ describe('Database', function () {
           });
         });
       });
+    });
 
   });   // ==== End of '#getCandidates' ==== //
 
@@ -2649,11 +2653,12 @@ describe('Database', function () {
     it('Results of getMatching should never contain duplicates', function (done) {
       d.ensureIndex({ fieldName: 'bad' });
       d.insert({ bad: ['a', 'b'] }, function () {
-        var res = d.getCandidates({ bad: { $in: ['a', 'b'] } });
+        d.getCandidates({ bad: { $in: ['a', 'b'] } }, function (err, res) {
           res.length.should.equal(1);
           done();
         });
       });
+    });
 
   });   // ==== End of 'Using indexes' ==== //