TTL indexes tested

lib/datastore.js

@@ -69,6 +69,7 @@ function Datastore (options) {
   // binary is always well-balanced
   this.indexes = {};
   this.indexes._id = new Index({ fieldName: '_id', unique: true });
+  this.ttlIndexes = {};
 
   // Queue a load of the database right away and call the onload handler
   // By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
@@ -115,7 +116,7 @@ Datastore.prototype.resetIndexes = function (newData) {
  * @param {String} options.fieldName
  * @param {Boolean} options.unique
  * @param {Boolean} options.sparse
- * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index
+ * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index (only works on Date fields, not arrays of Date)
  * @param {Function} cb Optional callback, signature: err
  */
 Datastore.prototype.ensureIndex = function (options, cb) {
@@ -132,6 +133,7 @@ Datastore.prototype.ensureIndex = function (options, cb) {
   if (this.indexes[options.fieldName]) { return callback(null); }
 
   this.indexes[options.fieldName] = new Index(options);
+  if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; }   // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
 
   try {
     this.indexes[options.fieldName].insert(this.getAllData());
@@ -247,13 +249,18 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) {
  * Returned candidates will be scanned to find and remove all expired documents
  *
  * @param {Query} query
+ * @param {Boolean} dontExpireStaleDocs Optional, defaults to false, if true don't remove stale docs. Useful for the remove function which shouldn't be impacted by expirations
  * @param {Function} callback Signature err, docs
  */
-Datastore.prototype.getCandidates = function (query, callback) {
+Datastore.prototype.getCandidates = function (query, dontExpireStaleDocs, callback) {
   var indexNames = Object.keys(this.indexes)
     , self = this
     , usableQueryKeys;
 
+  if (typeof dontExpireStaleDocs === 'function') {
+    callback = dontExpireStaleDocs;
+    dontExpireStaleDocs = false;
+  }
 
   async.waterfall([
   // STEP 1: get candidates list by checking indexes from most to least frequent usecase
@@ -299,7 +306,28 @@ Datastore.prototype.getCandidates = function (query, callback) {
   }
   // STEP 2: remove all expired documents
   , function (docs) {
-    return callback(null, docs);
+    if (dontExpireStaleDocs) { return callback(null, docs); }
+
+    var expiredDocsIds = [], validDocs = [], ttlIndexesFieldNames = Object.keys(self.ttlIndexes);
+
+    docs.forEach(function (doc) {
+      var valid = true;
+      ttlIndexesFieldNames.forEach(function (i) {
+        if (doc[i] !== undefined && util.isDate(doc[i]) && Date.now() > doc[i].getTime() + self.ttlIndexes[i] * 1000) {
+          valid = false;
+        }
+      });
+      if (valid) { validDocs.push(doc); } else { expiredDocsIds.push(doc._id); }
+    });
+
+    async.eachSeries(expiredDocsIds, function (_id, cb) {
+      self._remove({ _id: _id }, {}, function (err) {
+        if (err) { return callback(err); }
+        return cb();
+      });
+    }, function (err) {
+      return callback(null, validDocs);
+    });
   }]);
 };
 
@@ -629,7 +657,7 @@ Datastore.prototype._remove = function (query, options, cb) {
   callback = cb || function () {};
   multi = options.multi !== undefined ? options.multi : false;
 
-  this.getCandidates(query, function (err, candidates) {
+  this.getCandidates(query, true, function (err, candidates) {
     if (err) { return callback(err); }
 
     try {

lib/indexes.js

@@ -32,13 +32,11 @@ function projectForUnique (elt) {
  * @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields)
  * @param {Boolean} options.unique Optional, enforce a unique constraint (default: false)
  * @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false)
- * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index
  */
 function Index (options) {
   this.fieldName = options.fieldName;
   this.unique = options.unique || false;
   this.sparse = options.sparse || false;
-  if (options.expireAfterSeconds !== undefined) { this.expireAfterSeconds = options.expireAfterSeconds; }
 
   this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality };
 

test/db.test.js

@@ -424,7 +424,7 @@ describe('Database', function () {
   });   // ==== End of 'Insert' ==== //
 
 
-  describe('#getCandidates', function () {
+  describe.only('#getCandidates', function () {
 
     it('Can use an index to get docs with a basic match', function (done) {
       d.ensureIndex({ fieldName: 'tf' }, function (err) {
@@ -526,6 +526,67 @@ describe('Database', function () {
       });
     });
 
+    it("Can set a TTL index that expires documents", function (done) {
+      d.ensureIndex({ fieldName: 'exp', expireAfterSeconds: 0.2 }, function () {
+        d.insert({ hello: 'world', exp: new Date() }, function () {
+          setTimeout(function () {
+            d.findOne({}, function (err, doc) {
+              assert.isNull(err);
+              doc.hello.should.equal('world');
+
+              setTimeout(function () {
+                d.findOne({}, function (err, doc) {
+                  assert.isNull(err);
+                  assert.isNull(doc);
+
+                  d.findOne({}, function (err, doc) {
+                    assert.isNull(err);
+                    assert.isNull(doc);
+
+                    done();
+                  });
+                });
+              }, 101);
+            });
+          }, 100);
+        });
+      });
+    });
+
+    it("TTL indexes can expire multiple documents and only what needs to be expired", function (done) {
+      d.ensureIndex({ fieldName: 'exp', expireAfterSeconds: 0.2 }, function () {
+        d.insert({ hello: 'world1', exp: new Date() }, function () {
+          d.insert({ hello: 'world2', exp: new Date() }, function () {
+            d.insert({ hello: 'world3', exp: new Date((new Date()).getTime() + 100) }, function () {
+              setTimeout(function () {
+                d.find({}, function (err, docs) {
+                  assert.isNull(err);
+                  docs.length.should.equal(3);
+
+                  setTimeout(function () {
+                    d.find({}, function (err, docs) {
+                      assert.isNull(err);
+                      docs.length.should.equal(1);
+                      docs[0].hello.should.equal('world3');
+
+                      setTimeout(function () {
+                        d.find({}, function (err, docs) {
+                          assert.isNull(err);
+                          docs.length.should.equal(0);
+
+                          done();
+                        });
+                      }, 101);
+                    });
+                  }, 101);
+                });
+              }, 100);
+            });
+          });
+        });
+      });
+    });
+
   });   // ==== End of '#getCandidates' ==== //