From 7c25401f1640714267177cbd6a76e2bd4f96a5e2 Mon Sep 17 00:00:00 2001 From: Louis Chatriot Date: Mon, 18 Jan 2016 18:35:32 -0800 Subject: [PATCH] TTL indexes tested --- lib/datastore.js | 36 ++++++++++++++++++++++++--- lib/indexes.js | 2 -- test/db.test.js | 63 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 7 deletions(-) diff --git a/lib/datastore.js b/lib/datastore.js index 6ff9c60..06238f4 100755 --- a/lib/datastore.js +++ b/lib/datastore.js @@ -69,6 +69,7 @@ function Datastore (options) { // binary is always well-balanced this.indexes = {}; this.indexes._id = new Index({ fieldName: '_id', unique: true }); + this.ttlIndexes = {}; // Queue a load of the database right away and call the onload handler // By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception @@ -115,7 +116,7 @@ Datastore.prototype.resetIndexes = function (newData) { * @param {String} options.fieldName * @param {Boolean} options.unique * @param {Boolean} options.sparse - * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index + * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index (only works on Date fields, not arrays of Date) * @param {Function} cb Optional callback, signature: err */ Datastore.prototype.ensureIndex = function (options, cb) { @@ -132,6 +133,7 @@ Datastore.prototype.ensureIndex = function (options, cb) { if (this.indexes[options.fieldName]) { return callback(null); } this.indexes[options.fieldName] = new Index(options); + if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; } // With this implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here try { this.indexes[options.fieldName].insert(this.getAllData()); @@ -247,13 +249,18 @@ Datastore.prototype.updateIndexes = function (oldDoc, newDoc) { * Returned candidates will be scanned to find and remove all expired documents * * @param {Query} query + * @param {Boolean} dontExpireStaleDocs Optional, defaults to false, if true don't remove stale docs. Useful for the remove function which shouldn't be impacted by expirations * @param {Function} callback Signature err, docs */ -Datastore.prototype.getCandidates = function (query, callback) { +Datastore.prototype.getCandidates = function (query, dontExpireStaleDocs, callback) { var indexNames = Object.keys(this.indexes) , self = this , usableQueryKeys; + if (typeof dontExpireStaleDocs === 'function') { + callback = dontExpireStaleDocs; + dontExpireStaleDocs = false; + } async.waterfall([ // STEP 1: get candidates list by checking indexes from most to least frequent usecase @@ -299,7 +306,28 @@ Datastore.prototype.getCandidates = function (query, callback) { } // STEP 2: remove all expired documents , function (docs) { - return callback(null, docs); + if (dontExpireStaleDocs) { return callback(null, docs); } + + var expiredDocsIds = [], validDocs = [], ttlIndexesFieldNames = Object.keys(self.ttlIndexes); + + docs.forEach(function (doc) { + var valid = true; + ttlIndexesFieldNames.forEach(function (i) { + if (doc[i] !== undefined && util.isDate(doc[i]) && Date.now() > doc[i].getTime() + self.ttlIndexes[i] * 1000) { + valid = false; + } + }); + if (valid) { validDocs.push(doc); } else { expiredDocsIds.push(doc._id); } + }); + + async.eachSeries(expiredDocsIds, function (_id, cb) { + self._remove({ _id: _id }, {}, function (err) { + if (err) { return callback(err); } + return cb(); + }); + }, function (err) { + return callback(null, validDocs); + }); }]); }; @@ -629,7 +657,7 @@ Datastore.prototype._remove = function (query, options, cb) { callback = cb || function () {}; multi = options.multi !== undefined ? options.multi : false; - this.getCandidates(query, function (err, candidates) { + this.getCandidates(query, true, function (err, candidates) { if (err) { return callback(err); } try { diff --git a/lib/indexes.js b/lib/indexes.js index 2b43a6b..391837b 100755 --- a/lib/indexes.js +++ b/lib/indexes.js @@ -32,13 +32,11 @@ function projectForUnique (elt) { * @param {String} options.fieldName On which field should the index apply (can use dot notation to index on sub fields) * @param {Boolean} options.unique Optional, enforce a unique constraint (default: false) * @param {Boolean} options.sparse Optional, allow a sparse index (we can have documents for which fieldName is undefined) (default: false) - * @param {Number} options.expireAfterSeconds - Optional, if set this index becomes a TTL index */ function Index (options) { this.fieldName = options.fieldName; this.unique = options.unique || false; this.sparse = options.sparse || false; - if (options.expireAfterSeconds !== undefined) { this.expireAfterSeconds = options.expireAfterSeconds; } this.treeOptions = { unique: this.unique, compareKeys: model.compareThings, checkValueEquality: checkValueEquality }; diff --git a/test/db.test.js b/test/db.test.js index 12db34e..837827d 100755 --- a/test/db.test.js +++ b/test/db.test.js @@ -424,7 +424,7 @@ describe('Database', function () { }); // ==== End of 'Insert' ==== // - describe('#getCandidates', function () { + describe.only('#getCandidates', function () { it('Can use an index to get docs with a basic match', function (done) { d.ensureIndex({ fieldName: 'tf' }, function (err) { @@ -526,6 +526,67 @@ describe('Database', function () { }); }); + it("Can set a TTL index that expires documents", function (done) { + d.ensureIndex({ fieldName: 'exp', expireAfterSeconds: 0.2 }, function () { + d.insert({ hello: 'world', exp: new Date() }, function () { + setTimeout(function () { + d.findOne({}, function (err, doc) { + assert.isNull(err); + doc.hello.should.equal('world'); + + setTimeout(function () { + d.findOne({}, function (err, doc) { + assert.isNull(err); + assert.isNull(doc); + + d.findOne({}, function (err, doc) { + assert.isNull(err); + assert.isNull(doc); + + done(); + }); + }); + }, 101); + }); + }, 100); + }); + }); + }); + + it("TTL indexes can expire multiple documents and only what needs to be expired", function (done) { + d.ensureIndex({ fieldName: 'exp', expireAfterSeconds: 0.2 }, function () { + d.insert({ hello: 'world1', exp: new Date() }, function () { + d.insert({ hello: 'world2', exp: new Date() }, function () { + d.insert({ hello: 'world3', exp: new Date((new Date()).getTime() + 100) }, function () { + setTimeout(function () { + d.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(3); + + setTimeout(function () { + d.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(1); + docs[0].hello.should.equal('world3'); + + setTimeout(function () { + d.find({}, function (err, docs) { + assert.isNull(err); + docs.length.should.equal(0); + + done(); + }); + }, 101); + }); + }, 101); + }); + }, 100); + }); + }); + }); + }); + }); + }); // ==== End of '#getCandidates' ==== //