Finished doc insertion load and find with index

benchmarks/commonUtilities.js

@@ -53,6 +53,7 @@ module.exports.getRandomArray = getRandomArray;
  */
 module.exports.insertDocs = function (d, n, profiler, cb) {
   var beg = new Date()
+    , order = getRandomArray(n)
     ;
 
   profiler.step('Begin inserting ' + n + ' docs');
@@ -64,7 +65,7 @@ module.exports.insertDocs = function (d, n, profiler, cb) {
       return cb();
     }
 
-    d.insert({ docNumber: i }, function (err) {
+    d.insert({ docNumber: order[i] }, function (err) {
       executeAsap(function () {
         runFrom(i + 1);
       });

lib/datastore.js

@@ -1,3 +1,6 @@
+/**
+ * TODO: make ensureIndex work whenever it is called, not just right after loadDatabase
+ */
 var fs = require('fs')
   , path = require('path')
   , customUtils = require('./customUtils')
@@ -5,6 +8,8 @@ var fs = require('fs')
   , async = require('async')
   , Executor = require('./executor')
   , Index = require('./indexes')
+  , util = require('util')
+  , _ = require('underscore')
   ;
 
 
@@ -26,6 +31,7 @@ function Datastore (filename) {
 
 
 /**
+ * TODO: make it work whenever ensureIndex is called
  * Ensure an index is kept for this field. Same parameters as lib/indexes
  * For now this function is synchronous, we need to test how much time it takes
  * @param {String} options.fieldName
@@ -47,6 +53,48 @@ Datastore.prototype.ensureIndex = function (options) {
 };
 
 
+/**
+ * Add one or several document(s) to all indexes
+ */
+Datastore.prototype.addToIndexes = function (doc) {
+  var self = this;
+
+  Object.keys(this.indexes).forEach(function (i) {
+    self.indexes[i].insert(doc);
+  });
+};
+
+
+/**
+ * Return the list of candidates for a given query
+ * Very crude implementation for now, we return the candidates given by the first usable index if any
+ * Also indexes can only be used for direct matches (no $lt, $gt or array yet)
+ * This still gives a huge performance boost to finds (800x on a collection with 10k documents)
+ */
+Datastore.prototype.getCandidates = function (query) {
+  var indexNames = Object.keys(this.indexes)
+    , usableQueryKeys;
+
+  if (indexNames.length === 0) { return this.data; }   // No index defined, no specific candidate
+
+  // Usable query keys are the ones corresponding to a basic query (no use of $operators or arrays)
+  usableQueryKeys = [];
+  Object.keys(query).forEach(function (k) {
+    if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.isDate(query[k]) || query[k] === null) {
+      usableQueryKeys.push(k);
+    }
+  });
+
+  usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
+
+  if (usableQueryKeys.length > 0) {
+    return this.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]);
+  } else {
+    return this.data;
+  }
+};
+
+
 /**
  * Load the database
  * This means pulling data out of the data file or creating it if it doesn't exist
@@ -164,6 +212,7 @@ Datastore.prototype._insert = function (newDoc, cb) {
 
     var insertedDoc = model.deserialize(persistableNewDoc);
     self.data.push(insertedDoc);
+    self.addToIndexes(insertedDoc);
     self.datafileSize += 1;
     return callback(null, model.deepCopy(insertedDoc));
   });
@@ -181,13 +230,14 @@ Datastore.prototype.insert = function () {
 Datastore.prototype.find = function (query, callback) {
   var res = []
     , self = this
+    , candidates = this.getCandidates(query)
     , i
     ;
 
   try {
-    for (i = 0; i < self.data.length; i += 1) {
+    for (i = 0; i < candidates.length; i += 1) {
-      if (model.match(self.data[i], query)) {
+      if (model.match(candidates[i], query)) {
-        res.push(model.deepCopy(self.data[i]));
+        res.push(model.deepCopy(candidates[i]));
       }
     }
   } catch (err) {
@@ -365,6 +415,4 @@ Datastore.prototype.remove = function () {
 
 
 
-
-
 module.exports = Datastore;