TheDude
/
nedb
mirror of https://github.com/seald/nedb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Load tests

Browse Source
pull/2/head
Louis Chatriot 9 years ago
parent da2765a87f
commit a12ec8b89e
5 changed files with 109 additions and 47 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      browser-version/test/async.js
  2. 15
      browser-version/test/testLoad.html
  3. 45
      browser-version/test/testLoad.js
  4. 24
      lib/storage.js
  5. 70
      test/db.test.js

2
browser-version/test/async.js
View File

File diff suppressed because one or more lines are too long

15
browser-version/test/testLoad.html
Unescape View File

@ -0,0 +1,15 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Test NeDB persistence load in the browser</title>
<link rel="stylesheet" href="mocha.css">
</head>
<body>
<div id="results"></div>
<script src="./async.js"></script>
<script src="../out/nedb.js"></script>
<script src="./testLoad.js"></script>
</body>
</html>

45
browser-version/test/testLoad.js
Unescape View File

@ -0,0 +1,45 @@
console.log('BEGINNING');
var N = 4000
, db = new Nedb({ filename: 'loadTest', autoload: true })
, t, i
, sample = JSON.stringify({ data: Math.random(), _id: Math.random() });
;
// Some inserts in sequence, using the default storage mechanism (IndexedDB in my case)
function someInserts (sn, N, callback) {
var i = 0, beg = Date.now();
async.whilst( function () { return i < N; }
, function (_cb) {
db.insert({ data: Math.random() }, function (err) { i += 1; return _cb(err); });
}
, function (err) {
console.log("Inserts, series " + sn + " " + (Date.now() - beg));
return callback(err);
});
}
// Manually updating the localStorage
function someLS (sn, N, callback) {
var i = 0, beg = Date.now();
for (i = 0; i < N; i += 1) {
localStorage.setItem('loadTestLS', localStorage.getItem('loadTestLS') + sample);
}
console.log("localStorage, series " + sn + " " + (Date.now() - beg));
return callback();
}
localStorage.setItem('loadTestLS', '');
async.waterfall([
function (cb) { db.remove({}, { multi: true }, function (err) { return cb(err); }); }
//, async.apply(someInserts, "#1", N) // N=5000, 141s
//, async.apply(someInserts, "#2", N) // N=5000, 208s
//, async.apply(someInserts, "#3", N) // N=5000, 281s
//, async.apply(someInserts, "#4", N) // N=5000, 350s
, async.apply(someLS, "#1", N) // N=4000, 2.5s
, async.apply(someLS, "#2", N) // N=4000, 8.0s
, async.apply(someLS, "#3", N) // N=4000, 26.5s
, async.apply(someLS, "#4", N) // N=4000, 47.8s then crash (with N=5000 crash happens on second pass)
]);

24
lib/storage.js
Unescape View File

@ -38,13 +38,15 @@ storage.ensureFileDoesntExist = function (file, callback) {
* Flush data in OS buffer to storage if corresponding option is set
*/
storage.flushToStorage = function (filename, callback) {
// Hate having to do this but I can't seem to make fs.fsync work on Windows, and still don't even know whether fsync can be used on Windows.
// See this issue: https://github.com/nodejs/node/issues/3879
if (process.platform === 'win32' || process.platform === 'win64') { return callback(); }
fs.open(filename, 'r', function (err, fd) {
fs.open(filename, 'w', function (err, fd) {
if (err) { return callback(err); }
fs.fsync(fd, function (err) { return callback(err); });
fs.fsync(fd, function (err) {
return callback(err);
if (err) { return callback(err); }
fs.close(fd, function (err) { return callback(err); });
});
});
};
@ -67,22 +69,20 @@ storage.crashSafeWriteFile = function (filename, data, cb) {
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.rename(filename, oldFilename, function (err) {
if (err) { return cb(err); }
storage.flushToStorage(oldFilename, cb); // Ensure filename doesn't exist due to rename atomicity
});
storage.rename(filename, oldFilename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
//, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, filename)
//, async.apply(storage.flushToStorage, filename)
, async.apply(storage.ensureFileDoesntExist, oldFilename)
], function (err) { return callback(err); })
};

70
test/db.test.js
Unescape View File

@ -2392,41 +2392,41 @@ describe('Database', function () {
});
}); // ==== End of 'Updating indexes upon document remove' ==== //
describe('Persisting indexes', function () {
it('Indexes are persisted to a separate file and recreated upon reload', function (done) {
it.only('Indexes are persisted to a separate file and recreated upon reload', function (done) {
var persDb = "workspace/persistIndexes.db"
, db
;
if (fs.existsSync(persDb)) { fs.writeFileSync(persDb, '', 'utf8'); }
db = new Datastore({ filename: persDb, autoload: true });
Object.keys(db.indexes).length.should.equal(1);
Object.keys(db.indexes)[0].should.equal("_id");
db.insert({ planet: "Earth" }, function (err) {
assert.isNull(err);
db.insert({ planet: "Mars" }, function (err) {
assert.isNull(err);
db.ensureIndex({ fieldName: "planet" }, function (err) {
Object.keys(db.indexes).length.should.equal(2);
Object.keys(db.indexes)[0].should.equal("_id");
Object.keys(db.indexes)[1].should.equal("planet");
Object.keys(db.indexes)[1].should.equal("planet");
db.indexes._id.getAll().length.should.equal(2);
db.indexes.planet.getAll().length.should.equal(2);
db.indexes.planet.fieldName.should.equal("planet");
// After a reload the indexes are recreated
db = new Datastore({ filename: persDb });
db.loadDatabase(function (err) {
assert.isNull(err);
Object.keys(db.indexes).length.should.equal(2);
Object.keys(db.indexes)[0].should.equal("_id");
Object.keys(db.indexes)[1].should.equal("planet");
Object.keys(db.indexes)[1].should.equal("planet");
db.indexes._id.getAll().length.should.equal(2);
db.indexes.planet.getAll().length.should.equal(2);
db.indexes.planet.fieldName.should.equal("planet");
@ -2437,44 +2437,44 @@ describe('Database', function () {
assert.isNull(err);
Object.keys(db.indexes).length.should.equal(2);
Object.keys(db.indexes)[0].should.equal("_id");
Object.keys(db.indexes)[1].should.equal("planet");
Object.keys(db.indexes)[1].should.equal("planet");
db.indexes._id.getAll().length.should.equal(2);
db.indexes.planet.getAll().length.should.equal(2);
db.indexes.planet.fieldName.should.equal("planet");
done();
done();
});
});
});
});
});
});
it('Indexes are persisted with their options and recreated even if some db operation happen between loads', function (done) {
var persDb = "workspace/persistIndexes.db"
, db
;
;
if (fs.existsSync(persDb)) { fs.writeFileSync(persDb, '', 'utf8'); }
db = new Datastore({ filename: persDb, autoload: true });
Object.keys(db.indexes).length.should.equal(1);
Object.keys(db.indexes)[0].should.equal("_id");
db.insert({ planet: "Earth" }, function (err) {
assert.isNull(err);
db.insert({ planet: "Mars" }, function (err) {
assert.isNull(err);
db.ensureIndex({ fieldName: "planet", unique: true, sparse: false }, function (err) {
Object.keys(db.indexes).length.should.equal(2);
Object.keys(db.indexes)[0].should.equal("_id");
Object.keys(db.indexes)[1].should.equal("planet");
Object.keys(db.indexes)[1].should.equal("planet");
db.indexes._id.getAll().length.should.equal(2);
db.indexes.planet.getAll().length.should.equal(2);
db.indexes.planet.unique.should.equal(true);
db.indexes.planet.sparse.should.equal(false);
db.insert({ planet: "Jupiter" }, function (err) {
assert.isNull(err);
@ -2484,12 +2484,12 @@ describe('Database', function () {
assert.isNull(err);
Object.keys(db.indexes).length.should.equal(2);
Object.keys(db.indexes)[0].should.equal("_id");
Object.keys(db.indexes)[1].should.equal("planet");
Object.keys(db.indexes)[1].should.equal("planet");
db.indexes._id.getAll().length.should.equal(3);
db.indexes.planet.getAll().length.should.equal(3);
db.indexes.planet.unique.should.equal(true);
db.indexes.planet.sparse.should.equal(false);
db.ensureIndex({ fieldName: 'bloup', unique: false, sparse: true }, function (err) {
assert.isNull(err);
Object.keys(db.indexes).length.should.equal(3);
@ -2500,9 +2500,9 @@ describe('Database', function () {
db.indexes.planet.getAll().length.should.equal(3);
db.indexes.bloup.getAll().length.should.equal(0);
db.indexes.planet.unique.should.equal(true);
db.indexes.planet.sparse.should.equal(false);
db.indexes.planet.sparse.should.equal(false);
db.indexes.bloup.unique.should.equal(false);
db.indexes.bloup.sparse.should.equal(true);
db.indexes.bloup.sparse.should.equal(true);
// After another reload the indexes are still there (i.e. they are preserved during autocompaction)
db = new Datastore({ filename: persDb });
@ -2519,8 +2519,8 @@ describe('Database', function () {
db.indexes.planet.sparse.should.equal(false);
db.indexes.bloup.unique.should.equal(false);
db.indexes.bloup.sparse.should.equal(true);
done();
done();
});
});
});
@ -2529,23 +2529,23 @@ describe('Database', function () {
});
});
});
it('Indexes can also be removed and the remove persisted', function (done) {
var persDb = "workspace/persistIndexes.db"
, db
;
;
if (fs.existsSync(persDb)) { fs.writeFileSync(persDb, '', 'utf8'); }
db = new Datastore({ filename: persDb, autoload: true });
Object.keys(db.indexes).length.should.equal(1);
Object.keys(db.indexes)[0].should.equal("_id");
db.insert({ planet: "Earth" }, function (err) {
assert.isNull(err);
db.insert({ planet: "Mars" }, function (err) {
assert.isNull(err);
db.ensureIndex({ fieldName: "planet" }, function (err) {
assert.isNull(err);
db.ensureIndex({ fieldName: "another" }, function (err) {
@ -2557,7 +2557,7 @@ describe('Database', function () {
db.indexes._id.getAll().length.should.equal(2);
db.indexes.planet.getAll().length.should.equal(2);
db.indexes.planet.fieldName.should.equal("planet");
// After a reload the indexes are recreated
db = new Datastore({ filename: persDb });
db.loadDatabase(function (err) {

Write Preview
Loading…
Cancel
Save