mirror of https://github.com/seald/nedb
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
310 lines
10 KiB
310 lines
10 KiB
/**
|
|
* Handle every persistence-related task
|
|
* The interface Datastore expects to be implemented is
|
|
* * Persistence.loadDatabase(callback) and callback has signature err
|
|
* * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
|
|
*/
|
|
|
|
var storage = require('./storage')
|
|
, path = require('path')
|
|
, model = require('./model')
|
|
, async = require('async')
|
|
, customUtils = require('./customUtils')
|
|
, Index = require('./indexes')
|
|
;
|
|
|
|
|
|
/**
|
|
* Create a new Persistence object for database options.db
|
|
* @param {Datastore} options.db
|
|
* @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
|
|
* Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
|
|
*/
|
|
function Persistence (options) {
|
|
var i, j, randomString;
|
|
|
|
this.db = options.db;
|
|
this.inMemoryOnly = this.db.inMemoryOnly;
|
|
this.filename = this.db.filename;
|
|
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
|
|
|
|
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
|
|
throw "The datafile name can't end with a ~, which is reserved for crash safe backup files";
|
|
}
|
|
|
|
// After serialization and before deserialization hooks with some basic sanity checks
|
|
if (options.afterSerialization && !options.beforeDeserialization) {
|
|
throw "Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss";
|
|
}
|
|
if (!options.afterSerialization && options.beforeDeserialization) {
|
|
throw "Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss";
|
|
}
|
|
this.afterSerialization = options.afterSerialization || function (s) { return s; };
|
|
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
|
|
for (i = 1; i < 30; i += 1) {
|
|
for (j = 0; j < 10; j += 1) {
|
|
randomString = customUtils.uid(i);
|
|
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
|
|
throw "beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss";
|
|
}
|
|
}
|
|
}
|
|
|
|
// For NW apps, store data in the same directory where NW stores application data
|
|
if (this.filename && options.nodeWebkitAppName) {
|
|
console.log("==================================================================");
|
|
console.log("WARNING: The nodeWebkitAppName option is deprecated");
|
|
console.log("To get the path to the directory where Node Webkit stores the data");
|
|
console.log("for your app, use the internal nw.gui module like this");
|
|
console.log("require('nw.gui').App.dataPath");
|
|
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
|
|
console.log("==================================================================");
|
|
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
|
|
}
|
|
};
|
|
|
|
|
|
/**
|
|
* Check if a directory exists and create it on the fly if it is not the case
|
|
* cb is optional, signature: err
|
|
*/
|
|
Persistence.ensureDirectoryExists = function (dir, cb) {
|
|
var callback = cb || function () {}
|
|
;
|
|
|
|
storage.mkdirp(dir, function (err) { return callback(err); });
|
|
};
|
|
|
|
|
|
|
|
|
|
/**
|
|
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
|
|
* data for this application. Probably the best place to store data
|
|
*/
|
|
Persistence.getNWAppFilename = function (appName, relativeFilename) {
|
|
var home;
|
|
|
|
switch (process.platform) {
|
|
case 'win32':
|
|
case 'win64':
|
|
home = process.env.LOCALAPPDATA || process.env.APPDATA;
|
|
if (!home) { throw "Couldn't find the base application data folder"; }
|
|
home = path.join(home, appName);
|
|
break;
|
|
case 'darwin':
|
|
home = process.env.HOME;
|
|
if (!home) { throw "Couldn't find the base application data directory"; }
|
|
home = path.join(home, 'Library', 'Application Support', appName);
|
|
break;
|
|
case 'linux':
|
|
home = process.env.HOME;
|
|
if (!home) { throw "Couldn't find the base application data directory"; }
|
|
home = path.join(home, '.config', appName);
|
|
break;
|
|
default:
|
|
throw "Can't use the Node Webkit relative path for platform " + process.platform;
|
|
break;
|
|
}
|
|
|
|
return path.join(home, 'nedb-data', relativeFilename);
|
|
}
|
|
|
|
|
|
/**
|
|
* Persist cached database
|
|
* This serves as a compaction function since the cache always contains only the number of documents in the collection
|
|
* while the data file is append-only so it may grow larger
|
|
* @param {Function} cb Optional callback, signature: err
|
|
*/
|
|
Persistence.prototype.persistCachedDatabase = function (cb) {
|
|
var callback = cb || function () {}
|
|
, toPersist = ''
|
|
, self = this
|
|
;
|
|
|
|
if (this.inMemoryOnly) { return callback(null); }
|
|
|
|
this.db.getAllData().forEach(function (doc) {
|
|
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
|
|
});
|
|
Object.keys(this.db.indexes).forEach(function (fieldName) {
|
|
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
|
|
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
|
|
}
|
|
});
|
|
|
|
storage.crashSafeWriteFile(this.filename, toPersist, callback);
|
|
};
|
|
|
|
|
|
/**
|
|
* Queue a rewrite of the datafile
|
|
*/
|
|
Persistence.prototype.compactDatafile = function () {
|
|
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
|
|
};
|
|
|
|
|
|
/**
|
|
* Set automatic compaction every interval ms
|
|
* @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
|
|
*/
|
|
Persistence.prototype.setAutocompactionInterval = function (interval) {
|
|
var self = this
|
|
, minInterval = 5000
|
|
, realInterval = Math.max(interval || 0, minInterval)
|
|
;
|
|
|
|
this.stopAutocompaction();
|
|
|
|
this.autocompactionIntervalId = setInterval(function () {
|
|
self.compactDatafile();
|
|
}, realInterval);
|
|
};
|
|
|
|
|
|
/**
|
|
* Stop autocompaction (do nothing if autocompaction was not running)
|
|
*/
|
|
Persistence.prototype.stopAutocompaction = function () {
|
|
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
|
|
};
|
|
|
|
|
|
/**
|
|
* Persist new state for the given newDocs (can be insertion, update or removal)
|
|
* Use an append-only format
|
|
* @param {Array} newDocs Can be empty if no doc was updated/removed
|
|
* @param {Function} cb Optional, signature: err
|
|
*/
|
|
Persistence.prototype.persistNewState = function (newDocs, cb) {
|
|
var self = this
|
|
, toPersist = ''
|
|
, callback = cb || function () {}
|
|
;
|
|
|
|
// In-memory only datastore
|
|
if (self.inMemoryOnly) { return callback(null); }
|
|
|
|
newDocs.forEach(function (doc) {
|
|
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
|
|
});
|
|
|
|
if (toPersist.length === 0) { return callback(null); }
|
|
|
|
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
|
|
return callback(err);
|
|
});
|
|
};
|
|
|
|
|
|
/**
|
|
* From a database's raw data, return the corresponding
|
|
* machine understandable collection
|
|
*/
|
|
Persistence.prototype.treatRawData = function (rawData) {
|
|
var data = rawData.split('\n')
|
|
, dataById = {}
|
|
, tdata = []
|
|
, i
|
|
, indexes = {}
|
|
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
|
|
;
|
|
|
|
for (i = 0; i < data.length; i += 1) {
|
|
var doc;
|
|
|
|
try {
|
|
doc = model.deserialize(this.beforeDeserialization(data[i]));
|
|
if (doc._id) {
|
|
if (doc.$$deleted === true) {
|
|
delete dataById[doc._id];
|
|
} else {
|
|
dataById[doc._id] = doc;
|
|
}
|
|
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
|
|
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
|
|
} else if (typeof doc.$$indexRemoved === "string") {
|
|
delete indexes[doc.$$indexRemoved];
|
|
}
|
|
} catch (e) {
|
|
corruptItems += 1;
|
|
}
|
|
}
|
|
|
|
// A bit lenient on corruption
|
|
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
|
|
throw "More than 10% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss"
|
|
}
|
|
|
|
Object.keys(dataById).forEach(function (k) {
|
|
tdata.push(dataById[k]);
|
|
});
|
|
|
|
return { data: tdata, indexes: indexes };
|
|
};
|
|
|
|
|
|
/**
|
|
* Load the database
|
|
* 1) Create all indexes
|
|
* 2) Insert all data
|
|
* 3) Compact the database
|
|
* This means pulling data out of the data file or creating it if it doesn't exist
|
|
* Also, all data is persisted right away, which has the effect of compacting the database file
|
|
* This operation is very quick at startup for a big collection (60ms for ~10k docs)
|
|
* @param {Function} cb Optional callback, signature: err
|
|
*/
|
|
Persistence.prototype.loadDatabase = function (cb) {
|
|
var callback = cb || function () {}
|
|
, self = this
|
|
;
|
|
|
|
self.db.resetIndexes();
|
|
|
|
// In-memory only datastore
|
|
if (self.inMemoryOnly) { return callback(null); }
|
|
|
|
async.waterfall([
|
|
function (cb) {
|
|
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
|
|
storage.ensureDatafileIntegrity(self.filename, function (err) {
|
|
storage.readFile(self.filename, 'utf8', function (err, rawData) {
|
|
if (err) { return cb(err); }
|
|
|
|
try {
|
|
var treatedData = self.treatRawData(rawData);
|
|
} catch (e) {
|
|
return cb(e);
|
|
}
|
|
|
|
// Recreate all indexes in the datafile
|
|
Object.keys(treatedData.indexes).forEach(function (key) {
|
|
self.db.indexes[key] = new Index(treatedData.indexes[key]);
|
|
});
|
|
|
|
// Fill cached database (i.e. all indexes) with data
|
|
try {
|
|
self.db.resetIndexes(treatedData.data);
|
|
} catch (e) {
|
|
self.db.resetIndexes(); // Rollback any index which didn't fail
|
|
return cb(e);
|
|
}
|
|
|
|
self.db.persistence.persistCachedDatabase(cb);
|
|
});
|
|
});
|
|
});
|
|
}
|
|
], function (err) {
|
|
if (err) { return callback(err); }
|
|
|
|
self.db.executor.processBuffer();
|
|
return callback(null);
|
|
});
|
|
};
|
|
|
|
|
|
// Interface
|
|
module.exports = Persistence;
|
|
|