/** * Handle every persistence-related task * The interface Datastore expects to be implemented is * * Persistence.loadDatabase(callback) and callback has signature err * * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err */ var fs = require('fs') , path = require('path') , model = require('./model') , async = require('async') , mkdirp = require('mkdirp') ; /** * Create a new Persistence object for database options.db * @param {Datastore} options.db * @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where * Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion) */ function Persistence (options) { this.db = options.db; this.inMemoryOnly = this.db.inMemoryOnly; this.filename = this.db.filename; if (!this.inMemoryOnly && this.filename) { if (this.filename.charAt(this.filename.length - 1) === '~') { throw "The datafile name can't end with a ~, which is reserved for automatic backup files"; } else { this.tempFilename = this.filename + '~'; } } // For NW apps, store data in the same directory where NW stores application data if (this.filename && options.nodeWebkitAppName) { this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename); } }; /** * Check if a directory exists and create it on the fly if it is not the case * cb is optional, signature: err */ Persistence.ensureDirectoryExists = function (dir, cb) { var callback = cb || function () {} ; mkdirp(dir, function (err) { return callback(err); }); }; /** * Return the path the datafile if the given filename is relative to the directory where Node Webkit stores * data for this application. Probably the best place to store data */ Persistence.getNWAppFilename = function (appName, relativeFilename) { var home; switch (process.platform) { case 'win32': case 'win64': home = process.env.LOCALAPPDATA || process.env.APPDATA; if (!home) { throw "Couldn't find the base application data folder"; } home = path.join(home, appName); break; case 'darwin': home = process.env.HOME; if (!home) { throw "Couldn't find the base application data directory"; } home = path.join(home, 'Library', 'Application Support', appName); break; case 'linux': home = process.env.HOME; if (!home) { throw "Couldn't find the base application data directory"; } home = path.join(home, '.config', appName); break; default: throw "Can't use the Node Webkit relative path for platform " + process.platform; break; } return path.join(home, 'nedb-data', relativeFilename); } /** * Persist cached database * This serves as a compaction function since the cache always contains only the number of documents in the collection * while the data file is append-only so it may grow larger * @param {Function} cb Optional callback, signature: err */ Persistence.prototype.persistCachedDatabase = function (cb) { var callback = cb || function () {} , toPersist = '' , self = this ; if (this.inMemoryOnly) { return callback(null); } this.db.getAllData().forEach(function (doc) { toPersist += model.serialize(doc) + '\n'; }); async.waterfall([ function (cb) { fs.exists(self.tempFilename, function (exists) { if (exists) { // Shouldn't happen since ensureDatafileIntegrity removes the temp datafile fs.unlink(self.tempFilename, function (err) { return cb(err); }); } else { return cb(); } }); } , function (cb) { fs.exists(self.filename, function (exists) { if (exists) { fs.rename(self.filename, self.tempFilename, function (err) { return cb(err); }); } else { return cb(); } }); } , function (cb) { fs.writeFile(self.filename, toPersist, function (err) { return cb(err); }); } , function (cb) { fs.exists(self.tempFilename, function (exists) { if (exists) { fs.unlink(self.tempFilename, function (err) { return cb(err); }); } else { return cb(); } }); } ], function (err) { if (err) { return callback(err); } else { return callback(null); } }) }; /** * Queue a rewrite of the datafile */ Persistence.prototype.compactDatafile = function () { this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] }); }; /** * Set automatic compaction every interval ms * @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds */ Persistence.prototype.setAutocompactionInterval = function (interval) { var self = this; if (interval < 5000) { interval = 5000; } this.stopAutocompaction(); this.autocompactionIntervalId = setInterval(function () { self.compactDatafile(); }, interval); }; /** * Stop autocompaction (do nothing if autocompaction was not running) */ Persistence.prototype.stopAutocompaction = function () { if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); } }; /** * Persist new state for the given newDocs (can be insertion, update or removal) * Use an append-only format * @param {Array} newDocs Can be empty if no doc was updated/removed * @param {Function} cb Optional, signature: err */ Persistence.prototype.persistNewState = function (newDocs, cb) { var self = this , toPersist = '' , callback = cb || function () {} ; // In-memory only datastore if (self.inMemoryOnly) { return callback(null); } newDocs.forEach(function (doc) { toPersist += model.serialize(doc) + '\n'; }); if (toPersist.length === 0) { return callback(null); } fs.appendFile(self.filename, toPersist, 'utf8', function (err) { return callback(err); }); }; /** * From a database's raw data, return the corresponding * machine understandable collection */ Persistence.treatRawData = function (rawData) { var data = rawData.split('\n') , dataById = {} , res = [] , i; for (i = 0; i < data.length; i += 1) { var doc; try { doc = model.deserialize(data[i]); if (doc._id) { if (doc.$$deleted === true) { delete dataById[doc._id]; } else { dataById[doc._id] = doc; } } } catch (e) { } } Object.keys(dataById).forEach(function (k) { res.push(dataById[k]); }); return res; }; /** * Ensure that this.filename contains the most up-to-date version of the data * Even if a loadDatabase crashed before */ Persistence.prototype.ensureDatafileIntegrity = function (callback) { var self = this ; fs.exists(self.filename, function (filenameExists) { fs.exists(self.tempFilename, function (tempFilenameExists) { // Normal case if (filenameExists && !tempFilenameExists) { return callback(null); } // Process crashed right after renaming filename if (!filenameExists && tempFilenameExists) { return fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); }); } // No file exists, create empty datafile if (!filenameExists && !tempFilenameExists) { return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); }); } // Process crashed after or during write to datafile // If datafile is not empty, it means process crashed after the write so we use it // If it is empty, we don't know whether the database was emptied or we had a crash during write. The safest option is to use the temp datafile if (filenameExists && tempFilenameExists) { return fs.stat(self.filename, function (err, stats) { if (err) { return callback(err); } if (stats.size > 0) { fs.unlink(self.tempFilename, function (err) { return callback(err); }); } else { fs.unlink(self.filename, function (err) { if (err) { return callback(err); } fs.rename(self.tempFilename, self.filename, function (err) { return callback(err); }); }); } }); } }); }); }; /** * Load the database * This means pulling data out of the data file or creating it if it doesn't exist * Also, all data is persisted right away, which has the effect of compacting the database file * This operation is very quick at startup for a big collection (60ms for ~10k docs) * @param {Function} cb Optional callback, signature: err */ Persistence.prototype.loadDatabase = function (cb) { var callback = cb || function () {} , self = this ; self.db.resetIndexes(); // In-memory only datastore if (self.inMemoryOnly) { return callback(null); } async.waterfall([ function (cb) { Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) { self.ensureDatafileIntegrity(function (exists) { fs.readFile(self.filename, 'utf8', function (err, rawData) { if (err) { return cb(err); } var treatedData = Persistence.treatRawData(rawData); try { self.db.resetIndexes(treatedData); } catch (e) { self.db.resetIndexes(); // Rollback any index which didn't fail return cb(e); } self.db.persistence.persistCachedDatabase(cb); }); }); }); } ], function (err) { if (err) { return callback(err); } self.db.executor.processBuffer(); return callback(null); }); }; // Interface module.exports = Persistence;