Write file line by line when persisting cached database

browser-version/lib/storage.js

@@ -68,11 +68,15 @@ const mkdir = (dir, options, callback) => callback()
 // Nothing to do, no data corruption possible in the browser
 const ensureDatafileIntegrity = (filename, callback) => callback(null)
 
+const crashSafeWriteFileLines = (filename, lines, callback) => {
+  writeFile(filename, lines.join('\n'), callback)
+}
+
 // Interface
 module.exports.exists = exists
 module.exports.rename = rename
 module.exports.writeFile = writeFile
-module.exports.crashSafeWriteFile = writeFile // No need for a crash safe function in the browser
+module.exports.crashSafeWriteFileLines = crashSafeWriteFileLines
 module.exports.appendFile = appendFile
 module.exports.readFile = readFile
 module.exports.unlink = unlink

lib/persistence.js

@@ -73,26 +73,27 @@ class Persistence {
    * @param {Function} callback Optional callback, signature: err
    */
   persistCachedDatabase (callback = () => {}) {
-    let toPersist = ''
+
+    const lines = []
 
     if (this.inMemoryOnly) return callback(null)
 
     this.db.getAllData().forEach(doc => {
-      toPersist += this.afterSerialization(model.serialize(doc)) + '\n'
+      lines.push(this.afterSerialization(model.serialize(doc)))
     })
     Object.keys(this.db.indexes).forEach(fieldName => {
       if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted
-        toPersist += this.afterSerialization(model.serialize({
+        lines.push(this.afterSerialization(model.serialize({
           $$indexCreated: {
             fieldName: fieldName,
             unique: this.db.indexes[fieldName].unique,
             sparse: this.db.indexes[fieldName].sparse
           }
-        })) + '\n'
+        })))
       }
     })
 
-    storage.crashSafeWriteFile(this.filename, toPersist, err => {
+    storage.crashSafeWriteFileLines(this.filename, lines, err => {
       if (err) return callback(err)
       this.db.emit('compaction.done')
       return callback(null)

lib/storage.js

@@ -73,10 +73,10 @@ storage.flushToStorage = (options, callback) => {
 /**
  * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
  * @param {String} filename
- * @param {String} data
+ * @param {String[]} lines
  * @param {Function} callback Optional callback, signature: err
  */
-storage.crashSafeWriteFile = (filename, data, callback = () => {}) => {
+storage.crashSafeWriteFileLines = (filename, lines, callback = () => {}) => {
   const tempFilename = filename + '~'
 
   async.waterfall([
@@ -88,7 +88,17 @@ storage.crashSafeWriteFile = (filename, data, callback = () => {}) => {
       })
     },
     cb => {
-      storage.writeFile(tempFilename, data, err => cb(err))
+      try {
+        const stream = fs.createWriteStream(tempFilename, {
+          flags: 'w'
+        })
+        for (const line of lines) {
+          stream.write(line+'\n')
+        }
+        stream.close(() => cb())
+      } catch (err) {
+        cb(err)
+      }
     },
     async.apply(storage.flushToStorage, tempFilename),
     cb => {

test_lac/loadAndCrash.test.js

@@ -114,6 +114,18 @@ fs.writeFile = function (path, data, options, callback_) {
   }
 }
 
+fs.createWriteStream = function (path) {
+  let content = ''
+  return {
+    write(data) {
+      content += data
+    },
+    close(callback) {
+      fs.writeFile(path, content, callback)
+    }
+  }
+}
+
 // End of fs modification
 const Nedb = require('../lib/datastore.js')
 const db = new Nedb({ filename: 'workspace/lac.db' })