From ba0ee21c03ef63b5f2e55cca7e81024d331cb747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Rebours?= Date: Mon, 14 Mar 2022 21:01:28 +0100 Subject: [PATCH] add a mode option --- API.md | 11 ++++- CHANGELOG.md | 1 + lib/datastore.js | 9 +++- lib/persistence.js | 27 ++++++++---- lib/storage.js | 35 +++++++++------ test/persistence.async.test.js | 78 ++++++++++++++++++++++++++++++++++ 6 files changed, 136 insertions(+), 25 deletions(-) diff --git a/API.md b/API.md index 325d046..9c2171c 100644 --- a/API.md +++ b/API.md @@ -51,7 +51,7 @@ with appendfsync option set to no.

Generic async function.

GenericCallback : function

Callback with generic parameters.

-
document : Object.<string, *>
+
document : object

Generic document in NeDB. It consists of an Object with anything you want inside.

query : Object.<string, *>
@@ -309,6 +309,10 @@ automatically considered in-memory only. It cannot end with a ~ whi perform crash-safe writes. Not used if options.inMemoryOnly is true.

- [.inMemoryOnly] boolean = false -

If set to true, no data will be written in storage. This option has priority over options.filename.

+ - [.mode] object -

Permissions to use for FS. Only used for +Node.js storage module.

+ - [.fileMode] number = 0o644 -

Permissions to use for database files

+ - [.dirMode] number = 0o755 -

Permissions to use for database directories

- [.timestampData] boolean = false -

If set to true, createdAt and updatedAt will be created and populated automatically (if not specified by user)

- [.autoload] boolean = false -

If used, the database will automatically be loaded from the datafile @@ -834,6 +838,9 @@ with appendfsync option set to no.

- [.corruptAlertThreshold] Number -

Optional, threshold after which an alert is thrown if too much data is corrupt

- [.beforeDeserialization] [serializationHook](#serializationHook) -

Hook you can use to transform data after it was serialized and before it is written to disk.

- [.afterSerialization] [serializationHook](#serializationHook) -

Inverse of afterSerialization.

+ - [.mode] object -

Modes to use for FS permissions.

+ - [.fileMode] number = 0o644 -

Mode to use for files.

+ - [.dirMode] number = 0o755 -

Mode to use for directories.

@@ -934,7 +941,7 @@ with appendfsync option set to no.

-## document : Object.<string, \*> +## document : object

Generic document in NeDB. It consists of an Object with anything you want inside.

diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b27a6..561dd7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - An auto-generated JSDoc file is generated: [API.md](./API.md). - Added `Datastore#dropDatabaseAsync` and its callback equivalent. - The Error given when the `Datastore#corruptAlertThreshold` is reached now has three properties: `dataLength` which is the amount of lines in the database file (excluding empty lines), `corruptItems` which is the amount of corrupted lines, `corruptionRate` which the rate of corruption between 0 and 1. +- Added a `mode` option which allows to set the file and / or directory modes, by default, it uses `0o644` for files and `0o755` for directories, which may be breaking. ### Changed - The `corruptionAlertThreshold` now doesn't take into account empty lines, and the error message is slightly changed. diff --git a/lib/datastore.js b/lib/datastore.js index 38629ea..da39c05 100755 --- a/lib/datastore.js +++ b/lib/datastore.js @@ -70,7 +70,7 @@ const { isDate } = require('./utils.js') * @typedef document * @property {?string} [_id] Internal `_id` of the document, which can be `null` or undefined at some points (when not * inserted yet for example). - * @type {Object.} + * @type {object} */ /** @@ -165,6 +165,10 @@ class Datastore extends EventEmitter { * perform crash-safe writes. Not used if `options.inMemoryOnly` is `true`. * @param {boolean} [options.inMemoryOnly = false] If set to true, no data will be written in storage. This option has * priority over `options.filename`. + * @param {object} [options.mode] Permissions to use for FS. Only used for + * Node.js storage module. + * @param {number} [options.mode.fileMode = 0o644] Permissions to use for database files + * @param {number} [options.mode.dirMode = 0o755] Permissions to use for database directories * @param {boolean} [options.timestampData = false] If set to true, createdAt and updatedAt will be created and * populated automatically (if not specified by user) * @param {boolean} [options.autoload = false] If used, the database will automatically be loaded from the datafile @@ -258,7 +262,8 @@ class Datastore extends EventEmitter { db: this, afterSerialization: options.afterSerialization, beforeDeserialization: options.beforeDeserialization, - corruptAlertThreshold: options.corruptAlertThreshold + corruptAlertThreshold: options.corruptAlertThreshold, + mode: options.mode }) // This new executor is ready if we don't use persistence diff --git a/lib/persistence.js b/lib/persistence.js index 0efd7b1..b110ba5 100755 --- a/lib/persistence.js +++ b/lib/persistence.js @@ -6,6 +6,9 @@ const Index = require('./indexes.js') const model = require('./model.js') const storage = require('./storage.js') +const DEFAULT_DIR_MODE = 0o755 +const DEFAULT_FILE_MODE = 0o644 + /** * Under the hood, NeDB's persistence uses an append-only format, meaning that all * updates and deletes actually result in lines added at the end of the datafile, @@ -43,13 +46,18 @@ class Persistence { * @param {Number} [options.corruptAlertThreshold] Optional, threshold after which an alert is thrown if too much data is corrupt * @param {serializationHook} [options.beforeDeserialization] Hook you can use to transform data after it was serialized and before it is written to disk. * @param {serializationHook} [options.afterSerialization] Inverse of `afterSerialization`. + * @param {object} [options.mode] Modes to use for FS permissions. + * @param {number} [options.mode.fileMode=0o644] Mode to use for files. + * @param {number} [options.mode.dirMode=0o755] Mode to use for directories. */ constructor (options) { this.db = options.db this.inMemoryOnly = this.db.inMemoryOnly this.filename = this.db.filename this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1 - + this.mode = options.mode !== undefined ? options.mode : { fileMode: DEFAULT_FILE_MODE, dirMode: DEFAULT_DIR_MODE } + if (this.mode.fileMode === undefined) this.mode.fileMode = DEFAULT_FILE_MODE + if (this.mode.dirMode === undefined) this.mode.dirMode = DEFAULT_DIR_MODE if ( !this.inMemoryOnly && this.filename && @@ -104,7 +112,7 @@ class Persistence { } }) - await storage.crashSafeWriteFileLinesAsync(this.filename, lines) + await storage.crashSafeWriteFileLinesAsync(this.filename, lines, this.mode) this.db.emit('compaction.done') } @@ -155,7 +163,7 @@ class Persistence { if (toPersist.length === 0) return - await storage.appendFileAsync(this.filename, toPersist, 'utf8') + await storage.appendFileAsync(this.filename, toPersist, { encoding: 'utf8', mode: this.mode.fileMode }) } /** @@ -297,17 +305,17 @@ class Persistence { // In-memory only datastore if (this.inMemoryOnly) return - await Persistence.ensureDirectoryExistsAsync(path.dirname(this.filename)) - await storage.ensureDatafileIntegrityAsync(this.filename) + await Persistence.ensureDirectoryExistsAsync(path.dirname(this.filename), this.mode.dirMode) + await storage.ensureDatafileIntegrityAsync(this.filename, this.mode.fileMode) let treatedData if (storage.readFileStream) { // Server side - const fileStream = storage.readFileStream(this.filename, { encoding: 'utf8' }) + const fileStream = storage.readFileStream(this.filename, { encoding: 'utf8', mode: this.mode.fileMode }) treatedData = await this.treatRawStreamAsync(fileStream) } else { // Browser - const rawData = await storage.readFileAsync(this.filename, 'utf8') + const rawData = await storage.readFileAsync(this.filename, { encoding: 'utf8', mode: this.mode.fileMode }) treatedData = this.treatRawData(rawData) } // Recreate all indexes in the datafile @@ -357,11 +365,12 @@ class Persistence { /** * Check if a directory stat and create it on the fly if it is not the case. * @param {string} dir + * @param {number} [mode=0o777] * @return {Promise} * @private */ - static async ensureDirectoryExistsAsync (dir) { - await storage.mkdirAsync(dir, { recursive: true }) + static async ensureDirectoryExistsAsync (dir, mode = DEFAULT_DIR_MODE) { + await storage.mkdirAsync(dir, { recursive: true, mode }) } } diff --git a/lib/storage.js b/lib/storage.js index b6d72cc..32a9ca3 100755 --- a/lib/storage.js +++ b/lib/storage.js @@ -13,6 +13,9 @@ const fsPromises = fs.promises const path = require('path') const { Readable } = require('stream') +const DEFAULT_DIR_MODE = 0o755 +const DEFAULT_FILE_MODE = 0o644 + /** * Returns true if file exists. * @param {string} file @@ -126,6 +129,7 @@ const ensureFileDoesntExistAsync = async file => { * @param {object|string} options If options is a string, it is assumed that the flush of the file (not dir) called options was requested * @param {string} [options.filename] * @param {boolean} [options.isDir = false] Optional, defaults to false + * @param {number} [options.mode = 0o644] Optional, defaults to 0o644 * @return {Promise} * @alias module:storage.flushToStorageAsync * @async @@ -133,14 +137,16 @@ const ensureFileDoesntExistAsync = async file => { const flushToStorageAsync = async (options) => { let filename let flags + let mode if (typeof options === 'string') { filename = options flags = 'r+' + mode = DEFAULT_FILE_MODE } else { filename = options.filename flags = options.isDir ? 'r' : 'r+' + mode = options.mode || DEFAULT_FILE_MODE } - /** * Some OSes and/or storage backends (augmented node fs) do not support fsync (FlushFileBuffers) directories, * or calling open() on directories at all. Flushing fails silently in this case, supported by following heuristics: @@ -155,7 +161,7 @@ const flushToStorageAsync = async (options) => { let filehandle, errorOnFsync, errorOnClose try { - filehandle = await fsPromises.open(filename, flags) + filehandle = await fsPromises.open(filename, flags, mode) try { await filehandle.sync() } catch (errFS) { @@ -182,13 +188,14 @@ const flushToStorageAsync = async (options) => { * Fully write or rewrite the datafile. * @param {string} filename * @param {string[]} lines + * @param {number} [mode=0o644] * @return {Promise} * @alias module:storage.writeFileLinesAsync * @async */ -const writeFileLinesAsync = (filename, lines) => new Promise((resolve, reject) => { +const writeFileLinesAsync = (filename, lines, mode = DEFAULT_FILE_MODE) => new Promise((resolve, reject) => { try { - const stream = writeFileStream(filename) + const stream = writeFileStream(filename, { mode: mode }) const readable = Readable.from(lines) readable.on('data', (line) => { try { @@ -220,33 +227,37 @@ const writeFileLinesAsync = (filename, lines) => new Promise((resolve, reject) = * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost). * @param {string} filename * @param {string[]} lines + * @param {object} [mode={ fileMode: 0o644, dirMode: 0o755 }] + * @param {number} mode.dirMode + * @param {number} mode.fileMode * @return {Promise} * @alias module:storage.crashSafeWriteFileLinesAsync */ -const crashSafeWriteFileLinesAsync = async (filename, lines) => { +const crashSafeWriteFileLinesAsync = async (filename, lines, mode = { fileMode: DEFAULT_FILE_MODE, dirMode: DEFAULT_DIR_MODE }) => { const tempFilename = filename + '~' - await flushToStorageAsync({ filename: path.dirname(filename), isDir: true }) + await flushToStorageAsync({ filename: path.dirname(filename), isDir: true, mode: mode.dirMode }) const exists = await existsAsync(filename) - if (exists) await flushToStorageAsync({ filename }) + if (exists) await flushToStorageAsync({ filename, mode: mode.fileMode }) - await writeFileLinesAsync(tempFilename, lines) + await writeFileLinesAsync(tempFilename, lines, mode.fileMode) - await flushToStorageAsync(tempFilename) + await flushToStorageAsync({ filename: tempFilename, mode: mode.fileMode }) await renameAsync(tempFilename, filename) - await flushToStorageAsync({ filename: path.dirname(filename), isDir: true }) + await flushToStorageAsync({ filename: path.dirname(filename), isDir: true, mode: mode.dirMode }) } /** * Ensure the datafile contains all the data, even if there was a crash during a full file write. * @param {string} filename + * @param {number} [mode=0o644] * @return {Promise} * @alias module:storage.ensureDatafileIntegrityAsync */ -const ensureDatafileIntegrityAsync = async filename => { +const ensureDatafileIntegrityAsync = async (filename, mode = DEFAULT_FILE_MODE) => { const tempFilename = filename + '~' const filenameExists = await existsAsync(filename) @@ -255,7 +266,7 @@ const ensureDatafileIntegrityAsync = async filename => { const oldFilenameExists = await existsAsync(tempFilename) // New database - if (!oldFilenameExists) await writeFileAsync(filename, '', 'utf8') + if (!oldFilenameExists) await writeFileAsync(filename, '', { encoding: 'utf8', mode }) // Write failed, use old version else await renameAsync(tempFilename, filename) } diff --git a/test/persistence.async.test.js b/test/persistence.async.test.js index c414a36..dc709d3 100755 --- a/test/persistence.async.test.js +++ b/test/persistence.async.test.js @@ -999,3 +999,81 @@ describe('Persistence async', function () { }) }) // ==== End of 'dropDatabase' ==== }) + +const getMode = async path => { + const { mode } = await fs.lstat(path) + const octalString = mode.toString('8') + return parseInt(octalString.substring(octalString.length - 4), '8') +} + +const testPermissions = async (db, fileMode, dirMode) => { + assert.equal(db.persistence.mode.fileMode, fileMode) + assert.equal(db.persistence.mode.dirMode, dirMode) + await db.loadDatabaseAsync() + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.ensureIndex({ fieldName: 'foo' }) + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.insertAsync({ hello: 'world' }) + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.removeAsync({ hello: 'world' }) + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.updateAsync({ hello: 'world2' }) + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.removeIndex({ fieldName: 'foo' }) + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) + await db.compactDatafileAsync() + assert.equal(await getMode(db.filename), fileMode) + assert.equal(await getMode(path.dirname(db.filename)), dirMode) +} +describe('permissions', function () { + const testDb = 'workspace/permissions/test.db' + + beforeEach('cleanup', async () => { + try { + await fs.chmod(path.dirname(testDb), 0o755) + await fs.rm(testDb, { force: true }) + await fs.rmdir(path.dirname(testDb, { recursive: true })) + } catch (err) { + if (err.code !== 'ENOENT') throw err + } + }) + + it('ensureDirectoryExists forwards mode argument', async () => { + await Persistence.ensureDirectoryExistsAsync(path.dirname(testDb), 0o700) + assert.equal(await getMode(path.dirname(testDb)), 0o700) + }) + + it('Setting nothing', async () => { + const FILE_MODE = 0o644 + const DIR_MODE = 0o755 + const db = new Datastore({ filename: testDb }) + await testPermissions(db, FILE_MODE, DIR_MODE) + }) + + it('Setting only fileMode', async () => { + const FILE_MODE = 0o600 + const DIR_MODE = 0o755 + const db = new Datastore({ filename: testDb, mode: { fileMode: FILE_MODE } }) + await testPermissions(db, FILE_MODE, DIR_MODE) + }) + + it('Setting only dirMode', async () => { + const FILE_MODE = 0o644 + const DIR_MODE = 0o700 + const db = new Datastore({ filename: testDb, mode: { dirMode: DIR_MODE } }) + await testPermissions(db, FILE_MODE, DIR_MODE) + }) + + it('Setting fileMode & dirMode', async () => { + const FILE_MODE = 0o600 + const DIR_MODE = 0o700 + const db = new Datastore({ filename: testDb, mode: { dirMode: DIR_MODE, fileMode: FILE_MODE } }) + await testPermissions(db, FILE_MODE, DIR_MODE) + }) +})