Merge pull request #2704 from node-red/split-config

Split .config.json into separate files

packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/index.js

@@ -15,7 +15,6 @@
  **/
 
 var fs = require('fs-extra');
-var when = require('when');
 var fspath = require("path");
 
 var log = require("@node-red/util").log; // TODO: separate module
@@ -29,6 +28,11 @@ var projects = require("./projects");
 var initialFlowLoadComplete = false;
 var settings;
 
+function checkForConfigFile(dir) {
+    return fs.existsSync(fspath.join(dir,".config.json")) ||
+           fs.existsSync(fspath.join(dir,".config.nodes.json"))
+}
+
 var localfilesystem = {
     init: function(_settings, runtime) {
         settings = _settings;
@@ -36,34 +40,24 @@ var localfilesystem = {
         var promises = [];
 
         if (!settings.userDir) {
-            try {
+            if (checkForConfigFile(process.env.NODE_RED_HOME)) {
-                fs.statSync(fspath.join(process.env.NODE_RED_HOME,".config.json"));
+                settings.userDir = process.env.NODE_RED_HOME
-                settings.userDir = process.env.NODE_RED_HOME;
+            } else if (process.env.HOMEPATH && checkForConfigFile(fspath.join(process.env.HOMEPATH,".node-red"))) {
-            } catch(err) {
+                settings.userDir = fspath.join(process.env.HOMEPATH,".node-red");
-                try {
+            } else {
-                    // Consider compatibility for older versions
+                settings.userDir = fspath.join(process.env.HOME || process.env.USERPROFILE || process.env.HOMEPATH || process.env.NODE_RED_HOME,".node-red");
-                    if (process.env.HOMEPATH) {
-                        fs.statSync(fspath.join(process.env.HOMEPATH,".node-red",".config.json"));
-                        settings.userDir = fspath.join(process.env.HOMEPATH,".node-red");
-                    }
-                } catch(err) {
-                }
-                if (!settings.userDir) {
-                    settings.userDir = fspath.join(process.env.HOME || process.env.USERPROFILE || process.env.HOMEPATH || process.env.NODE_RED_HOME,".node-red");
-                    if (!settings.readOnly) {
-                        promises.push(fs.ensureDir(fspath.join(settings.userDir,"node_modules")));
-                    }
-                }
             }
         }
-
+        if (!settings.readOnly) {
+            promises.push(fs.ensureDir(fspath.join(settings.userDir,"node_modules")));
+        }
         sessions.init(settings);
-        runtimeSettings.init(settings);
+        promises.push(runtimeSettings.init(settings));
         promises.push(library.init(settings));
         promises.push(projects.init(settings, runtime));
 
         var packageFile = fspath.join(settings.userDir,"package.json");
-        var packagePromise = when.resolve();
+        var packagePromise = Promise.resolve();
 
         if (!settings.readOnly) {
             packagePromise = function() {
@@ -81,7 +75,7 @@ var localfilesystem = {
                 return true;
             }
         }
-        return when.all(promises).then(packagePromise);
+        return Promise.all(promises).then(packagePromise);
     },
 
 

packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/settings.js

@@ -14,41 +14,116 @@
  * limitations under the License.
  **/
 
-var when = require('when');
+const fs = require('fs-extra');
-var fs = require('fs-extra');
+const fspath = require("path");
-var fspath = require("path");
 
-var log = require("@node-red/util").log; // TODO: separate module
+const log = require("@node-red/util").log;
-var util = require("./util");
+const util = require("./util");
+
+const configSections = ['nodes','users','projects'];
+
+const settingsCache = {};
 
 var globalSettingsFile;
 var globalSettingsBackup;
 var settings;
 
+async function migrateToMultipleConfigFiles() {
+    const nodesFilename = getSettingsFilename("nodes");
+    if (fs.existsSync(nodesFilename)) {
+        // We have both .config.json and .config.nodes.json
+        // Use the more recently modified. This handles users going back to pre1.2
+        // and up again.
+        // We can remove this logic in 1.3+ and remove the old .config.json file entirely
+        //
+        const fsStatNodes = await fs.stat(nodesFilename);
+        const fsStatGlobal = await fs.stat(globalSettingsFile);
+        if (fsStatNodes.mtimeMs > fsStatGlobal.mtimeMs) {
+            // .config.nodes.json is newer than .config.json - no migration needed
+            return;
+        }
+    }
+    const data = await util.readFile(globalSettingsFile,globalSettingsBackup,{});
+    // In a later release we should remove the old settings file. But don't do
+    // that *yet* otherwise users won't be able to downgrade easily.
+    return writeSettings(data) // .then( () => fs.remove(globalSettingsFile) );
+}
+
+
+/**
+ * Takes the single settings object and splits it into separate files. This makes
+ * it easier to backup selected parts of the settings and also helps reduce the blast
+ * radius if a file is lost.
+ *
+ * The settings are written to four files:
+ *  - .config.nodes.json - the node registry
+ *  - .config.users.json - user specific settings (eg editor settings)
+ *  - .config.projects.json - project settings, including the active project
+ *  - .config.runtime.json - everything else - most notable _credentialSecret
+ */
+function writeSettings(data) {
+    const configKeys = Object.keys(data);
+    const writePromises = [];
+    configSections.forEach(key => {
+        const sectionData = data[key] || {};
+        delete data[key];
+        const sectionFilename = getSettingsFilename(key);
+        const sectionContent = JSON.stringify(sectionData,null,4);
+        if (sectionContent !== settingsCache[key]) {
+            settingsCache[key] = sectionContent;
+            writePromises.push(util.writeFile(sectionFilename,sectionContent,sectionFilename+".backup"))
+        }
+    })
+    // Having extracted nodes/users/projects, write whatever is left to the runtime config
+    const sectionFilename = getSettingsFilename("runtime");
+    const sectionContent = JSON.stringify(data,null,4);
+    if (sectionContent !== settingsCache["runtime"]) {
+        settingsCache["runtime"] = sectionContent;
+        writePromises.push(util.writeFile(sectionFilename,sectionContent,sectionFilename+".backup"));
+    }
+    return Promise.all(writePromises);
+}
+
+async function readSettings() {
+    // Read the 'runtime' settings file first
+    const runtimeFilename = getSettingsFilename("runtime");
+    const result = await util.readFile(runtimeFilename,runtimeFilename+".backup",{});
+    const readPromises = [];
+    // Read the other settings files and add them into the runtime settings
+    configSections.forEach(key => {
+        const sectionFilename = getSettingsFilename(key);
+        readPromises.push(util.readFile(sectionFilename,sectionFilename+".backup",{}).then(sectionData => {
+            if (Object.keys(sectionData).length > 0) {
+                result[key] = sectionData;
+            }
+        }))
+    });
+    return Promise.all(readPromises).then(() => result);
+}
+
+function getSettingsFilename(section) {
+    return fspath.join(settings.userDir,`.config.${section}.json`);
+}
+
 module.exports = {
     init: function(_settings) {
         settings = _settings;
         globalSettingsFile = fspath.join(settings.userDir,".config.json");
         globalSettingsBackup = fspath.join(settings.userDir,".config.json.backup");
+
+        if (fs.existsSync(globalSettingsFile) && !settings.readOnly) {
+            return migrateToMultipleConfigFiles();
+        } else {
+            return Promise.resolve();
+        }
     },
     getSettings: function() {
-        return when.promise(function(resolve,reject) {
+        return readSettings()
-            fs.readFile(globalSettingsFile,'utf8',function(err,data) {
-                if (!err) {
-                    try {
-                        return resolve(util.parseJSON(data));
-                    } catch(err2) {
-                        log.trace("Corrupted config detected - resetting");
-                    }
-                }
-                return resolve({});
-            })
-        })
     },
     saveSettings: function(newSettings) {
         if (settings.readOnly) {
-            return when.resolve();
+            return Promise.resolve();
         }
-        return util.writeFile(globalSettingsFile,JSON.stringify(newSettings,null,1),globalSettingsBackup);
+        return writeSettings(newSettings);
     }
 }

packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/util.js

@@ -14,11 +14,10 @@
  * limitations under the License.
  **/
 
-var fs = require('fs-extra');
+const fs = require('fs-extra');
-var fspath = require('path');
+const fspath = require('path');
-var when = require('when');
 
-var log = require("@node-red/util").log; // TODO: separate module
+const log = require("@node-red/util").log;
 
 function parseJSON(data) {
     if (data.charCodeAt(0) === 0xFEFF) {
@@ -27,7 +26,7 @@ function parseJSON(data) {
     return JSON.parse(data);
 }
 function readFile(path,backupPath,emptyResponse,type) {
-    return when.promise(function(resolve) {
+    return new Promise(function(resolve) {
         fs.readFile(path,'utf8',function(err,data) {
             if (!err) {
                 if (data.length === 0) {

test/unit/@node-red/runtime/lib/storage/localfilesystem/settings_spec.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  **/
 
-var should = require("should");
+const should = require("should");
-var fs = require('fs-extra');
+const fs = require('fs-extra');
-var path = require('path');
+const path = require('path');
 
 var NR_TEST_UTILS = require("nr-test-utils");
 
@@ -34,49 +34,100 @@ describe('storage/localfilesystem/settings', function() {
     });
 
     it('should handle non-existent settings', function(done) {
-        var settingsFile = path.join(userDir,".settings.json");
+        var settingsFile = path.join(userDir,".config.json");
-
+        localfilesystemSettings.init({userDir:userDir}).then(function() {
-        localfilesystemSettings.init({userDir:userDir});
+            fs.existsSync(settingsFile).should.be.false();
-        fs.existsSync(settingsFile).should.be.false();
+            return localfilesystemSettings.getSettings();
-        localfilesystemSettings.getSettings().then(function(settings) {
+        }).then(function(settings) {
             settings.should.eql({});
             done();
-        }).catch(function(err) {
+        }).catch(err => { done(err)});
-            done(err);
-        });
     });
 
-    it('should handle corrupt settings', function(done) {
+    it('should migrate single config.json to multiple files', function(done) {
         var settingsFile = path.join(userDir,".config.json");
-        fs.writeFileSync(settingsFile,"[This is not json","utf8");
+        fs.writeFileSync(settingsFile,JSON.stringify({
-        localfilesystemSettings.init({userDir:userDir});
+            nodes:{a:1},
-        fs.existsSync(settingsFile).should.be.true();
+            _credentialSecret: "foo",
-        localfilesystemSettings.getSettings().then(function(settings) {
+            users:{b:2},
-            settings.should.eql({});
+            projects: {c:3}
+        }),"utf8");
+
+        async function checkFile(sectionName, expectedContents) {
+            const file = path.join(userDir,".config."+sectionName+".json");
+            fs.existsSync(file).should.be.true();
+            var contents = await fs.readFile(file,'utf8');
+            var data = JSON.parse(contents);
+            data.should.eql(expectedContents)
+        }
+
+        localfilesystemSettings.init({userDir:userDir}).then(async function() {
+            fs.existsSync(settingsFile).should.be.false();
+            await checkFile("nodes",{a:1})
+            await checkFile("users",{b:2})
+            await checkFile("projects",{c:3})
+            await checkFile("runtime",{_credentialSecret:"foo"})
             done();
-        }).catch(function(err) {
+        }).catch(err => { done(err)});
-            done(err);
-        });
     });
 
-    it('should handle settings', function(done) {
+    it('should load separate settings file', async function() {
-        var settingsFile = path.join(userDir,".config.json");
+        await fs.writeFile( path.join(userDir,".config.nodes.json"),JSON.stringify({a:1}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.users.json"),JSON.stringify({b:2}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.projects.json"),JSON.stringify({c:3}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.runtime.json"),JSON.stringify({_credentialSecret:"foo"}),"utf8");
 
-        localfilesystemSettings.init({userDir:userDir});
+        return localfilesystemSettings.init({userDir:userDir})
-        fs.existsSync(settingsFile).should.be.false();
+            .then(localfilesystemSettings.getSettings)
+            .then(settings => {
+                settings.should.eql({
+                    nodes:{a:1},
+                    _credentialSecret: "foo",
+                    users:{b:2},
+                    projects: {c:3}
+                })
+        })
+    });
 
-        var settings = {"abc":{"type":"creds"}};
+    it('should write only the files that need writing', async function() {
+        await fs.writeFile( path.join(userDir,".config.nodes.json"),JSON.stringify({a:1}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.users.json"),JSON.stringify({b:2}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.projects.json"),JSON.stringify({c:3}),"utf8");
+        await fs.writeFile( path.join(userDir,".config.runtime.json"),JSON.stringify({_credentialSecret:"foo"}),"utf8");
 
-        localfilesystemSettings.saveSettings(settings).then(function() {
+        const fsStatNodes = await fs.stat(path.join(userDir,".config.nodes.json"))
-            fs.existsSync(settingsFile).should.be.true();
+        const fsStatUsers = await fs.stat(path.join(userDir,".config.users.json"))
-            localfilesystemSettings.getSettings().then(function(_settings) {
+        const fsStatProjects = await fs.stat(path.join(userDir,".config.projects.json"))
-                _settings.should.eql(settings);
+        const fsStatRuntime = await fs.stat(path.join(userDir,".config.runtime.json"))
-                done();
+
-            }).catch(function(err) {
+        return localfilesystemSettings.init({userDir:userDir}).then(function() {
-                done(err);
+            return new Promise(res => {
+                setTimeout(function() {
+                    res();
+                },10)
             });
-        }).catch(function(err) {
+        }).then(() => {
-            done(err);
+            return localfilesystemSettings.saveSettings({
-        });
+                nodes:{d:4},
+                _credentialSecret: "bar",
+                users:{b:2},
+                projects: {c:3}
+            })
+        }).then(async function() {
+
+            const newFsStatNodes = await fs.stat(path.join(userDir,".config.nodes.json"))
+            const newFsStatUsers = await fs.stat(path.join(userDir,".config.users.json"))
+            const newFsStatProjects = await fs.stat(path.join(userDir,".config.projects.json"))
+            const newFsStatRuntime = await fs.stat(path.join(userDir,".config.runtime.json"))
+
+            // Not changed
+            newFsStatUsers.mtimeMs.should.eql(fsStatUsers.mtimeMs);
+            newFsStatProjects.mtimeMs.should.eql(fsStatProjects.mtimeMs);
+
+            // Changed
+            newFsStatNodes.mtimeMs.should.not.eql(fsStatNodes.mtimeMs);
+            newFsStatRuntime.mtimeMs.should.not.eql(fsStatRuntime.mtimeMs);
+
+        })
     });
 });