frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Merge pull request #2704 from node-red/split-config

Browse Source 
Split .config.json into separate files
This commit is contained in:
Nick O'Leary 2020-09-25 15:56:32 +01:00 committed by GitHub
commit 085ff84bc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 201 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/index.js vendored
View File

@ -15,7 +15,6 @@
**/
var fs = require('fs-extra');
var when = require('when');
var fspath = require("path");
var log = require("@node-red/util").log; // TODO: separate module
@ -29,6 +28,11 @@ var projects = require("./projects");
var initialFlowLoadComplete = false;
var settings;
function checkForConfigFile(dir) {
return fs.existsSync(fspath.join(dir,".config.json")) ||
fs.existsSync(fspath.join(dir,".config.nodes.json"))
}
var localfilesystem = {
init: function(_settings, runtime) {
settings = _settings;
@ -36,34 +40,24 @@ var localfilesystem = {
var promises = [];
if (!settings.userDir) {
try {
fs.statSync(fspath.join(process.env.NODE_RED_HOME,".config.json"));
settings.userDir = process.env.NODE_RED_HOME;
} catch(err) {
try {
// Consider compatibility for older versions
if (process.env.HOMEPATH) {
fs.statSync(fspath.join(process.env.HOMEPATH,".node-red",".config.json"));
if (checkForConfigFile(process.env.NODE_RED_HOME)) {
settings.userDir = process.env.NODE_RED_HOME
} else if (process.env.HOMEPATH && checkForConfigFile(fspath.join(process.env.HOMEPATH,".node-red"))) {
settings.userDir = fspath.join(process.env.HOMEPATH,".node-red");
}
} catch(err) {
}
if (!settings.userDir) {
} else {
settings.userDir = fspath.join(process.env.HOME || process.env.USERPROFILE || process.env.HOMEPATH || process.env.NODE_RED_HOME,".node-red");
}
}
if (!settings.readOnly) {
promises.push(fs.ensureDir(fspath.join(settings.userDir,"node_modules")));
}
}
}
}
sessions.init(settings);
runtimeSettings.init(settings);
promises.push(runtimeSettings.init(settings));
promises.push(library.init(settings));
promises.push(projects.init(settings, runtime));
var packageFile = fspath.join(settings.userDir,"package.json");
var packagePromise = when.resolve();
var packagePromise = Promise.resolve();
if (!settings.readOnly) {
packagePromise = function() {
@ -81,7 +75,7 @@ var localfilesystem = {
return true;
}
}
return when.all(promises).then(packagePromise);
return Promise.all(promises).then(packagePromise);
},

113
packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/settings.js vendored
View File

@ -14,41 +14,116 @@
* limitations under the License.
**/
var when = require('when');
var fs = require('fs-extra');
var fspath = require("path");
const fs = require('fs-extra');
const fspath = require("path");
var log = require("@node-red/util").log; // TODO: separate module
var util = require("./util");
const log = require("@node-red/util").log;
const util = require("./util");
const configSections = ['nodes','users','projects'];
const settingsCache = {};
var globalSettingsFile;
var globalSettingsBackup;
var settings;
async function migrateToMultipleConfigFiles() {
const nodesFilename = getSettingsFilename("nodes");
if (fs.existsSync(nodesFilename)) {
// We have both .config.json and .config.nodes.json
// Use the more recently modified. This handles users going back to pre1.2
// and up again.
// We can remove this logic in 1.3+ and remove the old .config.json file entirely
//
const fsStatNodes = await fs.stat(nodesFilename);
const fsStatGlobal = await fs.stat(globalSettingsFile);
if (fsStatNodes.mtimeMs > fsStatGlobal.mtimeMs) {
// .config.nodes.json is newer than .config.json - no migration needed
return;
}
}
const data = await util.readFile(globalSettingsFile,globalSettingsBackup,{});
// In a later release we should remove the old settings file. But don't do
// that *yet* otherwise users won't be able to downgrade easily.
return writeSettings(data) // .then( () => fs.remove(globalSettingsFile) );
}
/**
* Takes the single settings object and splits it into separate files. This makes
* it easier to backup selected parts of the settings and also helps reduce the blast
* radius if a file is lost.
*
* The settings are written to four files:
* - .config.nodes.json - the node registry
* - .config.users.json - user specific settings (eg editor settings)
* - .config.projects.json - project settings, including the active project
* - .config.runtime.json - everything else - most notable _credentialSecret
*/
function writeSettings(data) {
const configKeys = Object.keys(data);
const writePromises = [];
configSections.forEach(key => {
const sectionData = data[key] || {};
delete data[key];
const sectionFilename = getSettingsFilename(key);
const sectionContent = JSON.stringify(sectionData,null,4);
if (sectionContent !== settingsCache[key]) {
settingsCache[key] = sectionContent;
writePromises.push(util.writeFile(sectionFilename,sectionContent,sectionFilename+".backup"))
}
})
// Having extracted nodes/users/projects, write whatever is left to the runtime config
const sectionFilename = getSettingsFilename("runtime");
const sectionContent = JSON.stringify(data,null,4);
if (sectionContent !== settingsCache["runtime"]) {
settingsCache["runtime"] = sectionContent;
writePromises.push(util.writeFile(sectionFilename,sectionContent,sectionFilename+".backup"));
}
return Promise.all(writePromises);
}
async function readSettings() {
// Read the 'runtime' settings file first
const runtimeFilename = getSettingsFilename("runtime");
const result = await util.readFile(runtimeFilename,runtimeFilename+".backup",{});
const readPromises = [];
// Read the other settings files and add them into the runtime settings
configSections.forEach(key => {
const sectionFilename = getSettingsFilename(key);
readPromises.push(util.readFile(sectionFilename,sectionFilename+".backup",{}).then(sectionData => {
if (Object.keys(sectionData).length > 0) {
result[key] = sectionData;
}
}))
});
return Promise.all(readPromises).then(() => result);
}
function getSettingsFilename(section) {
return fspath.join(settings.userDir,`.config.${section}.json`);
}
module.exports = {
init: function(_settings) {
settings = _settings;
globalSettingsFile = fspath.join(settings.userDir,".config.json");
globalSettingsBackup = fspath.join(settings.userDir,".config.json.backup");
if (fs.existsSync(globalSettingsFile) && !settings.readOnly) {
return migrateToMultipleConfigFiles();
} else {
return Promise.resolve();
}
},
getSettings: function() {
return when.promise(function(resolve,reject) {
fs.readFile(globalSettingsFile,'utf8',function(err,data) {
if (!err) {
try {
return resolve(util.parseJSON(data));
} catch(err2) {
log.trace("Corrupted config detected - resetting");
}
}
return resolve({});
})
})
return readSettings()
},
saveSettings: function(newSettings) {
if (settings.readOnly) {
return when.resolve();
return Promise.resolve();
}
return util.writeFile(globalSettingsFile,JSON.stringify(newSettings,null,1),globalSettingsBackup);
return writeSettings(newSettings);
}
}

9
packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/util.js vendored
View File

@ -14,11 +14,10 @@
* limitations under the License.
**/
var fs = require('fs-extra');
var fspath = require('path');
var when = require('when');
const fs = require('fs-extra');
const fspath = require('path');
var log = require("@node-red/util").log; // TODO: separate module
const log = require("@node-red/util").log;
function parseJSON(data) {
if (data.charCodeAt(0) === 0xFEFF) {
@ -27,7 +26,7 @@ function parseJSON(data) {
return JSON.parse(data);
}
function readFile(path,backupPath,emptyResponse,type) {
return when.promise(function(resolve) {
return new Promise(function(resolve) {
fs.readFile(path,'utf8',function(err,data) {
if (!err) {
if (data.length === 0) {

123
test/unit/@node-red/runtime/lib/storage/localfilesystem/settings_spec.js
View File

@ -14,9 +14,9 @@
* limitations under the License.
**/
var should = require("should");
var fs = require('fs-extra');
var path = require('path');
const should = require("should");
const fs = require('fs-extra');
const path = require('path');
var NR_TEST_UTILS = require("nr-test-utils");
@ -34,49 +34,100 @@ describe('storage/localfilesystem/settings', function() {
});
it('should handle non-existent settings', function(done) {
var settingsFile = path.join(userDir,".settings.json");
localfilesystemSettings.init({userDir:userDir});
var settingsFile = path.join(userDir,".config.json");
localfilesystemSettings.init({userDir:userDir}).then(function() {
fs.existsSync(settingsFile).should.be.false();
localfilesystemSettings.getSettings().then(function(settings) {
return localfilesystemSettings.getSettings();
}).then(function(settings) {
settings.should.eql({});
done();
}).catch(function(err) {
done(err);
});
}).catch(err => { done(err)});
});
it('should handle corrupt settings', function(done) {
it('should migrate single config.json to multiple files', function(done) {
var settingsFile = path.join(userDir,".config.json");
fs.writeFileSync(settingsFile,"[This is not json","utf8");
localfilesystemSettings.init({userDir:userDir});
fs.existsSync(settingsFile).should.be.true();
localfilesystemSettings.getSettings().then(function(settings) {
settings.should.eql({});
done();
}).catch(function(err) {
done(err);
});
});
fs.writeFileSync(settingsFile,JSON.stringify({
nodes:{a:1},
_credentialSecret: "foo",
users:{b:2},
projects: {c:3}
}),"utf8");
it('should handle settings', function(done) {
var settingsFile = path.join(userDir,".config.json");
async function checkFile(sectionName, expectedContents) {
const file = path.join(userDir,".config."+sectionName+".json");
fs.existsSync(file).should.be.true();
var contents = await fs.readFile(file,'utf8');
var data = JSON.parse(contents);
data.should.eql(expectedContents)
}
localfilesystemSettings.init({userDir:userDir});
localfilesystemSettings.init({userDir:userDir}).then(async function() {
fs.existsSync(settingsFile).should.be.false();
var settings = {"abc":{"type":"creds"}};
localfilesystemSettings.saveSettings(settings).then(function() {
fs.existsSync(settingsFile).should.be.true();
localfilesystemSettings.getSettings().then(function(_settings) {
_settings.should.eql(settings);
await checkFile("nodes",{a:1})
await checkFile("users",{b:2})
await checkFile("projects",{c:3})
await checkFile("runtime",{_credentialSecret:"foo"})
done();
}).catch(function(err) {
done(err);
});
}).catch(function(err) {
done(err);
}).catch(err => { done(err)});
});
it('should load separate settings file', async function() {
await fs.writeFile( path.join(userDir,".config.nodes.json"),JSON.stringify({a:1}),"utf8");
await fs.writeFile( path.join(userDir,".config.users.json"),JSON.stringify({b:2}),"utf8");
await fs.writeFile( path.join(userDir,".config.projects.json"),JSON.stringify({c:3}),"utf8");
await fs.writeFile( path.join(userDir,".config.runtime.json"),JSON.stringify({_credentialSecret:"foo"}),"utf8");
return localfilesystemSettings.init({userDir:userDir})
.then(localfilesystemSettings.getSettings)
.then(settings => {
settings.should.eql({
nodes:{a:1},
_credentialSecret: "foo",
users:{b:2},
projects: {c:3}
})
})
});
it('should write only the files that need writing', async function() {
await fs.writeFile( path.join(userDir,".config.nodes.json"),JSON.stringify({a:1}),"utf8");
await fs.writeFile( path.join(userDir,".config.users.json"),JSON.stringify({b:2}),"utf8");
await fs.writeFile( path.join(userDir,".config.projects.json"),JSON.stringify({c:3}),"utf8");
await fs.writeFile( path.join(userDir,".config.runtime.json"),JSON.stringify({_credentialSecret:"foo"}),"utf8");
const fsStatNodes = await fs.stat(path.join(userDir,".config.nodes.json"))
const fsStatUsers = await fs.stat(path.join(userDir,".config.users.json"))
const fsStatProjects = await fs.stat(path.join(userDir,".config.projects.json"))
const fsStatRuntime = await fs.stat(path.join(userDir,".config.runtime.json"))
return localfilesystemSettings.init({userDir:userDir}).then(function() {
return new Promise(res => {
setTimeout(function() {
res();
},10)
});
}).then(() => {
return localfilesystemSettings.saveSettings({
nodes:{d:4},
_credentialSecret: "bar",
users:{b:2},
projects: {c:3}
})
}).then(async function() {
const newFsStatNodes = await fs.stat(path.join(userDir,".config.nodes.json"))
const newFsStatUsers = await fs.stat(path.join(userDir,".config.users.json"))
const newFsStatProjects = await fs.stat(path.join(userDir,".config.projects.json"))
const newFsStatRuntime = await fs.stat(path.join(userDir,".config.runtime.json"))
// Not changed
newFsStatUsers.mtimeMs.should.eql(fsStatUsers.mtimeMs);
newFsStatProjects.mtimeMs.should.eql(fsStatProjects.mtimeMs);
// Changed
newFsStatNodes.mtimeMs.should.not.eql(fsStatNodes.mtimeMs);
newFsStatRuntime.mtimeMs.should.not.eql(fsStatRuntime.mtimeMs);
})
});
});