frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Handle subflow modules with their own npm dependencies

This commit is contained in:
Nick O'Leary 2020-11-25 19:07:30 +00:00
parent de15a1c36f
commit da96c85d32
No known key found for this signature in database
GPG Key ID: 4F2157149161A6C9
15 changed files with 237 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/node_modules/@node-red/editor-api/lib/admin/nodes.js vendored
View File

@ -60,6 +60,7 @@ module.exports = {
runtimeAPI.nodes.addModule(opts).then(function(info) {
res.json(info);
}).catch(function(err) {
console.log(err.stack);
apiUtils.rejectHandler(req,res,err);
})
},

1
packages/node_modules/@node-red/editor-client/src/js/red.js vendored
View File

@ -386,6 +386,7 @@ var RED = (function() {
}
});
RED.comms.subscribe("notification/node/#",function(topic,msg) {
console.log(topic,msg);
var i,m;
var typeList;
var info;

44
packages/node_modules/@node-red/registry/lib/installer.js vendored
View File

@ -94,20 +94,6 @@ function checkModulePath(folder) {
version: moduleVersion
};
}
function checkExistingModule(module,version) {
var info = registry.getModuleInfo(module);
if (info) {
if (!version || info.version === version) {
var err = new Error("Module already loaded");
err.code = "module_already_loaded";
throw err;
}
return true;
}
return false;
}
async function installModule(module,version,url) {
if (Buffer.isBuffer(module)) {
return installTarball(module)
@ -118,6 +104,7 @@ async function installModule(module,version,url) {
var installName = module;
let isRegistryPackage = true;
var isUpgrade = false;
var isExisting = false;
if (url) {
if (pkgurlRe.test(url) || localtgzRe.test(url)) {
// Git remote url or Tarball url - check the valid package url
@ -158,7 +145,21 @@ async function installModule(module,version,url) {
throw e;
}
}
isUpgrade = checkExistingModule(module,version);
var info = registry.getModuleInfo(module);
if (info) {
if (!info.user) {
log.debug(`Installing existing module: ${module}`)
isExisting = true;
} else if (!version || info.version === version) {
var err = new Error("Module already loaded");
err.code = "module_already_loaded";
throw err;
}
isUpgrade = true;
} else {
isUpgrade = false;
}
if (!isUpgrade) {
log.info(log._("server.install.installing",{name: module,version: version||"latest"}));
@ -172,7 +173,17 @@ async function installModule(module,version,url) {
return exec.run(npmCommand,args,{
cwd: installDir
}, true).then(result => {
if (!isUpgrade) {
if (isExisting) {
// This is a module we already have installed as a non-user module.
// That means it was discovered when loading, but was not listed
// in package.json and has been hidden from the editor.
// The user has requested to install this module. Having run
// the npm install above, it will now be listed in package.json.
// Update the registry to mark it as a user module so it will
// be available to the editor.
log.info(log._("server.install.installed",{name:module}));
return require("./registry").setUserInstalled(module,true).then(reportAddedModules);
} else if (!isUpgrade) {
log.info(log._("server.install.installed",{name:module}));
return require("./index").addModule(module).then(reportAddedModules);
} else {
@ -212,7 +223,6 @@ async function installModule(module,version,url) {
}
function reportAddedModules(info) {
//comms.publish("node/added",info.nodes,false);
if (info.nodes.length > 0) {
log.info(log._("server.added-types"));
for (var i=0;i<info.nodes.length;i++) {

28
packages/node_modules/@node-red/registry/lib/loader.js vendored
View File

@ -308,15 +308,37 @@ function addModule(module) {
throw new Error("Settings unavailable");
}
var nodes = [];
if (registry.getModuleInfo(module)) {
var existingInfo = registry.getModuleInfo(module);
if (existingInfo) {
// TODO: nls
var e = new Error("module_already_loaded");
e.code = "module_already_loaded";
return Promise.reject(e);
}
try {
var moduleFiles = localfilesystem.getModuleFiles(module);
return loadNodeFiles(moduleFiles);
var moduleFiles = {};
var moduleStack = [module];
while(moduleStack.length > 0) {
var moduleToLoad = moduleStack.shift();
var files = localfilesystem.getModuleFiles(moduleToLoad);
if (files[moduleToLoad]) {
moduleFiles[moduleToLoad] = files[moduleToLoad];
if (moduleFiles[moduleToLoad].dependencies) {
log.debug(`Loading dependencies for ${module}`)
for (var i=0; i<moduleFiles[moduleToLoad].dependencies.length; i++) {
var dep = moduleFiles[moduleToLoad].dependencies[i]
if (!registry.getModuleInfo(dep)) {
log.debug(` - load ${dep}`)
moduleStack.push(dep);
} else {
log.debug(` - already loaded ${dep}`)
registry.addModuleDependency(dep,moduleToLoad)
}
}
}
}
}
return loadNodeFiles(moduleFiles).then(() => module)
} catch(err) {
return Promise.reject(err);
}

109
packages/node_modules/@node-red/registry/lib/localfilesystem.js vendored
View File

@ -28,6 +28,7 @@ let loadDenyList = [];
var settings;
var disableNodePathScan = false;
var iconFileExtensions = [".png", ".gif", ".svg"];
var packageList = {};
function init(_settings) {
settings = _settings;
@ -187,9 +188,17 @@ function scanTreeForNodesModules(moduleName) {
var userDir;
if (settings.userDir) {
packageList = getPackageList();
userDir = path.join(settings.userDir,"node_modules");
results = scanDirForNodesModules(userDir,moduleName);
results.forEach(function(r) { r.local = true; });
results.forEach(function(r) {
// If it was found in <userDir>/node_modules then it is considered
// a local module.
// Also check to see if it is listed in the package.json file as a user-installed
// module. This distinguishes modules installed as a dependency
r.local = true;
r.user = !!packageList[r.package.name];
});
}
if (dir) {
@ -288,20 +297,19 @@ function getNodeFiles(disableNodePathScan) {
}
}
var nodeList = {
"node-red": {
name: "node-red",
version: settings.version,
nodes: {},
icons: iconList
}
var nodeList = {};
var coreNodeEntry = {
name: "node-red",
version: settings.version,
nodes: {},
icons: iconList
}
nodeFiles.forEach(function(node) {
nodeList["node-red"].nodes[node.name] = node;
coreNodeEntry.nodes[node.name] = node;
});
if (settings.coreNodesDir) {
var examplesDir = path.join(settings.coreNodesDir,"examples");
nodeList["node-red"].examples = {path: examplesDir};
coreNodeEntry.examples = {path: examplesDir};
}
if (!disableNodePathScan) {
@ -310,7 +318,6 @@ function getNodeFiles(disableNodePathScan) {
// Filter the module list to ignore global modules
// that have also been installed locally - allowing the user to
// update a module they may not otherwise be able to touch
moduleFiles.sort(function(A,B) {
if (A.local && !B.local) {
return -1
@ -323,7 +330,7 @@ function getNodeFiles(disableNodePathScan) {
moduleFiles = moduleFiles.filter(function(mod) {
var result;
if (!knownModules[mod.package.name]) {
knownModules[mod.package.name] = true;
knownModules[mod.package.name] = mod;
result = true;
} else {
result = false;
@ -332,48 +339,62 @@ function getNodeFiles(disableNodePathScan) {
return result;
});
moduleFiles.forEach(function(moduleFile) {
var nodeModuleFiles = getModuleNodeFiles(moduleFile);
nodeList[moduleFile.package.name] = {
name: moduleFile.package.name,
version: moduleFile.package.version,
path: moduleFile.dir,
local: moduleFile.local||false,
nodes: {},
icons: nodeModuleFiles.icons,
examples: nodeModuleFiles.examples
};
if (moduleFile.package['node-red'].version) {
nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
// Do a second pass to check we have all the declared node dependencies
// As this is only done as part of the initial palette load, `knownModules` will
// contain a list of everything discovered during this phase. This means
// we can check for missing dependencies here.
moduleFiles = moduleFiles.filter(function(mod) {
if (Array.isArray(mod.package["node-red"].dependencies)) {
const deps = mod.package["node-red"].dependencies;
const missingDeps = mod.package["node-red"].dependencies.filter(dep => {
if (knownModules[dep]) {
knownModules[dep].usedBy = knownModules[dep].usedBy || [];
knownModules[dep].usedBy.push(mod.package.name)
} else {
return true;
}
})
if (missingDeps.length > 0) {
log.error(`Module: ${mod.package.name} missing dependencies:`);
missingDeps.forEach(m => { log.error(` - ${m}`)});
return false;
}
}
nodeModuleFiles.files.forEach(function(node) {
node.local = moduleFile.local||false;
nodeList[moduleFile.package.name].nodes[node.name] = node;
});
nodeFiles = nodeFiles.concat(nodeModuleFiles.files);
return true;
});
nodeList = convertModuleFileListToObject(moduleFiles);
} else {
// console.log("node path scan disabled");
}
nodeList["node-red"] = coreNodeEntry;
return nodeList;
}
function getModuleFiles(module) {
var nodeList = {};
// Update the package list
var moduleFiles = scanTreeForNodesModules(module);
if (moduleFiles.length === 0) {
var err = new Error(log._("nodes.registry.localfilesystem.module-not-found", {module:module}));
err.code = 'MODULE_NOT_FOUND';
throw err;
}
// Unlike when doing the initial palette load, this call cannot verify the
// dependencies of the new module as it doesn't have visiblity of what
// is in the registry. That will have to be done be the caller in loader.js
return convertModuleFileListToObject(moduleFiles);
}
function convertModuleFileListToObject(moduleFiles) {
const nodeList = {};
moduleFiles.forEach(function(moduleFile) {
var nodeModuleFiles = getModuleNodeFiles(moduleFile);
nodeList[moduleFile.package.name] = {
name: moduleFile.package.name,
version: moduleFile.package.version,
path: moduleFile.dir,
local: moduleFile.local||false,
user: moduleFile.user||false,
nodes: {},
icons: nodeModuleFiles.icons,
examples: nodeModuleFiles.examples
@ -381,7 +402,14 @@ function getModuleFiles(module) {
if (moduleFile.package['node-red'].version) {
nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
}
if (moduleFile.package['node-red'].dependencies) {
nodeList[moduleFile.package.name].dependencies = moduleFile.package['node-red'].dependencies;
}
if (moduleFile.usedBy) {
nodeList[moduleFile.package.name].usedBy = moduleFile.usedBy;
}
nodeModuleFiles.files.forEach(function(node) {
node.local = moduleFile.local||false;
nodeList[moduleFile.package.name].nodes[node.name] = node;
nodeList[moduleFile.package.name].nodes[node.name].local = moduleFile.local || false;
});
@ -412,6 +440,23 @@ function scanIconDir(dir) {
})
return iconList;
}
/**
* Gets the list of modules installed in this runtime as reported by package.json
* Note: these may include non-Node-RED modules
*/
function getPackageList() {
var list = {};
if (settings.userDir) {
try {
var userPackage = path.join(settings.userDir,"package.json");
var pkg = JSON.parse(fs.readFileSync(userPackage,"utf-8"));
return pkg.dependencies;
} catch(err) {
log.error(err);
}
}
return list;
}
module.exports = {
init: init,

80
packages/node_modules/@node-red/registry/lib/registry.js vendored
View File

@ -58,7 +58,8 @@ function filterNodeInfo(n) {
name: n.name,
types: n.types,
enabled: n.enabled,
local: n.local||false
local: n.local||false,
user: n.user || false
};
if (n.hasOwnProperty("module")) {
r.module = n.module;
@ -94,6 +95,7 @@ function saveNodeList() {
name: module,
version: moduleConfigs[module].version,
local: moduleConfigs[module].local||false,
user: moduleConfigs[module].user||false,
nodes: {}
};
if (moduleConfigs[module].hasOwnProperty('pending_version')) {
@ -179,6 +181,7 @@ function loadNodeConfigs() {
function addModule(module) {
moduleNodes[module.name] = [];
moduleConfigs[module.name] = module;
// console.log("registry.js.addModule",module.name,"user?",module.user,"usedBy",module.usedBy,"dependencies",module.dependencies)
for (var setName in module.nodes) {
if (module.nodes.hasOwnProperty(setName)) {
var set = module.nodes[setName];
@ -240,21 +243,47 @@ function removeNode(id) {
return filterNodeInfo(config);
}
function removeModule(module) {
function removeModule(name,skipSave) {
if (!settings.available()) {
throw new Error("Settings unavailable");
}
var nodes = moduleNodes[module];
if (!nodes) {
throw new Error("Unrecognised module: "+module);
}
var infoList = [];
for (var i=0;i<nodes.length;i++) {
infoList.push(removeNode(module+"/"+nodes[i]));
var module = moduleConfigs[name];
var nodes = moduleNodes[name];
if (!nodes) {
throw new Error("Unrecognised module: "+name);
}
if (module.usedBy && module.usedBy > 0) {
// We are removing a module that is used by other modules... so whilst
// this module should be removed from the editor palette, it needs to
// stay in the runtime... for now.
module.user = false;
for (var i=0;i<nodes.length;i++) {
infoList.push(filterNodeInfo(nodes[i]));
}
} else {
if (module.dependencies) {
module.dependencies.forEach(function(dep) {
// Check each dependency of this module to see if it is a non-user-installed
// module that we can expect to disappear once npm uninstall is run
if (!moduleConfigs[dep].user) {
moduleConfigs[dep].usedBy = moduleConfigs[dep].usedBy.filter(m => m !== name);
if (moduleConfigs[dep].usedBy.length === 0) {
// Remove the dependency
removeModule(dep,true);
}
}
});
}
for (var i=0;i<nodes.length;i++) {
infoList.push(removeNode(name+"/"+nodes[i]));
}
delete moduleNodes[name];
delete moduleConfigs[name];
}
if (!skipSave) {
saveNodeList();
}
delete moduleNodes[module];
delete moduleConfigs[module];
saveNodeList();
return infoList;
}
@ -307,6 +336,9 @@ function getNodeList(filter) {
for (var module in moduleConfigs) {
/* istanbul ignore else */
if (moduleConfigs.hasOwnProperty(module)) {
if (!moduleConfigs[module].user && (moduleConfigs[module].usedBy && moduleConfigs[module].usedBy.length > 0)) {
continue;
}
var nodes = moduleConfigs[module].nodes;
for (var node in nodes) {
/* istanbul ignore else */
@ -346,9 +378,13 @@ function getModuleInfo(module) {
name: module,
version: moduleConfigs[module].version,
local: moduleConfigs[module].local,
user: moduleConfigs[module].user,
path: moduleConfigs[module].path,
nodes: []
};
if (moduleConfigs[module].dependencies) {
m.dependencies = moduleConfigs[module].dependencies;
}
if (moduleConfigs[module] && moduleConfigs[module].pending_version) {
m.pending_version = moduleConfigs[module].pending_version;
}
@ -425,7 +461,13 @@ function getAllNodeConfigs(lang) {
var script = "";
for (var i=0;i<nodeList.length;i++) {
var id = nodeList[i];
var config = moduleConfigs[getModule(id)].nodes[getNode(id)];
var module = moduleConfigs[getModule(id)]
if (!module.user && (module.usedBy && module.usedBy.length > 0)) {
continue;
}
var config = module.nodes[getNode(id)];
if (config.enabled && !config.err) {
result += "\n<!-- --- [red-module:"+id+"] --- -->\n";
result += config.config;
@ -587,6 +629,17 @@ function setModulePendingUpdated(module,version) {
});
}
function setUserInstalled(module,userInstalled) {
moduleConfigs[module].user = userInstalled;
return saveNodeList().then(function() {
return getModuleInfo(module);
});
}
function addModuleDependency(module,usedBy) {
moduleConfigs[module].usedBy = moduleConfigs[module].usedBy || [];
moduleConfigs[module].usedBy.push(usedBy);
}
var icon_paths = { };
var iconCache = {};
@ -648,6 +701,9 @@ var registry = module.exports = {
disableNodeSet: disableNodeSet,
setModulePendingUpdated: setModulePendingUpdated,
setUserInstalled: setUserInstalled,
addModuleDependency:addModuleDependency,
removeModule: removeModule,
getNodeInfo: getNodeInfo,

2
packages/node_modules/@node-red/runtime/lib/api/nodes.js vendored
View File

@ -194,7 +194,7 @@ var api = module.exports = {
}
if (opts.module) {
var existingModule = runtime.nodes.getModuleInfo(opts.module);
if (existingModule) {
if (existingModule && existingModule.user) {
if (!opts.version || existingModule.version === opts.version) {
runtime.log.audit({event: "nodes.install",module:opts.module, version:opts.version, error:"module_already_loaded"}, opts.req);
var err = new Error("Module already loaded");

7
packages/node_modules/@node-red/runtime/lib/nodes/index.js vendored
View File

@ -181,11 +181,12 @@ function installModule(module,version,url) {
function uninstallModule(module) {
var info = registry.getModuleInfo(module);
if (!info) {
if (!info || !info.user) {
throw new Error(log._("nodes.index.unrecognised-module", {module:module}));
} else {
for (var i=0;i<info.nodes.length;i++) {
flows.checkTypeInUse(module+"/"+info.nodes[i].name);
var nodeTypesToCheck = info.nodes.map(n => `${module}/${n.name}`);
for (var i=0;i<nodeTypesToCheck.length;i++) {
flows.checkTypeInUse(nodeTypesToCheck[i]);
}
return registry.uninstallModule(module).then(function(list) {
events.emit("runtime-event",{id:"node/removed",retain:false,payload:list});

3
test/resources/subflow/package/README.md Normal file
View File

@ -0,0 +1,3 @@
curl -H "Content-Type: application/json" --request POST --data '{"url":"/Users/nol/code/node-red/node-red/test/resources/subflow/test-subflow-mod-1.0.0.tgz","module":"test-subflow-mod"}' http://localhost:1880/nodes
curl --request DELETE http://localhost:1880/nodes/test-subflow-mod

18
test/resources/subflow/package/package.json Normal file
View File

@ -0,0 +1,18 @@
{
"name": "test-subflow-mod",
"version": "1.0.1",
"description": "",
"keywords": [],
"license": "ISC",
"node-red": {
"nodes": {
"test-subflow": "subflow.js"
},
"dependencies": [
"node-red-node-random"
]
},
"dependencies": {
"node-red-node-random": "*"
}
}

4
test/resources/subflow/package/subflow.js Normal file
View File

@ -0,0 +1,4 @@
module.exports = function(RED) {
RED.nodes.registerSubflow(JSON.parse(require('fs').readFileSync(require("path").join(__dirname,"subflow.json"))))
}

4
test/resources/subflow/package/subflow.json Normal file
View File

@ -0,0 +1,4 @@
{"id":"caf258cc.4e2c48","type":"subflow","name":"Test Subflow","info":"","category":"common","in":[{"x":120,"y":100,"wires":[{"id":"2f1d674f.a02d28"}]}],"out":[{"x":560,"y":100,"wires":[{"id":"1497236e.07f12d","port":0}]},{"x":360,"y":200,"wires":[{"id":"f4334f5f.4905c","port":0}]}],"env":[{"name":"FOO","type":"cred","ui":{"icon":"font-awesome/fa-thermometer-0"}},{"name":"BAR","type":"str","value":"1","ui":{"icon":"font-awesome/fa-thermometer-2","type":"select","opts":{"opts":[{"l":{"en-US":"option 1"},"v":"1"},{"l":{"en-US":"option 2"},"v":"2"},{"l":{"en-US":"option 3"},"v":"3"}]}}},{"name":"onewithaverylongname","type":"str","value":""},{"name":"BARRY","type":"bool","value":"true","ui":{"icon":"font-awesome/fa-thermometer-4","type":"checkbox"}},{"name":"WILMA","type":"num","value":"10","ui":{"icon":"font-awesome/fbomb","type":"spinner"}},{"name":"awg","type":"num","value":"","ui":{"icon":"font-awesome/fa-address-book-o","type":"spinner"}},{"name":"awf","type":"str","value":"","ui":{"type":"select","opts":{"opts":[{"l":{"en-US":"one"},"v":"1"},{"l":{"en-US":"two"},"v":"2"},{"l":{"en-US":"three"},"v":"3"}]}}},{"name":"aawf","type":"bool","value":"true","ui":{"type":"checkbox"}},{"name":"awgawgawg","type":"str","value":"","ui":{"type":"none"}},{"name":"seagseg","type":"str","value":"","ui":{"type":"hide"}}],"meta":{},"color":"#A6BBCF","icon":"font-awesome/fa-space-shuttle","status":{"x":500,"y":300,"wires":[{"id":"8252d1cc.54f94","port":0}]},
"flow":
[{"id":"2f1d674f.a02d28","type":"function","z":"caf258cc.4e2c48","name":"","func":"node.error(\"subflow error \"+msg.payload,msg);\nmsg.payload = {\n FOO: env.get(\"FOO\"),\n BAR: env.get(\"BAR\"),\n WILMA: env.get(\"WILMA\"),\n BARRY: env.get(\"BARRY\")\n}\nnode.warn(\"warning\");\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","x":240,"y":100,"wires":[["1497236e.07f12d"]]},{"id":"f4334f5f.4905c","type":"catch","z":"caf258cc.4e2c48","name":"","scope":null,"uncaught":false,"x":220,"y":200,"wires":[["8252d1cc.54f94"]]},{"id":"8252d1cc.54f94","type":"change","z":"caf258cc.4e2c48","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"error.message","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":350,"y":300,"wires":[[]]},{"id":"1497236e.07f12d","type":"random","z":"caf258cc.4e2c48","name":"","low":"1","high":"10","inte":"true","property":"random","x":420,"y":100,"wires":[[]]},{"id":"876fc49e.f15268","type":"subflow:caf258cc.4e2c48","z":"d607ce33.4fa5a","name":"","x":200,"y":760,"wires":[[],[]]}]
}

BIN
test/resources/subflow/test-subflow-mod-1.0.1.tgz Normal file
View File

Binary file not shown.

2
test/unit/@node-red/registry/lib/installer_spec.js
View File

@ -131,6 +131,7 @@ describe('nodes/registry/installer', function() {
it("rejects when update requested to existing version", function(done) {
sinon.stub(typeRegistry,"getModuleInfo", function() {
return {
user: true,
version: "0.1.1"
}
});
@ -142,6 +143,7 @@ describe('nodes/registry/installer', function() {
it("rejects when update requested to existing version and url", function(done) {
sinon.stub(typeRegistry,"getModuleInfo", function() {
return {
user: true,
version: "0.1.1"
}
});

4
test/unit/@node-red/registry/lib/loader_spec.js
View File

@ -528,7 +528,7 @@ describe("red/nodes/registry/loader",function() {
stubs.push(sinon.stub(nodes,"registerType"));
loader.init({nodes:nodes,log:{info:function(){},_:function(){}},settings:{available:function(){return true;}}});
loader.addModule("TestNodeModule").then(function(result) {
result.should.eql("a node list");
result.should.eql("TestNodeModule");
registry.addModule.called.should.be.true();
var module = registry.addModule.lastCall.args[0];
@ -585,7 +585,7 @@ describe("red/nodes/registry/loader",function() {
stubs.push(sinon.stub(nodes,"registerType"));
loader.init({log:{"_":function(){},warn:function(){}},nodes:nodes,version: function() { return "0.12.0"}, settings:{available:function(){return true;}}});
loader.addModule("TestNodeModule").then(function(result) {
result.should.eql("a node list");
result.should.eql("TestNodeModule");
registry.addModule.called.should.be.false();
nodes.registerType.called.should.be.false();
done();