From d51aefa1563005247c23c11973fb97b13a404b1e Mon Sep 17 00:00:00 2001 From: Hiroyasu Nishiyama Date: Mon, 11 Jan 2021 19:32:16 +0900 Subject: [PATCH 01/16] initial support for npm module installation --- .../nodes/core/function/10-function.html | 343 +++++++++++++++++- .../nodes/core/function/10-function.js | 323 ++++++++++------- .../nodes/locales/en-US/messages.json | 7 +- .../@node-red/registry/lib/util.js | 6 +- .../@node-red/runtime/lib/nodes/index.js | 14 +- .../@node-red/runtime/lib/nodes/npmModule.js | 281 ++++++++++++++ 6 files changed, 829 insertions(+), 145 deletions(-) create mode 100644 packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index f0e5c7d68..74f2c0f75 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -1,3 +1,279 @@ + + + + + @@ -64,7 +349,8 @@ outputs: {value:1}, noerr: {value:0,required:true,validate:function(v) { return !v; }}, initialize: {value:""}, - finalize: {value:""} + finalize: {value:""}, + libs: {value:"", type:"library-config", required:false} }, inputs:1, outputs:1, @@ -97,6 +383,11 @@ id: "func-tab-finalize", label: that._("function.label.finalize") }); + tabs.addTab({ + id: "func-tab-config", + label: "Config" + }); + tabs.activateTab("func-tab-body"); $( "#node-input-outputs" ).spinner({ @@ -205,7 +496,56 @@ RED.popover.tooltip($("#node-function-expand-js"), RED._("node-red:common.label.expand")); RED.popover.tooltip($("#node-finalize-expand-js"), RED._("node-red:common.label.expand")); + $("#node-input-require-container").css('min-height','250px').css('min-width','450px').editableList({ + addItem: function(container,i,opt) { + var row = $("
").appendTo(container); + var fvar = $("", { + class: "node-input-require-var", + placeholder: RED._("node-red:function.require.var"), + type: "text", + disabled: true + }).css({ + width: "130px", + "margin-left": "5px" + }).appendTo(row); + var fmodule = $("", { + class: "node-input-require-val", + placeholder: RED._("node-red:function.require.module"), + type: "text", + disabled: true + }).css({ + width: "390px", + "margin-left": "5px" + }).appendTo(row); + if (opt) { + if (opt.vname) { + fvar.val(opt.vname); + } + if (opt.name) { + fmodule.val(opt.name); + } + } + }, + addButton: false, + removable: false + }); + function updateLibs() { + var id = $("#node-input-libs").val(); + var node = RED.nodes.node(id); + if (node && node.libs) { + var libs = node.libs; + $("#node-input-require-container").editableList("empty"); + libs.forEach(function (r) { + $("#node-input-require-container").editableList("addItem", r); + }); + } + } + + updateLibs(); + $("#node-input-libs").on("change", function () { + updateLibs(); + }); }, oneditsave: function() { var node = this; @@ -271,6 +611,7 @@ this.editor.resize(); this.finalizeEditor.resize(); + $("#node-input-require-container").css("height", (height -155)+"px"); } }); diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index eb07c319d..708cdba0b 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -16,6 +16,19 @@ module.exports = function(RED) { "use strict"; + + function LibsConfigNode(n) { + RED.nodes.createNode(this, n); + this.name = n.name; + this.libs = n.libs; + } + RED.nodes.registerType("library-config", LibsConfigNode); + + RED.httpNode.get("/function/modules", function (req, res) { + var list = RED.nodes.listNPMModules(); + res.send(list); + }); + var util = require("util"); var vm = require("vm"); @@ -88,12 +101,16 @@ module.exports = function(RED) { } function FunctionNode(n) { - RED.nodes.createNode(this,n); + var libConf = RED.nodes.getNode(n.libs); + var libs = libConf ? libConf.libs : []; + n.modules = libs.map(x => x.name).filter(x => (x && (x !== ""))); + var loadPromise = RED.nodes.createNode(this,n); var node = this; node.name = n.name; node.func = n.func; node.ini = n.initialize ? n.initialize.trim() : ""; node.fin = n.finalize ? n.finalize.trim() : ""; + node.libs = libs; var handleNodeDoneCall = true; @@ -105,23 +122,23 @@ module.exports = function(RED) { } var functionText = "var results = null;"+ - "results = (async function(msg,__send__,__done__){ "+ - "var __msgid__ = msg._msgid;"+ - "var node = {"+ - "id:__node__.id,"+ - "name:__node__.name,"+ - "log:__node__.log,"+ - "error:__node__.error,"+ - "warn:__node__.warn,"+ - "debug:__node__.debug,"+ - "trace:__node__.trace,"+ - "on:__node__.on,"+ - "status:__node__.status,"+ - "send:function(msgs,cloneMsg){ __node__.send(__send__,__msgid__,msgs,cloneMsg);},"+ - "done:__done__"+ - "};\n"+ - node.func+"\n"+ - "})(msg,__send__,__done__);"; + "results = (async function(msg,__send__,__done__){ "+ + "var __msgid__ = msg._msgid;"+ + "var node = {"+ + "id:__node__.id,"+ + "name:__node__.name,"+ + "log:__node__.log,"+ + "error:__node__.error,"+ + "warn:__node__.warn,"+ + "debug:__node__.debug,"+ + "trace:__node__.trace,"+ + "on:__node__.on,"+ + "status:__node__.status,"+ + "send:function(msgs,cloneMsg){ __node__.send(__send__,__msgid__,msgs,cloneMsg);},"+ + "done:__done__"+ + "};\n"+ + node.func+"\n"+ + "})(msg,__send__,__done__);"; var finScript = null; var finOpt = null; node.topic = n.topic; @@ -266,12 +283,47 @@ module.exports = function(RED) { }; sandbox.promisify = util.promisify; } - var context = vm.createContext(sandbox); - try { - var iniScript = null; - var iniOpt = null; - if (node.ini && (node.ini !== "")) { - var iniText = ` + + const RESOLVING = 0; + const RESOLVED = 1; + const ERROR = 2; + var state = RESOLVING; + var messages = []; + var processMessage = (() => {}); + + node.on("input", function(msg,send,done) { + if(state === RESOLVING) { + messages.push({msg:msg, send:send, done:done}); + } + else if(state === RESOLVED) { + processMessage(msg, send, done); + } + }); + + // wait for module installation + loadPromise.then(function () { + if (node.hasOwnProperty("libs")) { + var modules = node.libs; + modules.forEach(module => { + var vname = module.hasOwnProperty("vname") ? module.vname : null; + if (vname && (vname !== "")) { + sandbox[vname] = null; + try { + var lib = RED.require(module.name); + sandbox[vname] = lib; + } + catch (e) { + node.warn("failed to load library: "+ module.name); + } + } + }); + } + var context = vm.createContext(sandbox); + try { + var iniScript = null; + var iniOpt = null; + if (node.ini && (node.ini !== "")) { + var iniText = ` (async function(__send__) { var node = { id:__node__.id, @@ -288,141 +340,130 @@ module.exports = function(RED) { }; `+ node.ini +` })(__initSend__);`; - iniOpt = createVMOpt(node, " setup"); - iniScript = new vm.Script(iniText, iniOpt); - } - node.script = vm.createScript(functionText, createVMOpt(node, "")); - if (node.fin && (node.fin !== "")) { - var finText = "(function () {\n"+node.fin +"\n})();"; - finOpt = createVMOpt(node, " cleanup"); - finScript = new vm.Script(finText, finOpt); - } - var promise = Promise.resolve(); - if (iniScript) { - context.__initSend__ = function(msgs) { node.send(msgs); }; - promise = iniScript.runInContext(context, iniOpt); - } + iniOpt = createVMOpt(node, " setup"); + iniScript = new vm.Script(iniText, iniOpt); + } + node.script = vm.createScript(functionText, createVMOpt(node, "")); + if (node.fin && (node.fin !== "")) { + var finText = "(function () {\n"+node.fin +"\n})();"; + finOpt = createVMOpt(node, " cleanup"); + finScript = new vm.Script(finText, finOpt); + } + var promise = Promise.resolve(); + if (iniScript) { + context.__initSend__ = function(msgs) { node.send(msgs); }; + promise = iniScript.runInContext(context, iniOpt); + } - function processMessage(msg, send, done) { - var start = process.hrtime(); - context.msg = msg; - context.__send__ = send; - context.__done__ = done; + processMessage = function (msg, send, done) { + var start = process.hrtime(); + context.msg = msg; + context.__send__ = send; + context.__done__ = done; - node.script.runInContext(context); - context.results.then(function(results) { - sendResults(node,send,msg._msgid,results,false); - if (handleNodeDoneCall) { - done(); - } + node.script.runInContext(context); + context.results.then(function(results) { + sendResults(node,send,msg._msgid,results,false); + if (handleNodeDoneCall) { + done(); + } - var duration = process.hrtime(start); - var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; - node.metric("duration", msg, converted); - if (process.env.NODE_RED_FUNCTION_TIME) { - node.status({fill:"yellow",shape:"dot",text:""+converted}); - } - }).catch(err => { - if ((typeof err === "object") && err.hasOwnProperty("stack")) { - //remove unwanted part - var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); - err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); - var stack = err.stack.split(/\r?\n/); + var duration = process.hrtime(start); + var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; + node.metric("duration", msg, converted); + if (process.env.NODE_RED_FUNCTION_TIME) { + node.status({fill:"yellow",shape:"dot",text:""+converted}); + } + }).catch(err => { + if ((typeof err === "object") && err.hasOwnProperty("stack")) { + //remove unwanted part + var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); + err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); + var stack = err.stack.split(/\r?\n/); - //store the error in msg to be used in flows - msg.error = err; + //store the error in msg to be used in flows + msg.error = err; - var line = 0; - var errorMessage; - if (stack.length > 0) { - while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { - line++; - } + var line = 0; + var errorMessage; + if (stack.length > 0) { + while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { + line++; + } - if (line < stack.length) { - errorMessage = stack[line]; - var m = /:(\d+):(\d+)$/.exec(stack[line+1]); - if (m) { - var lineno = Number(m[1])-1; - var cha = m[2]; - errorMessage += " (line "+lineno+", col "+cha+")"; + if (line < stack.length) { + errorMessage = stack[line]; + var m = /:(\d+):(\d+)$/.exec(stack[line+1]); + if (m) { + var lineno = Number(m[1])-1; + var cha = m[2]; + errorMessage += " (line "+lineno+", col "+cha+")"; + } } } + if (!errorMessage) { + errorMessage = err.toString(); + } + done(errorMessage); } - if (!errorMessage) { - errorMessage = err.toString(); + else if (typeof err === "string") { + done(err); + } + else { + done(JSON.stringify(err)); + } + }); + } + + node.on("close", function() { + if (finScript) { + try { + finScript.runInContext(context, finOpt); + } + catch (err) { + node.error(err); } - done(errorMessage); } - else if (typeof err === "string") { - done(err); + while (node.outstandingTimers.length > 0) { + clearTimeout(node.outstandingTimers.pop()); } - else { - done(JSON.stringify(err)); + while (node.outstandingIntervals.length > 0) { + clearInterval(node.outstandingIntervals.pop()); + } + if (node.clearStatus) { + node.status({}); } }); - } - const RESOLVING = 0; - const RESOLVED = 1; - const ERROR = 2; - var state = RESOLVING; - var messages = []; - - node.on("input", function(msg,send,done) { - if(state === RESOLVING) { - messages.push({msg:msg, send:send, done:done}); - } - else if(state === RESOLVED) { - processMessage(msg, send, done); - } - }); - node.on("close", function() { - if (finScript) { - try { - finScript.runInContext(context, finOpt); - } - catch (err) { - node.error(err); - } - } - while (node.outstandingTimers.length > 0) { - clearTimeout(node.outstandingTimers.pop()); - } - while (node.outstandingIntervals.length > 0) { - clearInterval(node.outstandingIntervals.pop()); - } - if (node.clearStatus) { - node.status({}); - } - }); - - promise.then(function (v) { - var msgs = messages; - messages = []; - while (msgs.length > 0) { - msgs.forEach(function (s) { - processMessage(s.msg, s.send, s.done); - }); - msgs = messages; + promise.then(function (v) { + var msgs = messages; messages = []; - } - state = RESOLVED; - }).catch((error) => { - messages = []; - state = ERROR; - node.error(error); - }); + while (msgs.length > 0) { + msgs.forEach(function (s) { + processMessage(s.msg, s.send, s.done); + }); + msgs = messages; + messages = []; + } + state = RESOLVED; + }).catch((error) => { + messages = []; + state = ERROR; + node.error(error); + }); - } - catch(err) { - // eg SyntaxError - which v8 doesn't include line number information - // so we can't do better than this - updateErrorInfo(err); - node.error(err); - } + } + catch(err) { + // eg SyntaxError - which v8 doesn't include line number information + // so we can't do better than this + updateErrorInfo(err); + node.error(err); + } + }); } - RED.nodes.registerType("function",FunctionNode); + RED.nodes.registerType("function",FunctionNode, { + dynamicModuleList: "modules" + }); RED.library.register("functions"); }; diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index eb623ce20..a2e529d16 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -212,12 +212,17 @@ "function": "Function", "initialize": "Setup", "finalize": "Close", - "outputs": "Outputs" + "outputs": "Outputs", + "require": "Require" }, "text": { "initialize": "// Code added here will be run once\n// whenever the node is deployed.\n", "finalize": "// Code added here will be run when the\n// node is being stopped or re-deployed.\n" }, + "require": { + "var": "name", + "module": "module" + }, "error": { "inputListener":"Cannot add listener to 'input' event within Function", "non-message-returned":"Function tried to send a message of type __type__" diff --git a/packages/node_modules/@node-red/registry/lib/util.js b/packages/node_modules/@node-red/registry/lib/util.js index 94690d6cb..8dda6ece5 100644 --- a/packages/node_modules/@node-red/registry/lib/util.js +++ b/packages/node_modules/@node-red/registry/lib/util.js @@ -45,6 +45,10 @@ function requireModule(name) { var relPath = path.relative(__dirname, moduleInfo.path); return require(relPath); } else { + var npm = runtime.nodes.loadNPMModule(name); + if (npm) { + return npm; + } var err = new Error(`Cannot find module '${name}'`); err.code = "MODULE_NOT_FOUND"; throw err; @@ -79,7 +83,7 @@ function createNodeApi(node) { httpAdmin: runtime.adminApp, server: runtime.server } - copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials" ]); + copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials", "listNPMModules"]); red.nodes.registerType = function(type,constructor,opts) { runtime.nodes.registerType(node.id,type,constructor,opts); } diff --git a/packages/node_modules/@node-red/runtime/lib/nodes/index.js b/packages/node_modules/@node-red/runtime/lib/nodes/index.js index 73e7baee8..f9323be18 100644 --- a/packages/node_modules/@node-red/runtime/lib/nodes/index.js +++ b/packages/node_modules/@node-red/runtime/lib/nodes/index.js @@ -26,6 +26,7 @@ var flows = require("../flows"); var flowUtil = require("../flows/util") var context = require("./context"); var Node = require("./Node"); +var npmModule = require("./npmModule"); var log; const events = require("@node-red/util").events; @@ -49,6 +50,7 @@ function registerType(nodeSet,type,constructor,opts) { type = nodeSet; nodeSet = ""; } + var dynModule = null; if (opts) { if (opts.credentials) { credentials.register(type,opts.credentials); @@ -60,7 +62,11 @@ function registerType(nodeSet,type,constructor,opts) { log.warn("["+type+"] "+err.message); } } + if (opts.dynamicModuleList) { + dynModule = opts.dynamicModuleList; + } } + npmModule.register(type, dynModule); if(!(constructor.prototype instanceof Node)) { if(Object.getPrototypeOf(constructor.prototype) === Object.prototype) { util.inherits(constructor,Node); @@ -110,6 +116,7 @@ function createNode(node,def) { } else if (credentials.getDefinition(node.type)) { node.credentials = {}; } + return npmModule.checkInstall(def); } function registerSubflow(nodeSet, subflow) { @@ -138,6 +145,7 @@ function init(runtime) { flows.init(runtime); registry.init(runtime); context.init(runtime.settings); + npmModule.init(runtime); } function disableNode(id) { @@ -261,5 +269,9 @@ module.exports = { // Contexts loadContextsPlugin: context.load, closeContextsPlugin: context.close, - listContextStores: context.listStores + listContextStores: context.listStores, + + // NPM modules + listNPMModules: npmModule.list, + loadNPMModule: npmModule.load }; diff --git a/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js b/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js new file mode 100644 index 000000000..ae976de48 --- /dev/null +++ b/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js @@ -0,0 +1,281 @@ +/** + * Copyright JS Foundation and other contributors, http://js.foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +var path = require("path"); +var fs = require("fs-extra"); +var os = require("os"); +var util = require("@node-red/registry/lib/util"); + +var api; + +var runtime; +var settings; +var exec; +var log; + +var npmCommand = (process.platform === "win32") ? "npm.cmd" : "npm"; + +var moduleProp = {}; +var moduleBase = null; + +var allowInstall = true; +var allowList = ["*"]; +var denyList = []; + +/** + * Initialise npm install module. + * @param {Object} _runtime - runtime object + */ +function init(_runtime) { + runtime = _runtime; + settings = _runtime.settings; + exec = _runtime.exec; + log = _runtime.log; + + moduleProp = {}; + moduleBase = settings.userDir || process.env.NODE_RED_HOME || "."; + + if (settings.hasOwnProperty("externalModules")) { + var em = settings.externalModules; + if (em && em.hasOwnProperty("modules")) { + var mod = em.modules; + if (mod.hasOwnProperty("allowInstall")) { + allowInstall = mod.allowInstall; + } + if (mod.hasOwnProperty("allowList")) { + var alist = mod.allowList; + if (Array.isArray(alist)) { + allowList = alist; + } + else { + log.warn("unexpected value of externalModule.allowList in settings.js"); + } + } + if (mod.hasOwnProperty("denyList")) { + var dlist = mod.denyList; + if (Array.isArray(dlist)) { + denyList = dlist; + } + else { + log.warn("unexpected value of externalModule.denyList in settings.js"); + } + } + } + } +} + +/** + * Register dynamic module installation property. + * @param {string} type - node type + * @param {string} prop - property name + */ +function register(type, prop) { + if (prop) { + moduleProp[type] = prop; + } + else { + delete moduleProp[prop] + } +} + +/** + * Get path to install modules + */ +function modulePath() { // takes variable length arguments in `arguments` + var result = moduleBase; + result = path.join(result, "lib", "node_modules"); + for(var i = 0; i < arguments.length; i++) { + result = path.join(result, arguments[i]); + } + return result; +} + +/** + * Decompose NPM module specification string + * @param {string} module - module specification + */ +function moduleName(module) { + var match = /^([^@]+@.+)/.exec(module); + if (match) { + return [match[1], match[2]]; + } + return [module, undefined]; +} + +/** + * Get NPM module package info + * @param {string} name - module name + * @param {string} name - module version + */ +function infoNPM(name, ver) { + var path = modulePath(name, "package.json"); + try { + var pkg = require(path); + return pkg; + } + catch (e) { + } + return null; +} + +/** + * Ensure existance of module installation directory + */ +function ensureLibDirectory() { + var path = modulePath(); + if (!fs.existsSync(path)) { + fs.mkdirSync(path, { + recursive: true + }); + return fs.existsSync(path); + } + return true; +} + +/** + * Install NPM module + * @param {string} module - module specification + */ +function installNPM(module) { + var [name, ver] = moduleName(module); + if (!ensureLibDirectory()) { + log.warn("failed to install: "+name); + return; + } + var pkg = infoNPM(name, ver); + if (!pkg) { + var args = ["install", module]; + var dir = modulePath(); + return exec.run(npmCommand, args, { + cwd: dir + }, true).then(result => { + if (result && (result.code === 0)) { + log.info("successfully installed: "+name); + } + else { + log.warn("failed to install: "+name); + } + }).catch(e => { + var msg = e.hasOwnProperty("stderr") ? e.stderr : e; + log.warn("failed to install: "+name); + }); + } + else { + log.info("already installed: "+name); + } + return Promise.resolve(); +} + +/** + * Check allowance of NPM module installation + * @param {string} name - module specification + */ +function isAllowed(name) { + if (!allowInstall) { + return false; + } + var [module, ver] = moduleName(name); + var aList = util.parseModuleList(allowList); + var dList = util.parseModuleList(denyList); + return util.checkModuleAllowed(module, ver, aList, dList); +} + +/** + * Check and install NPM module according to dynamic module specification + * @param {Object} node - node object + */ +function checkInstall(node) { + var name = null; + if(moduleProp.hasOwnProperty(node.type)) { + name = moduleProp[node.type]; + } + var promises = []; + if (name && node.hasOwnProperty(name)) { + var modules = node[name]; + modules.forEach(module => { + var name = module; + if ((typeof module === "object") && + module && + module.hasOwnProperty("name")) { + name = module.name; + } + if (isAllowed(name)) { + promises.push(installNPM(name)); + } + else { + log.info("installation not allowed: "+name); + } + }); + } + return Promise.all(promises); +} + +/** + * Load NPM module + * @param {string} module - module to load + */ +function load(module) { + try { + var [name, ver] = moduleName(module); + var path = modulePath(name); + var npm = require(path); + return npm; + } + catch (e) { + return null; + } +} + +/** + * Get list of installed modules + */ +function listModules() { + var modPath = modulePath(); + if (!fs.existsSync(modPath)) { + return []; + } + var dir = fs.opendirSync(modPath); + var modules = []; + if (dir) { + var ent = null; + while (ent = dir.readSync()) { + var name = ent.name; + if (ent.isDirectory() && + (name[0] !== ".")) { + var pkgPath = path.join(modPath, name, "package.json"); + if (fs.existsSync(pkgPath)) { + var pkg = fs.readJSONSync(pkgPath); + var info = { + name: pkg.name, + version: pkg.version + }; + modules.push(info); + } + } + } + dir.closeSync(); + } + return modules; +} + +api = { + init: init, + register: register, + checkInstall: checkInstall, + load: load, + list: listModules +}; +module.exports = api; From 4a1d66f210d563253ebf9423148dee6198e096db Mon Sep 17 00:00:00 2001 From: Hiroyasu Nishiyama Date: Wed, 27 Jan 2021 22:27:54 +0900 Subject: [PATCH 02/16] update UI, Runtime API, metadata handling, and others --- .../@node-red/editor-api/lib/admin/index.js | 5 + .../@node-red/editor-api/lib/admin/nodes.js | 40 + .../nodes/core/function/10-function.html | 884 +++++++++++------- .../nodes/core/function/10-function.js | 29 +- .../@node-red/registry/lib/util.js | 2 +- .../@node-red/runtime/lib/api/nodes.js | 42 + .../@node-red/runtime/lib/index.js | 1 + .../@node-red/runtime/lib/nodes/index.js | 2 + .../@node-red/runtime/lib/nodes/npmModule.js | 315 +++++-- 9 files changed, 900 insertions(+), 420 deletions(-) diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/index.js b/packages/node_modules/@node-red/editor-api/lib/admin/index.js index 34c47b2cb..5de09acad 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/index.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/index.js @@ -67,6 +67,11 @@ module.exports = { adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.read"),nodes.getSet,apiUtil.errorHandler); adminApp.put(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.write"),nodes.putSet,apiUtil.errorHandler); + // NPM Modules + adminApp.get("/modules", needsPermission("nodes.read"), nodes.listNPMModules, apiUtil.errorHandler); + adminApp.delete("/modules/:spec", needsPermission("nodes.write"), nodes.uninstallNPMModule, apiUtil.errorHandler); + adminApp.post("/modules", needsPermission("nodes.write"), nodes.updateNPMModule, apiUtil.errorHandler); + // Context adminApp.get("/context/:scope(global)",needsPermission("context.read"),context.get,apiUtil.errorHandler); adminApp.get("/context/:scope(global)/*",needsPermission("context.read"),context.get,apiUtil.errorHandler); diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js index 058053a29..77efedd11 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js @@ -191,5 +191,45 @@ module.exports = { runtimeAPI.nodes.getIconList(opts).then(function(list) { res.json(list); }); + }, + + listNPMModules: function(req, res) { + var opts = { + user: req.user, + req: apiUtils.getRequestLogObject(req) + } + runtimeAPI.nodes.listNPMModules(opts).then(function(list) { + res.json(list); + }).catch(err => { + apiUtils.rejectHandler(req,res,err); + });; + }, + + uninstallNPMModule: function(req, res) { + var opts = { + user: req.user, + spec: req.params.spec, + req: apiUtils.getRequestLogObject(req) + } + runtimeAPI.nodes.uninstallNPMModule(opts).then(function(result) { + res.json(result); + }).catch(err => { + apiUtils.rejectHandler(req,res,err); + }); + }, + + updateNPMModule: function(req, res) { + var body = req.body; + var opts = { + user: req.user, + spec: body.spec, + update: body.update, + req: apiUtils.getRequestLogObject(req) + } + runtimeAPI.nodes.updateNPMModule(opts).then(function(result) { + res.json(result); + }).catch(err => { + apiUtils.rejectHandler(req,res,err); + }); } }; diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index 74f2c0f75..e28413790 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -1,280 +1,3 @@ - - - - - - diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index 708cdba0b..1b53374c2 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -17,18 +17,6 @@ module.exports = function(RED) { "use strict"; - function LibsConfigNode(n) { - RED.nodes.createNode(this, n); - this.name = n.name; - this.libs = n.libs; - } - RED.nodes.registerType("library-config", LibsConfigNode); - - RED.httpNode.get("/function/modules", function (req, res) { - var list = RED.nodes.listNPMModules(); - res.send(list); - }); - var util = require("util"); var vm = require("vm"); @@ -101,9 +89,8 @@ module.exports = function(RED) { } function FunctionNode(n) { - var libConf = RED.nodes.getNode(n.libs); - var libs = libConf ? libConf.libs : []; - n.modules = libs.map(x => x.name).filter(x => (x && (x !== ""))); + var libs = n.libs || []; + n.modules = libs.map(x => x.spec).filter(x => (x && (x !== ""))); var loadPromise = RED.nodes.createNode(this,n); var node = this; node.name = n.name; @@ -301,7 +288,8 @@ module.exports = function(RED) { }); // wait for module installation - loadPromise.then(function () { + loadPromise.catch(()=>{ + }).finally(function () { if (node.hasOwnProperty("libs")) { var modules = node.libs; modules.forEach(module => { @@ -309,11 +297,14 @@ module.exports = function(RED) { if (vname && (vname !== "")) { sandbox[vname] = null; try { - var lib = RED.require(module.name); - sandbox[vname] = lib; + var spec = module.spec; + if (spec && (spec !== "")) { + var lib = RED.require(module.spec); + sandbox[vname] = lib; + } } catch (e) { - node.warn("failed to load library: "+ module.name); + node.warn("failed to load library: "+ module.spec); } } }); diff --git a/packages/node_modules/@node-red/registry/lib/util.js b/packages/node_modules/@node-red/registry/lib/util.js index 8dda6ece5..15eca575b 100644 --- a/packages/node_modules/@node-red/registry/lib/util.js +++ b/packages/node_modules/@node-red/registry/lib/util.js @@ -83,7 +83,7 @@ function createNodeApi(node) { httpAdmin: runtime.adminApp, server: runtime.server } - copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials", "listNPMModules"]); + copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials"]); red.nodes.registerType = function(type,constructor,opts) { runtime.nodes.registerType(node.id,type,constructor,opts); } diff --git a/packages/node_modules/@node-red/runtime/lib/api/nodes.js b/packages/node_modules/@node-red/runtime/lib/api/nodes.js index 556e57df9..3e74e3a97 100644 --- a/packages/node_modules/@node-red/runtime/lib/api/nodes.js +++ b/packages/node_modules/@node-red/runtime/lib/api/nodes.js @@ -447,5 +447,47 @@ var api = module.exports = { } else { return null } + }, + + /** + * Gets list of NPM modules + * @param {Object} opts + * @param {User} opts.user - the user calling the api + * @param {Object} opts.req - the request to log (optional) + * @return {Promise} - list of installed NPM modules + * @memberof @node-red/runtime_nodes + */ + listNPMModules: async function(opts) { + var promise = runtime.nodes.listNPMModules(); + return promise; + }, + + /** + * Uninstall NPM modules + * @param {Object} opts + * @param {User} opts.user - the user calling the api + * @param {Object} opts.req - the request to log (optional) + * @return {Promise} - object for request result + * @memberof @node-red/runtime_nodes + */ + uninstallNPMModule: async function(opts) { + var spec = opts.spec; + var promise = runtime.nodes.uninstallNPMModule(spec); + return promise; + }, + + /** + * Update NPM modules + * @param {Object} opts + * @param {User} opts.user - the user calling the api + * @param {Object} opts.req - the request to log (optional) + * @return {Promise} - object for request result + * @memberof @node-red/runtime_nodes + */ + updateNPMModule: async function(opts) { + var spec = opts.spec; + var isUpdate = opts.update; + var promise = runtime.nodes.updateNPMModule(spec, isUpdate); + return promise; } } diff --git a/packages/node_modules/@node-red/runtime/lib/index.js b/packages/node_modules/@node-red/runtime/lib/index.js index 40aea7e36..86c9156fa 100644 --- a/packages/node_modules/@node-red/runtime/lib/index.js +++ b/packages/node_modules/@node-red/runtime/lib/index.js @@ -270,6 +270,7 @@ function stop() { }); } + // This is the internal api var runtime = { version: getVersion, diff --git a/packages/node_modules/@node-red/runtime/lib/nodes/index.js b/packages/node_modules/@node-red/runtime/lib/nodes/index.js index f9323be18..cade54d55 100644 --- a/packages/node_modules/@node-red/runtime/lib/nodes/index.js +++ b/packages/node_modules/@node-red/runtime/lib/nodes/index.js @@ -273,5 +273,7 @@ module.exports = { // NPM modules listNPMModules: npmModule.list, + uninstallNPMModule: npmModule.uninstall, + updateNPMModule: npmModule.update, loadNPMModule: npmModule.load }; diff --git a/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js b/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js index ae976de48..b4d4f64a7 100644 --- a/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js +++ b/packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js @@ -28,6 +28,8 @@ var log; var npmCommand = (process.platform === "win32") ? "npm.cmd" : "npm"; +var metadataFileName = "npm-modules.json"; + var moduleProp = {}; var moduleBase = null; @@ -35,6 +37,8 @@ var allowInstall = true; var allowList = ["*"]; var denyList = []; +var inProgress = {}; + /** * Initialise npm install module. * @param {Object} _runtime - runtime object @@ -46,6 +50,8 @@ function init(_runtime) { log = _runtime.log; moduleProp = {}; + inProgress = {}; + moduleBase = settings.userDir || process.env.NODE_RED_HOME || "."; if (settings.hasOwnProperty("externalModules")) { @@ -96,7 +102,6 @@ function register(type, prop) { */ function modulePath() { // takes variable length arguments in `arguments` var result = moduleBase; - result = path.join(result, "lib", "node_modules"); for(var i = 0; i < arguments.length; i++) { result = path.join(result, arguments[i]); } @@ -106,9 +111,10 @@ function modulePath() { // takes variable length arguments in `arguments` /** * Decompose NPM module specification string * @param {string} module - module specification + * @return {Object} array [name, version], where name is name part and version is version part of the module */ function moduleName(module) { - var match = /^([^@]+@.+)/.exec(module); + var match = /^([^@]+)@(.+)/.exec(module); if (match) { return [match[1], match[2]]; } @@ -116,15 +122,15 @@ function moduleName(module) { } /** - * Get NPM module package info + * Get NPM module info * @param {string} name - module name - * @param {string} name - module version + * @return {Object} package.json for specified NPM module */ -function infoNPM(name, ver) { - var path = modulePath(name, "package.json"); +function infoNPM(name) { + var path = modulePath("node_modules", name, "package.json"); try { - var pkg = require(path); - return pkg; + var pkg = fs.readFileSync(path); + return JSON.parse(pkg); } catch (e) { } @@ -132,17 +138,50 @@ function infoNPM(name, ver) { } /** - * Ensure existance of module installation directory + * Load NPM module metadata + * @return {object} module metadata object */ -function ensureLibDirectory() { - var path = modulePath(); - if (!fs.existsSync(path)) { - fs.mkdirSync(path, { - recursive: true - }); - return fs.existsSync(path); +function loadMetadata() { + var path = modulePath(metadataFileName); + try { + var pkg = fs.readFileSync(path); + return JSON.parse(pkg); } - return true; + catch (e) { + } + return { + modules: [] + }; +} + +/** + * Save NPM module metadata + * @param {string} data - module metadata object + */ +function saveMetadata(data) { + var path = modulePath(metadataFileName); + var str = JSON.stringify(data, null, 4); + fs.writeFileSync(path, str); +} + +/** + * Find item in metadata + * @param {Object} meta - metadata + * @param {string} name - module name + * @return {object} metadata item + */ +function findModule(meta, name) { + var modules = meta.modules; + var item = modules.find(item => (item.name === name)); + return item; +} + +function setInProgress(name) { + inProgress[name] = true; +} + +function clearInProgress(name) { + inProgress[name] = false; } /** @@ -151,32 +190,60 @@ function ensureLibDirectory() { */ function installNPM(module) { var [name, ver] = moduleName(module); - if (!ensureLibDirectory()) { - log.warn("failed to install: "+name); - return; - } - var pkg = infoNPM(name, ver); - if (!pkg) { - var args = ["install", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - log.info("successfully installed: "+name); + setInProgress(name); + return new Promise((resolve, reject) => { + var pkg = infoNPM(name); + if (!pkg) { + var args = ["install", module]; + var dir = modulePath(); + return exec.run(npmCommand, args, { + cwd: dir + }, true).then(result => { + if (result && (result.code === 0)) { + pkg = infoNPM(name); + var spec = name +(pkg ? "@"+pkg.version : ""); + log.info("successfully installed: "+spec); + var meta = loadMetadata(); + var item = { + name: name, + spec: module, + status: "installed", + }; + meta.modules.push(item); + saveMetadata(meta); + clearInProgress(name); + resolve(true); + } + else { + clearInProgress(name); + var msg = "failed to install: "+name; + log.warn(msg); + reject(msg); + } + }).catch(e => { + clearInProgress(name); + var msg = "failed to install: "+name + log.warn(msg); + reject(msg); + }); + } + else { + var meta = loadMetadata(); + if (!findModule(meta, name)) { + var item = { + name: name, + spec: module, + status: "preinstalled", + }; + meta.modules.push(item); + saveMetadata(meta); } - else { - log.warn("failed to install: "+name); - } - }).catch(e => { - var msg = e.hasOwnProperty("stderr") ? e.stderr : e; - log.warn("failed to install: "+name); - }); - } - else { - log.info("already installed: "+name); - } - return Promise.resolve(); + clearInProgress(name); + var spec = name +(pkg ? ("@"+pkg.version) : ""); + log.info("already installed: "+spec); + } + resolve(true); + }); } /** @@ -213,6 +280,8 @@ function checkInstall(node) { name = module.name; } if (isAllowed(name)) { + var [n, v] = moduleName(name); + setInProgress(name); promises.push(installNPM(name)); } else { @@ -230,7 +299,7 @@ function checkInstall(node) { function load(module) { try { var [name, ver] = moduleName(module); - var path = modulePath(name); + var path = modulePath("node_modules", name); var npm = require(path); return npm; } @@ -243,32 +312,140 @@ function load(module) { * Get list of installed modules */ function listModules() { - var modPath = modulePath(); - if (!fs.existsSync(modPath)) { - return []; - } - var dir = fs.opendirSync(modPath); - var modules = []; - if (dir) { - var ent = null; - while (ent = dir.readSync()) { - var name = ent.name; - if (ent.isDirectory() && - (name[0] !== ".")) { - var pkgPath = path.join(modPath, name, "package.json"); - if (fs.existsSync(pkgPath)) { - var pkg = fs.readJSONSync(pkgPath); - var info = { - name: pkg.name, - version: pkg.version - }; - modules.push(info); - } + return new Promise((resolve, reject) => { + var meta = loadMetadata(); + var modules = meta.modules; + modules.forEach(item => { + var name = item.name; + var info = infoNPM(name); + if (info) { + item.version = info.version; + } + item.inProgress = ((name in inProgress) && inProgress[name]); + }); + Object.keys(inProgress).forEach(name => { + if (inProgress[name] && + !modules.find(item => (item.name === name))) { + modules.push({ + name: name, + spec: name, + state: "inprogress", + inProgress: true + }); + } + }); + resolve(meta.modules); + }); +} + +/** + * Uninstall NPM modules + */ +function uninstall(module) { + var [name, ver] = moduleName(module); + setInProgress(name); + return new Promise((resolve, reject) => { + var pkg = infoNPM(name); + var meta = loadMetadata(); + var item = findModule(meta, name); + if (pkg && item) { + if (item.status === "preinstalled") { + clearInProgress(name); + var msg = "can't uninstall preinstalled: "+name; + log.warn(msg); + reject(msg); + } + else { + var args = ["uninstall", module]; + var dir = modulePath(); + return exec.run(npmCommand, args, { + cwd: dir + }, true).then(result => { + if (result && (result.code === 0)) { + log.info("successfully uninstalled: "+name); + var meta = loadMetadata(); + var items = meta.modules.filter(item => (item.name !== name)); + meta.modules = items; + saveMetadata(meta); + clearInProgress(name); + resolve(true); + } + else { + clearInProgress(name); + var msg = "failed to uninstall: "+name; + log.warn(msg); + reject(msg); + } + }).catch(e => { + clearInProgress(name); + var msg = "failed to uninstall: "+name; + log.warn(msg); + reject(msg); + }); } } - dir.closeSync(); - } - return modules; + else { + clearInProgress(name); + var msg = "module not installed: "+name; + log.info(msg); + reject(msg); + } + }); +} + +/** + * Update NPM modules + */ +function update(module, isUpdate) { + var act = (isUpdate ? "updated": "install") + var acted = (isUpdate ? "updated": "installed") + var [name, ver] = moduleName(module); + setInProgress(name); + return new Promise((resolve, reject) => { + var pkg = infoNPM(name); + if (!pkg || isUpdate) { + var args = ["install", module]; + var dir = modulePath(); + return exec.run(npmCommand, args, { + cwd: dir + }, true).then(result => { + if (result && (result.code === 0)) { + pkg = infoNPM(name); + var spec = name +(pkg ? "@"+pkg.version : ""); + log.info("successfully "+acted+": "+spec); + var meta = loadMetadata(); + var items = meta.modules.filter(item => (item.name !== name)); + var item = { + name: name, + spec: module, + status: "installed", + }; + items.push(item); + meta.modules = items; + saveMetadata(meta); + clearInProgress(name); + resolve(true); + } + else { + clearInProgress(name); + var msg = "failed to "+act+": "+name; + log.warn(msg); + reject(msg); + } + }).catch(e => { + clearInProgress(name); + var msg = "failed to "+act+": "+name; + log.warn(msg); + reject(msg); + }); + } + else { + clearInProgress(name); + var msg = "not "+acted+": "+name; + log.info(msg); + reject(msg); + } + }); } api = { @@ -276,6 +453,8 @@ api = { register: register, checkInstall: checkInstall, load: load, - list: listModules + list: listModules, + uninstall: uninstall, + update: update }; module.exports = api; From 9c09ee3b7106e9df39ace68d61f6959daee7b04a Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Fri, 12 Feb 2021 18:14:13 +0000 Subject: [PATCH 03/16] Rework Function node module integration --- .../@node-red/editor-api/lib/admin/index.js | 5 - .../@node-red/editor-api/lib/admin/nodes.js | 40 - .../editor-client/locales/en-US/editor.json | 1 + .../@node-red/editor-client/src/js/red.js | 14 + .../editor-client/src/js/ui/common/tabs.js | 4 +- .../editor-client/src/js/ui/palette-editor.js | 68 +- .../editor-client/src/js/ui/utils.js | 65 +- .../editor-client/src/sass/base.scss | 7 + .../nodes/core/function/10-function.html | 807 ++++++------------ .../nodes/core/function/10-function.js | 312 +++---- .../nodes/locales/en-US/messages.json | 2 +- .../@node-red/registry/lib/externalModules.js | 209 +++++ .../@node-red/registry/lib/index.js | 4 + .../@node-red/registry/lib/registry.js | 19 +- .../@node-red/registry/lib/util.js | 11 +- .../@node-red/runtime/lib/api/nodes.js | 42 - .../@node-red/runtime/lib/flows/index.js | 78 +- .../@node-red/runtime/lib/nodes/index.js | 16 +- .../@node-red/runtime/lib/nodes/npmModule.js | 460 ---------- .../registry/lib/externalModules_spec.js | 20 + 20 files changed, 800 insertions(+), 1384 deletions(-) create mode 100644 packages/node_modules/@node-red/registry/lib/externalModules.js delete mode 100644 packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js create mode 100644 test/unit/@node-red/registry/lib/externalModules_spec.js diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/index.js b/packages/node_modules/@node-red/editor-api/lib/admin/index.js index 5de09acad..34c47b2cb 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/index.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/index.js @@ -67,11 +67,6 @@ module.exports = { adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.read"),nodes.getSet,apiUtil.errorHandler); adminApp.put(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.write"),nodes.putSet,apiUtil.errorHandler); - // NPM Modules - adminApp.get("/modules", needsPermission("nodes.read"), nodes.listNPMModules, apiUtil.errorHandler); - adminApp.delete("/modules/:spec", needsPermission("nodes.write"), nodes.uninstallNPMModule, apiUtil.errorHandler); - adminApp.post("/modules", needsPermission("nodes.write"), nodes.updateNPMModule, apiUtil.errorHandler); - // Context adminApp.get("/context/:scope(global)",needsPermission("context.read"),context.get,apiUtil.errorHandler); adminApp.get("/context/:scope(global)/*",needsPermission("context.read"),context.get,apiUtil.errorHandler); diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js index 77efedd11..058053a29 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js @@ -191,45 +191,5 @@ module.exports = { runtimeAPI.nodes.getIconList(opts).then(function(list) { res.json(list); }); - }, - - listNPMModules: function(req, res) { - var opts = { - user: req.user, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.listNPMModules(opts).then(function(list) { - res.json(list); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - });; - }, - - uninstallNPMModule: function(req, res) { - var opts = { - user: req.user, - spec: req.params.spec, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.uninstallNPMModule(opts).then(function(result) { - res.json(result); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - }); - }, - - updateNPMModule: function(req, res) { - var body = req.body; - var opts = { - user: req.user, - spec: body.spec, - update: body.update, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.updateNPMModule(opts).then(function(result) { - res.json(result); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - }); } }; diff --git a/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json b/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json index 7894f1ed2..e4c585811 100755 --- a/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json +++ b/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json @@ -142,6 +142,7 @@ "nodeActionDisabled": "node actions disabled", "nodeActionDisabledSubflow": "node actions disabled within subflow", "missing-types": "

Flows stopped due to missing node types.

", + "missing-modules": "

Flows stopped due to missing modules.

", "safe-mode":"

Flows stopped in safe mode.

You can modify your flows and deploy the changes to restart.

", "restartRequired": "Node-RED must be restarted to enable upgraded modules", "credentials_load_failed": "

Flows stopped as the credentials could not be decrypted.

The flow credential file is encrypted, but the project's encryption key is missing or invalid.

", diff --git a/packages/node_modules/@node-red/editor-client/src/js/red.js b/packages/node_modules/@node-red/editor-client/src/js/red.js index 97159e6d2..e6eb381ff 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/red.js +++ b/packages/node_modules/@node-red/editor-client/src/js/red.js @@ -248,6 +248,7 @@ var RED = (function() { id: notificationId } if (notificationId === "runtime-state") { + RED.events.emit("runtime-state",msg); if (msg.error === "safe-mode") { options.buttons = [ { @@ -280,6 +281,16 @@ var RED = (function() { } ] } + } else if (msg.error === "missing-modules") { + text+="
  • "+msg.modules.map(function(m) { return RED.utils.sanitize(m.module)+(m.error?(" - "+RED.utils.sanitize(""+m.error)+""):"")}).join("
  • ")+"
"; + options.buttons = [ + { + text: RED._("common.label.close"), + click: function() { + persistentNotifications[notificationId].hideNotification(); + } + } + ] } else if (msg.error === "credentials_load_failed") { if (RED.settings.theme("projects.enabled",false)) { // projects enabled @@ -370,6 +381,9 @@ var RED = (function() { } else if (persistentNotifications.hasOwnProperty(notificationId)) { persistentNotifications[notificationId].close(); delete persistentNotifications[notificationId]; + if (notificationId === 'runtime-state') { + RED.events.emit("runtime-state",msg); + } } }); RED.comms.subscribe("status/#",function(topic,msg) { diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js index 81cd407eb..1d6c75c32 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js @@ -509,8 +509,8 @@ RED.tabs = (function() { li.attr('id',"red-ui-tab-"+(tab.id.replace(".","-"))); li.data("tabId",tab.id); - if (options.maximumTabWidth) { - li.css("maxWidth",options.maximumTabWidth+"px"); + if (options.maximumTabWidth || tab.maximumTabWidth) { + li.css("maxWidth",(options.maximumTabWidth || tab.maximumTabWidth) +"px"); } var link = $("",{href:"#"+tab.id, class:"red-ui-tab-label"}).appendTo(li); if (tab.icon) { diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js b/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js index 3ff3e7436..89df706f2 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js @@ -331,7 +331,7 @@ RED.palette.editor = (function() { if (v.modules) { var a = false; v.modules = v.modules.filter(function(m) { - if (checkModuleAllowed(m.id,m.version,installAllowList,installDenyList)) { + if (RED.utils.checkModuleAllowed(m.id,m.version,installAllowList,installDenyList)) { loadedIndex[m.id] = m; m.index = [m.id]; if (m.keywords) { @@ -445,68 +445,6 @@ RED.palette.editor = (function() { var installAllowList = ['*']; var installDenyList = []; - function parseModuleList(list) { - list = list || ["*"]; - return list.map(function(rule) { - var m = /^(.+?)(?:@(.*))?$/.exec(rule); - var wildcardPos = m[1].indexOf("*"); - wildcardPos = wildcardPos===-1?Infinity:wildcardPos; - - return { - module: new RegExp("^"+m[1].replace(/\*/g,".*")+"$"), - version: m[2], - wildcardPos: wildcardPos - } - }) - } - - function checkAgainstList(module,version,list) { - for (var i=0;i deniedRule.wildcardPos - } else { - // First wildcard in same position. - // Go with the longer matching rule. This isn't going to be 100% - // right, but we are deep into edge cases at this point. - return allowedRule.module.toString().length > deniedRule.module.toString().length - } - return false; - } - function init() { if (RED.settings.get('externalModules.palette.allowInstall', true) === false) { return; @@ -517,8 +455,8 @@ RED.palette.editor = (function() { installAllowList = settingsAllowList; installDenyList = settingsDenyList } - installAllowList = parseModuleList(installAllowList); - installDenyList = parseModuleList(installDenyList); + installAllowList = RED.utils.parseModuleList(installAllowList); + installDenyList = RED.utils.parseModuleList(installDenyList); createSettingsPane(); diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js b/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js index 230f561f9..5be2d6c40 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js @@ -1122,6 +1122,67 @@ RED.utils = (function() { return '#'+'000000'.slice(0, 6-s.length)+s; } + function parseModuleList(list) { + list = list || ["*"]; + return list.map(function(rule) { + var m = /^(.+?)(?:@(.*))?$/.exec(rule); + var wildcardPos = m[1].indexOf("*"); + wildcardPos = wildcardPos===-1?Infinity:wildcardPos; + + return { + module: new RegExp("^"+m[1].replace(/\*/g,".*")+"$"), + version: m[2], + wildcardPos: wildcardPos + } + }) + } + + function checkAgainstList(module,version,list) { + for (var i=0;i deniedRule.wildcardPos + } else { + // First wildcard in same position. + // Go with the longer matching rule. This isn't going to be 100% + // right, but we are deep into edge cases at this point. + return allowedRule.module.toString().length > deniedRule.module.toString().length + } + return false; + } return { createObjectElement: buildMessageElement, getMessageProperty: getMessageProperty, @@ -1141,6 +1202,8 @@ RED.utils = (function() { sanitize: sanitize, renderMarkdown: renderMarkdown, createNodeIcon: createNodeIcon, - getDarkerColor: getDarkerColor + getDarkerColor: getDarkerColor, + parseModuleList: parseModuleList, + checkModuleAllowed: checkModuleAllowed } })(); diff --git a/packages/node_modules/@node-red/editor-client/src/sass/base.scss b/packages/node_modules/@node-red/editor-client/src/sass/base.scss index 391ddf83f..0a2d7407e 100644 --- a/packages/node_modules/@node-red/editor-client/src/sass/base.scss +++ b/packages/node_modules/@node-red/editor-client/src/sass/base.scss @@ -143,6 +143,13 @@ background-size: contain } + .red-ui-font-code { + font-family: $monospace-font; + font-size: $primary-font-size; + color: $info-text-code-color; + white-space: nowrap; + } + code { font-family: $monospace-font; font-size: $primary-font-size; diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index e28413790..5b0e8da44 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -1,20 +1,67 @@ diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index 1b53374c2..10ffc1189 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -91,13 +91,17 @@ module.exports = function(RED) { function FunctionNode(n) { var libs = n.libs || []; n.modules = libs.map(x => x.spec).filter(x => (x && (x !== ""))); - var loadPromise = RED.nodes.createNode(this,n); + RED.nodes.createNode(this,n); var node = this; node.name = n.name; node.func = n.func; node.ini = n.initialize ? n.initialize.trim() : ""; node.fin = n.finalize ? n.finalize.trim() : ""; - node.libs = libs; + node.libs = libs || []; + + if (RED.settings.functionExternalModules === false && node.libs.length > 0) { + throw new Error("Function node not allowed to load external modules"); + } var handleNodeDoneCall = true; @@ -287,173 +291,173 @@ module.exports = function(RED) { } }); - // wait for module installation - loadPromise.catch(()=>{ - }).finally(function () { - if (node.hasOwnProperty("libs")) { - var modules = node.libs; - modules.forEach(module => { - var vname = module.hasOwnProperty("vname") ? module.vname : null; - if (vname && (vname !== "")) { - sandbox[vname] = null; - try { - var spec = module.spec; - if (spec && (spec !== "")) { - var lib = RED.require(module.spec); - sandbox[vname] = lib; + if (node.hasOwnProperty("libs")) { + var modules = node.libs; + modules.forEach(module => { + var vname = module.hasOwnProperty("var") ? module.var : null; + if (vname && (vname !== "")) { + sandbox[vname] = null; + try { + var spec = module.spec; + if (spec && (spec !== "")) { + var lib = RED.require(module.spec); + sandbox[vname] = lib; + } + } + catch (e) { + console.log(e); + node.warn("failed to load library: "+ module.spec); + } + } + }); + } + var context = vm.createContext(sandbox); + try { + var iniScript = null; + var iniOpt = null; + if (node.ini && (node.ini !== "")) { + var iniText = ` + (async function(__send__) { + var node = { + id:__node__.id, + name:__node__.name, + log:__node__.log, + error:__node__.error, + warn:__node__.warn, + debug:__node__.debug, + trace:__node__.trace, + status:__node__.status, + send: function(msgs, cloneMsg) { + __node__.send(__send__, RED.util.generateId(), msgs, cloneMsg); + } + }; + `+ node.ini +` + })(__initSend__);`; + iniOpt = createVMOpt(node, " setup"); + iniScript = new vm.Script(iniText, iniOpt); + } + node.script = vm.createScript(functionText, createVMOpt(node, "")); + if (node.fin && (node.fin !== "")) { + var finText = "(function () {\n"+node.fin +"\n})();"; + finOpt = createVMOpt(node, " cleanup"); + finScript = new vm.Script(finText, finOpt); + } + var promise = Promise.resolve(); + if (iniScript) { + context.__initSend__ = function(msgs) { node.send(msgs); }; + promise = iniScript.runInContext(context, iniOpt); + } + + processMessage = function (msg, send, done) { + var start = process.hrtime(); + context.msg = msg; + context.__send__ = send; + context.__done__ = done; + + node.script.runInContext(context); + context.results.then(function(results) { + sendResults(node,send,msg._msgid,results,false); + if (handleNodeDoneCall) { + done(); + } + + var duration = process.hrtime(start); + var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; + node.metric("duration", msg, converted); + if (process.env.NODE_RED_FUNCTION_TIME) { + node.status({fill:"yellow",shape:"dot",text:""+converted}); + } + }).catch(err => { + if ((typeof err === "object") && err.hasOwnProperty("stack")) { + //remove unwanted part + var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); + err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); + var stack = err.stack.split(/\r?\n/); + + //store the error in msg to be used in flows + msg.error = err; + + var line = 0; + var errorMessage; + if (stack.length > 0) { + while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { + line++; + } + + if (line < stack.length) { + errorMessage = stack[line]; + var m = /:(\d+):(\d+)$/.exec(stack[line+1]); + if (m) { + var lineno = Number(m[1])-1; + var cha = m[2]; + errorMessage += " (line "+lineno+", col "+cha+")"; + } } } - catch (e) { - node.warn("failed to load library: "+ module.spec); + if (!errorMessage) { + errorMessage = err.toString(); } + done(errorMessage); + } + else if (typeof err === "string") { + done(err); + } + else { + done(JSON.stringify(err)); } }); } - var context = vm.createContext(sandbox); - try { - var iniScript = null; - var iniOpt = null; - if (node.ini && (node.ini !== "")) { - var iniText = ` - (async function(__send__) { - var node = { - id:__node__.id, - name:__node__.name, - log:__node__.log, - error:__node__.error, - warn:__node__.warn, - debug:__node__.debug, - trace:__node__.trace, - status:__node__.status, - send: function(msgs, cloneMsg) { - __node__.send(__send__, RED.util.generateId(), msgs, cloneMsg); - } - }; - `+ node.ini +` - })(__initSend__);`; - iniOpt = createVMOpt(node, " setup"); - iniScript = new vm.Script(iniText, iniOpt); + + node.on("close", function() { + if (finScript) { + try { + finScript.runInContext(context, finOpt); + } + catch (err) { + node.error(err); + } } - node.script = vm.createScript(functionText, createVMOpt(node, "")); - if (node.fin && (node.fin !== "")) { - var finText = "(function () {\n"+node.fin +"\n})();"; - finOpt = createVMOpt(node, " cleanup"); - finScript = new vm.Script(finText, finOpt); + while (node.outstandingTimers.length > 0) { + clearTimeout(node.outstandingTimers.pop()); } - var promise = Promise.resolve(); - if (iniScript) { - context.__initSend__ = function(msgs) { node.send(msgs); }; - promise = iniScript.runInContext(context, iniOpt); + while (node.outstandingIntervals.length > 0) { + clearInterval(node.outstandingIntervals.pop()); } + if (node.clearStatus) { + node.status({}); + } + }); - processMessage = function (msg, send, done) { - var start = process.hrtime(); - context.msg = msg; - context.__send__ = send; - context.__done__ = done; - - node.script.runInContext(context); - context.results.then(function(results) { - sendResults(node,send,msg._msgid,results,false); - if (handleNodeDoneCall) { - done(); - } - - var duration = process.hrtime(start); - var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; - node.metric("duration", msg, converted); - if (process.env.NODE_RED_FUNCTION_TIME) { - node.status({fill:"yellow",shape:"dot",text:""+converted}); - } - }).catch(err => { - if ((typeof err === "object") && err.hasOwnProperty("stack")) { - //remove unwanted part - var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); - err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); - var stack = err.stack.split(/\r?\n/); - - //store the error in msg to be used in flows - msg.error = err; - - var line = 0; - var errorMessage; - if (stack.length > 0) { - while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { - line++; - } - - if (line < stack.length) { - errorMessage = stack[line]; - var m = /:(\d+):(\d+)$/.exec(stack[line+1]); - if (m) { - var lineno = Number(m[1])-1; - var cha = m[2]; - errorMessage += " (line "+lineno+", col "+cha+")"; - } - } - } - if (!errorMessage) { - errorMessage = err.toString(); - } - done(errorMessage); - } - else if (typeof err === "string") { - done(err); - } - else { - done(JSON.stringify(err)); - } + promise.then(function (v) { + var msgs = messages; + messages = []; + while (msgs.length > 0) { + msgs.forEach(function (s) { + processMessage(s.msg, s.send, s.done); }); + msgs = messages; + messages = []; } + state = RESOLVED; + }).catch((error) => { + messages = []; + state = ERROR; + node.error(error); + }); - node.on("close", function() { - if (finScript) { - try { - finScript.runInContext(context, finOpt); - } - catch (err) { - node.error(err); - } - } - while (node.outstandingTimers.length > 0) { - clearTimeout(node.outstandingTimers.pop()); - } - while (node.outstandingIntervals.length > 0) { - clearInterval(node.outstandingIntervals.pop()); - } - if (node.clearStatus) { - node.status({}); - } - }); - - promise.then(function (v) { - var msgs = messages; - messages = []; - while (msgs.length > 0) { - msgs.forEach(function (s) { - processMessage(s.msg, s.send, s.done); - }); - msgs = messages; - messages = []; - } - state = RESOLVED; - }).catch((error) => { - messages = []; - state = ERROR; - node.error(error); - }); - - } - catch(err) { - // eg SyntaxError - which v8 doesn't include line number information - // so we can't do better than this - updateErrorInfo(err); - node.error(err); - } - }); + } + catch(err) { + // eg SyntaxError - which v8 doesn't include line number information + // so we can't do better than this + updateErrorInfo(err); + node.error(err); + } } RED.nodes.registerType("function",FunctionNode, { - dynamicModuleList: "modules" + dynamicModuleList: "libs", + settings: { + functionExternalModules: { value: true, exportable: true } + } }); RED.library.register("functions"); }; diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index a2e529d16..20180f33d 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -220,7 +220,7 @@ "finalize": "// Code added here will be run when the\n// node is being stopped or re-deployed.\n" }, "require": { - "var": "name", + "var": "variable", "module": "module" }, "error": { diff --git a/packages/node_modules/@node-red/registry/lib/externalModules.js b/packages/node_modules/@node-red/registry/lib/externalModules.js new file mode 100644 index 000000000..6486ce8b3 --- /dev/null +++ b/packages/node_modules/@node-red/registry/lib/externalModules.js @@ -0,0 +1,209 @@ +// This module handles the management of modules required by the runtime and flows. +// Essentially this means keeping track of what extra modules a flow requires, +// ensuring those modules are installed and providing a standard way for nodes +// to require those modules safely. + +const fs = require("fs-extra"); +const registryUtil = require("./util"); +const path = require("path"); +const exec = require("@node-red/util").exec; +const log = require("@node-red/util").log; + +const BUILTIN_MODULES = require('module').builtinModules; +const EXTERNAL_MODULES_DIR = "externalModules"; + +// TODO: outsource running npm to a plugin +const NPM_COMMAND = (process.platform === "win32") ? "npm.cmd" : "npm"; + +let registeredTypes = {}; +let settings; + +let knownExternalModules = {}; + +let installEnabled = true; +let installAllowList = ['*']; +let installDenyList = []; + +function getInstallDir() { + return path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "externalModules")); +} + +async function refreshExternalModules() { + const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR)); + try { + const pkgFile = JSON.parse(await fs.readFile(path.join(externalModuleDir,"package.json"),"utf-8")); + knownExternalModules = pkgFile.dependencies; + } catch(err) { + } +} + +function init(_settings) { + settings = _settings; + path.resolve(settings.userDir || process.env.NODE_RED_HOME || "."); + + if (settings.externalModules && settings.externalModules.modules) { + if (settings.externalModules.modules.allowList || settings.externalModules.modules.denyList) { + installAllowList = settings.externalModules.modules.allowList; + installDenyList = settings.externalModules.modules.denyList; + } + if (settings.externalModules.modules.hasOwnProperty("allowInstall")) { + installEnabled = settings.externalModules.modules.allowInstall + } + } + installAllowList = registryUtil.parseModuleList(installAllowList); + installDenyList = registryUtil.parseModuleList(installDenyList); +} + +function register(type, dynamicModuleListProperty) { + registeredTypes[type] = dynamicModuleListProperty; +} + +function requireModule(module) { + if (!registryUtil.checkModuleAllowed( module, null,installAllowList,installDenyList)) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + throw e; + } + if (BUILTIN_MODULES.indexOf(module) !== -1) { + return require(module); + } + if (!knownExternalModules[module]) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + throw e; + } + const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR)); + const moduleDir = path.join(externalModuleDir,"node_modules",module); + return require(moduleDir); +} + +function parseModuleName(module) { + var match = /((?:@[^/]+\/)?[^/@]+)(?:@([\s\S]+))?/.exec(module); + if (match) { + return { + spec: module, + module: match[1], + version: match[2], + builtin: BUILTIN_MODULES.indexOf(match[1]) !== -1, + known: !!knownExternalModules[match[1]] + } + } + return null; +} + +function isInstalled(moduleDetails) { + return moduleDetails.builtin || moduleDetails.known; +} + +async function checkFlowDependencies(flowConfig) { + await refreshExternalModules(); + + const checkedModules = {}; + const promises = []; + const errors = []; + + flowConfig.forEach(n => { + if (registeredTypes[n.type]) { + let nodeModules = n[registeredTypes[n.type]] || []; + if (!Array.isArray(nodeModules)) { + nodeModules = [nodeModules] + } + nodeModules.forEach(module => { + if (typeof module !== 'string') { + module = module.module || ""; + } + if (module) { + let moduleDetails = parseModuleName(module) + if (moduleDetails && checkedModules[moduleDetails.module] === undefined) { + checkedModules[moduleDetails.module] = isInstalled(moduleDetails) + if (!checkedModules[moduleDetails.module]) { + if (installEnabled) { + promises.push(installModule(moduleDetails).catch(err => { + errors.push({module: moduleDetails,error:err}); + })) + } else if (!installEnabled) { + const e = new Error("Module install disabled - externalModules.modules.allowInstall=false"); + e.code = "install_not_allowed"; + errors.push({module: moduleDetails,error:e}); + } + } else if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + errors.push({module: moduleDetails,error:e}); + } + } + } + }) + } + }) + + return Promise.all(promises).then(() => { + if (errors.length > 0) { + throw errors; + } + }) +} + + +async function ensureModuleDir() { + const installDir = getInstallDir(); + + if (!fs.existsSync(installDir)) { + await fs.ensureDir(installDir); + } + const pkgFile = path.join(installDir,"package.json"); + if (!fs.existsSync(pkgFile)) { + await fs.writeFile(path.join(installDir,"package.json"),`{ + "name": "Node-RED External Modules", + "version": "1.0.0", + "private": true, + "dependencies": {} +}`) + } +} + +async function installModule(moduleDetails) { + let installSpec = moduleDetails.module; + if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) { + const e = new Error("Install not allowed"); + e.code = "install_not_allowed"; + throw e; + } + if (moduleDetails.version) { + installSpec = installSpec+"@"+moduleDetails.version; + } + log.info(log._("server.install.installing",{name: moduleDetails.module,version: moduleDetails.version||"latest"})); + const installDir = getInstallDir(); + + await ensureModuleDir(); + + var args = ["install", installSpec, "--production"]; + return exec.run(NPM_COMMAND, args, { + cwd: installDir + },true).then(result => { + log.info("successfully installed: "+installSpec); + }).catch(result => { + var output = result.stderr; + var e; + var lookForVersionNotFound = new RegExp("version not found: ","m"); + if (/E404/.test(output) || /ETARGET/.test(output)) { + log.error(log._("server.install.install-failed-not-found",{name:installSpec})); + e = new Error("Module not found"); + e.code = 404; + throw e; + } else { + log.error(log._("server.install.install-failed-long",{name:installSpec})); + log.error("------------------------------------------"); + log.error(output); + log.error("------------------------------------------"); + throw new Error(log._("server.install.install-failed")); + } + }) +} + +module.exports = { + init: init, + register: register, + checkFlowDependencies: checkFlowDependencies, + require: requireModule +} \ No newline at end of file diff --git a/packages/node_modules/@node-red/registry/lib/index.js b/packages/node_modules/@node-red/registry/lib/index.js index 03f979424..ef091992a 100644 --- a/packages/node_modules/@node-red/registry/lib/index.js +++ b/packages/node_modules/@node-red/registry/lib/index.js @@ -28,6 +28,7 @@ var registry = require("./registry"); var loader = require("./loader"); var installer = require("./installer"); var library = require("./library"); +const externalModules = require("./externalModules") /** * Initialise the registry with a reference to a runtime object @@ -42,6 +43,7 @@ function init(runtime) { loader.init(runtime); registry.init(runtime.settings,loader); library.init(); + externalModules.init(runtime.settings); } /** @@ -297,6 +299,8 @@ module.exports = { */ getNodeExampleFlowPath: library.getExampleFlowPath, + checkFlowDependencies: externalModules.checkFlowDependencies, + deprecated: require("./deprecated") }; diff --git a/packages/node_modules/@node-red/registry/lib/registry.js b/packages/node_modules/@node-red/registry/lib/registry.js index 730280d72..f2ea23b2f 100644 --- a/packages/node_modules/@node-red/registry/lib/registry.js +++ b/packages/node_modules/@node-red/registry/lib/registry.js @@ -21,6 +21,7 @@ var fs = require("fs"); var library = require("./library"); const {events} = require("@node-red/util") var subflows = require("./subflow"); +var externalModules = require("./externalModules") var settings; var loader; @@ -28,6 +29,7 @@ var nodeConfigCache = {}; var moduleConfigs = {}; var nodeList = []; var nodeConstructors = {}; +var nodeOptions = {}; var subflowModules = {}; var nodeTypeToId = {}; @@ -36,12 +38,7 @@ var moduleNodes = {}; function init(_settings,_loader) { settings = _settings; loader = _loader; - moduleNodes = {}; - nodeTypeToId = {}; - nodeConstructors = {}; - subflowModules = {}; - nodeList = []; - nodeConfigCache = {}; + clear(); } function load() { @@ -234,6 +231,7 @@ function removeNode(id) { if (typeId === id) { delete subflowModules[t]; delete nodeConstructors[t]; + delete nodeOptions[t]; delete nodeTypeToId[t]; } }); @@ -411,7 +409,7 @@ function getCaller(){ return stack[0].getFileName(); } -function registerNodeConstructor(nodeSet,type,constructor) { +function registerNodeConstructor(nodeSet,type,constructor,options) { if (nodeConstructors.hasOwnProperty(type)) { throw new Error(type+" already registered"); } @@ -431,6 +429,12 @@ function registerNodeConstructor(nodeSet,type,constructor) { } nodeConstructors[type] = constructor; + nodeOptions[type] = options; + if (options) { + if (options.dynamicModuleList) { + externalModules.register(type,options.dynamicModuleList); + } + } events.emit("type-registered",type); } @@ -525,6 +529,7 @@ function clear() { moduleConfigs = {}; nodeList = []; nodeConstructors = {}; + nodeOptions = {}; subflowModules = {}; nodeTypeToId = {}; } diff --git a/packages/node_modules/@node-red/registry/lib/util.js b/packages/node_modules/@node-red/registry/lib/util.js index 15eca575b..d6433f6f4 100644 --- a/packages/node_modules/@node-red/registry/lib/util.js +++ b/packages/node_modules/@node-red/registry/lib/util.js @@ -17,6 +17,7 @@ const path = require("path"); const semver = require("semver"); const {events,i18n,log} = require("@node-red/util"); + var runtime; function copyObjectProperties(src,dst,copyList,blockList) { @@ -45,13 +46,8 @@ function requireModule(name) { var relPath = path.relative(__dirname, moduleInfo.path); return require(relPath); } else { - var npm = runtime.nodes.loadNPMModule(name); - if (npm) { - return npm; - } - var err = new Error(`Cannot find module '${name}'`); - err.code = "MODULE_NOT_FOUND"; - throw err; + // Require it here to avoid the circular dependency + return require("./externalModules").require(name); } } @@ -129,7 +125,6 @@ function checkAgainstList(module,version,list) { } function checkModuleAllowed(module,version,allowList,denyList) { - // console.log("checkModuleAllowed",module,version);//,allowList,denyList) if (!allowList && !denyList) { // Default to allow return true; diff --git a/packages/node_modules/@node-red/runtime/lib/api/nodes.js b/packages/node_modules/@node-red/runtime/lib/api/nodes.js index 3e74e3a97..556e57df9 100644 --- a/packages/node_modules/@node-red/runtime/lib/api/nodes.js +++ b/packages/node_modules/@node-red/runtime/lib/api/nodes.js @@ -447,47 +447,5 @@ var api = module.exports = { } else { return null } - }, - - /** - * Gets list of NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - list of installed NPM modules - * @memberof @node-red/runtime_nodes - */ - listNPMModules: async function(opts) { - var promise = runtime.nodes.listNPMModules(); - return promise; - }, - - /** - * Uninstall NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - object for request result - * @memberof @node-red/runtime_nodes - */ - uninstallNPMModule: async function(opts) { - var spec = opts.spec; - var promise = runtime.nodes.uninstallNPMModule(spec); - return promise; - }, - - /** - * Update NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - object for request result - * @memberof @node-red/runtime_nodes - */ - updateNPMModule: async function(opts) { - var spec = opts.spec; - var isUpdate = opts.update; - var promise = runtime.nodes.updateNPMModule(spec, isUpdate); - return promise; } } diff --git a/packages/node_modules/@node-red/runtime/lib/flows/index.js b/packages/node_modules/@node-red/runtime/lib/flows/index.js index b31dc7755..bdcb9ae75 100644 --- a/packages/node_modules/@node-red/runtime/lib/flows/index.js +++ b/packages/node_modules/@node-red/runtime/lib/flows/index.js @@ -187,35 +187,35 @@ function setFlows(_config,_credentials,type,muteLog,forceStart,user) { }); } - return configSavePromise - .then(function(flowRevision) { - if (!isLoad) { - log.debug("saved flow revision: "+flowRevision); - } - activeConfig = { - flows:config, - rev:flowRevision - }; - activeFlowConfig = newFlowConfig; - if (forceStart || started) { - // Flows are running (or should be) - // Stop the active flows (according to deploy type and the diff) - return stop(type,diff,muteLog).then(() => { - // Once stopped, allow context to remove anything no longer needed - return context.clean(activeFlowConfig) - }).then(() => { - // Start the active flows - start(type,diff,muteLog).then(() => { - events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); - }); - // Return the new revision asynchronously to the actual start - return flowRevision; - }).catch(function(err) { }) - } else { - events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); - } - }); + return configSavePromise.then(flowRevision => { + if (!isLoad) { + log.debug("saved flow revision: "+flowRevision); + } + activeConfig = { + flows:config, + rev:flowRevision + }; + activeFlowConfig = newFlowConfig; + if (forceStart || started) { + // Flows are running (or should be) + + // Stop the active flows (according to deploy type and the diff) + return stop(type,diff,muteLog).then(() => { + // Once stopped, allow context to remove anything no longer needed + return context.clean(activeFlowConfig) + }).then(() => { + // Start the active flows + start(type,diff,muteLog).then(() => { + events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); + }); + // Return the new revision asynchronously to the actual start + return flowRevision; + }).catch(function(err) { }) + } else { + events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); + } + }); } function getNode(id) { @@ -246,7 +246,7 @@ function getFlows() { return activeConfig; } -function start(type,diff,muteLog) { +async function start(type,diff,muteLog) { type = type||"full"; started = true; var i; @@ -271,7 +271,21 @@ function start(type,diff,muteLog) { log.info(" "+settings.userDir); } events.emit("runtime-event",{id:"runtime-state",payload:{error:"missing-types", type:"warning",text:"notification.warnings.missing-types",types:activeFlowConfig.missingTypes},retain:true}); - return Promise.resolve(); + return; + } + + try { + await typeRegistry.checkFlowDependencies(activeConfig.flows); + } catch(err) { + log.info("Failed to load external modules required by this flow:"); + const missingModules = []; + for (i=0;i (item.name === name)); - return item; -} - -function setInProgress(name) { - inProgress[name] = true; -} - -function clearInProgress(name) { - inProgress[name] = false; -} - -/** - * Install NPM module - * @param {string} module - module specification - */ -function installNPM(module) { - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - if (!pkg) { - var args = ["install", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - pkg = infoNPM(name); - var spec = name +(pkg ? "@"+pkg.version : ""); - log.info("successfully installed: "+spec); - var meta = loadMetadata(); - var item = { - name: name, - spec: module, - status: "installed", - }; - meta.modules.push(item); - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to install: "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to install: "+name - log.warn(msg); - reject(msg); - }); - } - else { - var meta = loadMetadata(); - if (!findModule(meta, name)) { - var item = { - name: name, - spec: module, - status: "preinstalled", - }; - meta.modules.push(item); - saveMetadata(meta); - } - clearInProgress(name); - var spec = name +(pkg ? ("@"+pkg.version) : ""); - log.info("already installed: "+spec); - } - resolve(true); - }); -} - -/** - * Check allowance of NPM module installation - * @param {string} name - module specification - */ -function isAllowed(name) { - if (!allowInstall) { - return false; - } - var [module, ver] = moduleName(name); - var aList = util.parseModuleList(allowList); - var dList = util.parseModuleList(denyList); - return util.checkModuleAllowed(module, ver, aList, dList); -} - -/** - * Check and install NPM module according to dynamic module specification - * @param {Object} node - node object - */ -function checkInstall(node) { - var name = null; - if(moduleProp.hasOwnProperty(node.type)) { - name = moduleProp[node.type]; - } - var promises = []; - if (name && node.hasOwnProperty(name)) { - var modules = node[name]; - modules.forEach(module => { - var name = module; - if ((typeof module === "object") && - module && - module.hasOwnProperty("name")) { - name = module.name; - } - if (isAllowed(name)) { - var [n, v] = moduleName(name); - setInProgress(name); - promises.push(installNPM(name)); - } - else { - log.info("installation not allowed: "+name); - } - }); - } - return Promise.all(promises); -} - -/** - * Load NPM module - * @param {string} module - module to load - */ -function load(module) { - try { - var [name, ver] = moduleName(module); - var path = modulePath("node_modules", name); - var npm = require(path); - return npm; - } - catch (e) { - return null; - } -} - -/** - * Get list of installed modules - */ -function listModules() { - return new Promise((resolve, reject) => { - var meta = loadMetadata(); - var modules = meta.modules; - modules.forEach(item => { - var name = item.name; - var info = infoNPM(name); - if (info) { - item.version = info.version; - } - item.inProgress = ((name in inProgress) && inProgress[name]); - }); - Object.keys(inProgress).forEach(name => { - if (inProgress[name] && - !modules.find(item => (item.name === name))) { - modules.push({ - name: name, - spec: name, - state: "inprogress", - inProgress: true - }); - } - }); - resolve(meta.modules); - }); -} - -/** - * Uninstall NPM modules - */ -function uninstall(module) { - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - var meta = loadMetadata(); - var item = findModule(meta, name); - if (pkg && item) { - if (item.status === "preinstalled") { - clearInProgress(name); - var msg = "can't uninstall preinstalled: "+name; - log.warn(msg); - reject(msg); - } - else { - var args = ["uninstall", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - log.info("successfully uninstalled: "+name); - var meta = loadMetadata(); - var items = meta.modules.filter(item => (item.name !== name)); - meta.modules = items; - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to uninstall: "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to uninstall: "+name; - log.warn(msg); - reject(msg); - }); - } - } - else { - clearInProgress(name); - var msg = "module not installed: "+name; - log.info(msg); - reject(msg); - } - }); -} - -/** - * Update NPM modules - */ -function update(module, isUpdate) { - var act = (isUpdate ? "updated": "install") - var acted = (isUpdate ? "updated": "installed") - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - if (!pkg || isUpdate) { - var args = ["install", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - pkg = infoNPM(name); - var spec = name +(pkg ? "@"+pkg.version : ""); - log.info("successfully "+acted+": "+spec); - var meta = loadMetadata(); - var items = meta.modules.filter(item => (item.name !== name)); - var item = { - name: name, - spec: module, - status: "installed", - }; - items.push(item); - meta.modules = items; - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to "+act+": "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to "+act+": "+name; - log.warn(msg); - reject(msg); - }); - } - else { - clearInProgress(name); - var msg = "not "+acted+": "+name; - log.info(msg); - reject(msg); - } - }); -} - -api = { - init: init, - register: register, - checkInstall: checkInstall, - load: load, - list: listModules, - uninstall: uninstall, - update: update -}; -module.exports = api; diff --git a/test/unit/@node-red/registry/lib/externalModules_spec.js b/test/unit/@node-red/registry/lib/externalModules_spec.js new file mode 100644 index 000000000..a08c30ed8 --- /dev/null +++ b/test/unit/@node-red/registry/lib/externalModules_spec.js @@ -0,0 +1,20 @@ + // init: init, + // register: register, + // checkFlowDependencies: checkFlowDependencies, + // require: requireModule + // + +const should = require("should"); + +const fs = require("fs"); +const path = require("path"); + +var NR_TEST_UTILS = require("nr-test-utils"); + +var externalModules = NR_TEST_UTILS.require("@node-red/registry/lib/externalModules"); + + +describe("externalModules api", function() { + + +}); \ No newline at end of file From a94c19a6cfa6ce3df360d27086b133f118a55a29 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Fri, 12 Feb 2021 22:40:30 +0000 Subject: [PATCH 04/16] Fix up loading of freshly installed modules in Function node --- .../@node-red/nodes/core/function/10-function.js | 8 +++----- .../@node-red/registry/lib/externalModules.js | 5 +++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index 10ffc1189..b9fae8626 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -89,15 +89,13 @@ module.exports = function(RED) { } function FunctionNode(n) { - var libs = n.libs || []; - n.modules = libs.map(x => x.spec).filter(x => (x && (x !== ""))); RED.nodes.createNode(this,n); var node = this; node.name = n.name; node.func = n.func; node.ini = n.initialize ? n.initialize.trim() : ""; node.fin = n.finalize ? n.finalize.trim() : ""; - node.libs = libs || []; + node.libs = n.libs || []; if (RED.settings.functionExternalModules === false && node.libs.length > 0) { throw new Error("Function node not allowed to load external modules"); @@ -298,9 +296,9 @@ module.exports = function(RED) { if (vname && (vname !== "")) { sandbox[vname] = null; try { - var spec = module.spec; + var spec = module.module; if (spec && (spec !== "")) { - var lib = RED.require(module.spec); + var lib = RED.require(module.module); sandbox[vname] = lib; } } diff --git a/packages/node_modules/@node-red/registry/lib/externalModules.js b/packages/node_modules/@node-red/registry/lib/externalModules.js index 6486ce8b3..f794f1fc6 100644 --- a/packages/node_modules/@node-red/registry/lib/externalModules.js +++ b/packages/node_modules/@node-red/registry/lib/externalModules.js @@ -137,7 +137,7 @@ async function checkFlowDependencies(flowConfig) { } }) - return Promise.all(promises).then(() => { + return Promise.all(promises).then(refreshExternalModules).then(() => { if (errors.length > 0) { throw errors; } @@ -154,7 +154,8 @@ async function ensureModuleDir() { const pkgFile = path.join(installDir,"package.json"); if (!fs.existsSync(pkgFile)) { await fs.writeFile(path.join(installDir,"package.json"),`{ - "name": "Node-RED External Modules", + "name": "Node-RED-External-Modules", + "description": "These modules are automatically installed by Node-RED to use in Function nodes.", "version": "1.0.0", "private": true, "dependencies": {} From e899d2d5b84f8e610f2f72ae01f86fbe2ab18c46 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sat, 13 Feb 2021 00:18:04 +0000 Subject: [PATCH 05/16] Fix tests for externalModules component --- package.json | 2 +- .../unit/@node-red/registry/lib/index_spec.js | 2 +- .../@node-red/runtime/lib/flows/index_spec.js | 62 +++++++++++++++++-- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index cddf1f76e..82688f4e7 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "marked": "1.2.7", "minami": "1.2.3", "mocha": "^5.2.0", - "node-red-node-test-helper": "^0.2.5", + "node-red-node-test-helper": "^0.2.7", "node-sass": "^4.14.1", "nodemon": "2.0.6", "should": "13.2.3", diff --git a/test/unit/@node-red/registry/lib/index_spec.js b/test/unit/@node-red/registry/lib/index_spec.js index 75a057730..03ce5d509 100644 --- a/test/unit/@node-red/registry/lib/index_spec.js +++ b/test/unit/@node-red/registry/lib/index_spec.js @@ -40,7 +40,7 @@ describe('red/registry/index', function() { stubs.push(sinon.stub(loader,"init")); stubs.push(sinon.stub(typeRegistry,"init")); - registry.init({}); + registry.init({settings:{}}); installer.init.called.should.be.true(); loader.init.called.should.be.true(); typeRegistry.init.called.should.be.true(); diff --git a/test/unit/@node-red/runtime/lib/flows/index_spec.js b/test/unit/@node-red/runtime/lib/flows/index_spec.js index e230d2407..2e645df0d 100644 --- a/test/unit/@node-red/runtime/lib/flows/index_spec.js +++ b/test/unit/@node-red/runtime/lib/flows/index_spec.js @@ -36,6 +36,7 @@ describe('flows/index', function() { var flowCreate; var getType; + var checkFlowDependencies; var mockLog = { log: sinon.stub(), @@ -52,9 +53,16 @@ describe('flows/index', function() { getType = sinon.stub(typeRegistry,"get",function(type) { return type.indexOf('missing') === -1; }); + checkFlowDependencies = sinon.stub(typeRegistry, "checkFlowDependencies", async function(flow) { + if (flow[0].id === "node-with-missing-modules") { + throw new Error("Missing module"); + } + }); }); + after(function() { getType.restore(); + checkFlowDependencies.restore(); }); @@ -306,7 +314,7 @@ describe('flows/index', function() { flows.init({log:mockLog, settings:{},storage:storage}); flows.load().then(function() { - flows.startFlows(); + return flows.startFlows(); }); }); it('does not start if nodes missing', function(done) { @@ -321,9 +329,14 @@ describe('flows/index', function() { flows.init({log:mockLog, settings:{},storage:storage}); flows.load().then(function() { - flows.startFlows(); - flowCreate.called.should.be.false(); - done(); + return flows.startFlows(); + }).then(() => { + try { + flowCreate.called.should.be.false(); + done(); + } catch(err) { + done(err); + } }); }); @@ -339,9 +352,9 @@ describe('flows/index', function() { } flows.init({log:mockLog, settings:{},storage:storage}); flows.load().then(function() { - flows.startFlows(); + return flows.startFlows(); + }).then(() => { flowCreate.called.should.be.false(); - events.emit("type-registered","missing"); setTimeout(function() { flowCreate.called.should.be.false(); @@ -354,7 +367,44 @@ describe('flows/index', function() { }); }); + it('does not start if external modules missing', function(done) { + var originalConfig = [ + {id:"node-with-missing-modules",x:10,y:10,z:"t1",type:"test",wires:[]}, + {id:"t1",type:"tab"} + ]; + storage.getFlows = function() { + return Promise.resolve({flows:originalConfig}); + } + var receivedEvent = null; + var handleEvent = function(payload) { + receivedEvent = payload; + } + + events.on("runtime-event",handleEvent); + + //{id:"runtime-state",payload:{error:"missing-modules", type:"warning",text:"notification.warnings.missing-modules",modules:missingModules},retain:true});" + + + flows.init({log:mockLog, settings:{},storage:storage}); + flows.load().then(flows.startFlows).then(() => { + events.removeListener("runtime-event",handleEvent); + try { + flowCreate.called.should.be.false(); + receivedEvent.should.have.property('id','runtime-state'); + receivedEvent.should.have.property('payload', + { error: 'missing-modules', + type: 'warning', + text: 'notification.warnings.missing-modules', + modules: [] } + ); + + done(); + }catch(err) { + done(err) + } + }); + }); }); From d2c9ccbfddb6c089db26c9566bb6e6ca540d066e Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sun, 14 Feb 2021 00:02:08 +0000 Subject: [PATCH 06/16] Detect externalModule dependencies inside subflow modules Not sure this is 100% the right approach. If a subflow module has a dependency it should be in the subflow's package.json and therefore installed next to the subflow module in ~/.node-red/node_modules. By treating it as a 'normal' external module, it will be dynamically installed in ~/.node-red/externalModules. That then exposes the module to the user who won't know why its there and may remove it. It would be better to allow nodes inside a subflow module to require from ~/.node-red/node_modules and not limit it to the externalModules dir. The hard part is knowing when to do that. --- .../@node-red/registry/lib/externalModules.js | 22 ++++++++++++++---- .../@node-red/registry/lib/registry.js | 3 +++ test/resources/subflow/package/package.json | 5 ++-- test/resources/subflow/package/subflow.json | 1 + .../subflow/test-subflow-mod-1.0.1.tgz | Bin 1682 -> 0 bytes .../subflow/test-subflow-mod-1.0.2.tgz | Bin 0 -> 1709 bytes 6 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 test/resources/subflow/test-subflow-mod-1.0.1.tgz create mode 100644 test/resources/subflow/test-subflow-mod-1.0.2.tgz diff --git a/packages/node_modules/@node-red/registry/lib/externalModules.js b/packages/node_modules/@node-red/registry/lib/externalModules.js index f794f1fc6..1e9ec4ebe 100644 --- a/packages/node_modules/@node-red/registry/lib/externalModules.js +++ b/packages/node_modules/@node-red/registry/lib/externalModules.js @@ -6,6 +6,7 @@ const fs = require("fs-extra"); const registryUtil = require("./util"); const path = require("path"); +const clone = require("clone"); const exec = require("@node-red/util").exec; const log = require("@node-red/util").log; @@ -16,6 +17,7 @@ const EXTERNAL_MODULES_DIR = "externalModules"; const NPM_COMMAND = (process.platform === "win32") ? "npm.cmd" : "npm"; let registeredTypes = {}; +let subflowTypes = {}; let settings; let knownExternalModules = {}; @@ -58,6 +60,10 @@ function register(type, dynamicModuleListProperty) { registeredTypes[type] = dynamicModuleListProperty; } +function registerSubflow(type, subflowConfig) { + subflowTypes[type] = subflowConfig; +} + function requireModule(module) { if (!registryUtil.checkModuleAllowed( module, null,installAllowList,installDenyList)) { const e = new Error("Module not allowed"); @@ -95,15 +101,21 @@ function isInstalled(moduleDetails) { return moduleDetails.builtin || moduleDetails.known; } + async function checkFlowDependencies(flowConfig) { + let nodes = clone(flowConfig); await refreshExternalModules(); const checkedModules = {}; const promises = []; const errors = []; - - flowConfig.forEach(n => { - if (registeredTypes[n.type]) { + const checkedSubflows = {}; + while (nodes.length > 0) { + let n = nodes.shift(); + if (subflowTypes[n.type] && !checkedSubflows[n.type]) { + checkedSubflows[n.type] = true; + nodes = nodes.concat(subflowTypes[n.type].flow) + } else if (registeredTypes[n.type]) { let nodeModules = n[registeredTypes[n.type]] || []; if (!Array.isArray(nodeModules)) { nodeModules = [nodeModules] @@ -135,8 +147,7 @@ async function checkFlowDependencies(flowConfig) { } }) } - }) - + } return Promise.all(promises).then(refreshExternalModules).then(() => { if (errors.length > 0) { throw errors; @@ -205,6 +216,7 @@ async function installModule(moduleDetails) { module.exports = { init: init, register: register, + registerSubflow: registerSubflow, checkFlowDependencies: checkFlowDependencies, require: requireModule } \ No newline at end of file diff --git a/packages/node_modules/@node-red/registry/lib/registry.js b/packages/node_modules/@node-red/registry/lib/registry.js index a4d504fac..82de4e442 100644 --- a/packages/node_modules/@node-red/registry/lib/registry.js +++ b/packages/node_modules/@node-red/registry/lib/registry.js @@ -456,6 +456,9 @@ function registerSubflow(nodeSet, subflow) { nodeSetInfo.config = result.config; } subflowModules[result.type] = result; + externalModules.registerSubflow(result.type,subflow); + + events.emit("type-registered",result.type); return result; } diff --git a/test/resources/subflow/package/package.json b/test/resources/subflow/package/package.json index 92ff33e2b..831428c6a 100644 --- a/test/resources/subflow/package/package.json +++ b/test/resources/subflow/package/package.json @@ -1,6 +1,6 @@ { "name": "test-subflow-mod", - "version": "1.0.1", + "version": "1.0.2", "description": "", "keywords": [], "license": "ISC", @@ -13,6 +13,7 @@ ] }, "dependencies": { - "node-red-node-random": "*" + "node-red-node-random": "*", + "cowsay2": "*" } } diff --git a/test/resources/subflow/package/subflow.json b/test/resources/subflow/package/subflow.json index e5c6b25ab..db43cdff4 100644 --- a/test/resources/subflow/package/subflow.json +++ b/test/resources/subflow/package/subflow.json @@ -189,6 +189,7 @@ "noerr": 0, "initialize": "", "finalize": "", + "libs": [ {"var":"cowsay2","module":"cowsay2"}], "x": 240, "y": 100, "wires": [ diff --git a/test/resources/subflow/test-subflow-mod-1.0.1.tgz b/test/resources/subflow/test-subflow-mod-1.0.1.tgz deleted file mode 100644 index daa53b77a4d73bc6ca062f72c7559485d30bc892..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1682 zcmV;D25tEtiwFP!000006YUynZ`(GopZzO1?L(5yk|kNT6BqrEwkg)3U4}TtP}D)e zP?V)bCN+|Z;}-t!BPGkRUQS{=>DrkvVCn93ygS~5uLz(7=QC;TW_Xm zb{z*)!Bw-{7I@a~bWO|Zc1=kqAM9RR{%>EDa*MfuOp3B8pHvsH`2S8?V@%yR#0I{J zC=(pK0A5Ux2q@VQ95B4U z=kdJ*NeJCHL5K%af)*$(g2LAhjK66>_CJ4i14iH&H?>>uqeMH9tug!7hWVegXdeIb z|6~NV-NzmA-%&ANi2sh$*~R};G!@^P3PP<9G=aI$=dq~E#Go}5_8K!T^MN9;4AZbu zs7wZk1(BFzQs5PyPAGG^gnYV4AYp(o;R*M*gI6h)$|O3&t^}s(We(#@fCRWKkK%Bf zCbT)(teG)zafFGB2?`QEpO*P#9qQ==BrY9i94;DjE=3MG(Mn%SX?{^GW{p|nTaV!; zEh^RjCysxsV>#vc?>g2l{-2_yUakcy7&QbvyK{(;;b0p%hguq3nwUm-Df}{UmNm|0 zl!8GX7Z8tXeeK*2IFP^b6r>Xjco|}lPC3Aw-H;5)Tj0|O1EI*dPbVBe4tyL&KV=LN z6nI40G!c`I$1>|Iy2^$7asrXq8;P`Rvzdn~HkMh0PUIA&m^-SNE0tIYZ%4}&f2E%L~#;P#+DtZ+90Lx!&=rkGEDRj10^R-W=Fs(XN2W8t9Ho+yX(}rwL~kve;j@6 zE$Fj?$CK;R-hw_Wc+4s>7bWOy?SQ2OPXggXIZ>a6l#CMi8WF5Lpih}3&OX+4a!F}e z+lydvNxDr^g|jMEARnWvOM0_@HvH@DyZ6U+ijsI-pAXH=w7jI_%avI$3JAe$ea+xx zw2oezjvaJeS;)CAo#aZVkF24$Rvg?@!|$#h5&WZ+A~B&qN)-}5!`m2_Z(4%jics5H zHP~KKm#HuPJwDs2sx)$4?@tAGq~`=GHs1^1>Xi1rINr>QIkDnCU#r7VtEr zOi}!N+MV!y5>B4w5o!iJJDw@xztnHwENC*HD#8`d)Kgt- zV8$rhA7;BfAbAHv?bk6M84;8-Oz1Y{^TG2WSptC<;Omemc5;>52XaMhjIfY=BZ+#hc9$3yF(W_< z`@E<*jYSkERoE)pib}=R5EtM92}A(H;7he715%4qj%qub?%-6<7D{_JC8~V3zt85( zJ@!v-Q!*6D-=<4)L=(;=4#Q#^BM8PLU&WsXLyoJ=T)E@-kUgauGZ!+$)~viT<17_E zNJgb{xrv!s96oGQO{6ucYPOr;BZZ%*gj$+}i44a&0znl)l6Z`=G_Ne&^w3sJ!iHyc>~3vQpV!*{9Y#$I z?{>`|!tJ)ur-x8FyDX{>v>p|2;c7e*ON$7`u*D8j4v6^xuH?O0K`KT!Nj;l+*_>io!rn>STp%C6}^*uB%I2 z>Zte6)4@5A?=BP&?0wZF2d%HQK5)!LqBXVjs93TuT}_$o0bhuy-)e;v!O*8%^sU1~GwF{v8mVFnp+px?v0kI`o4Fb7;Z;=2@w?SV+u)mHX1kWI@aSAEb3A%#A~$k;{m2 zDbP#g#nh1jqgzJ@Z?@4@2SI~JjWU-AjgxU8B;m)Iy=?t?eE63ZVJ^rPC;!F)CtJ@w z@q8OPV)tzjlH(b5mMA5{(pR>%uPmTjU%t2jr+7lT%C+?|s%%5+%)hr{{wFP3`2XTR zoq-p5O8wpO>#SE2NmNc;0A}H30TTs43M}%IP`C~g z%A(u6n-g$JM5s%s6Qp!L7wO~!s@VgkE}P^OPCE-KRSG56YF`64zbVVX9X1tsW@gZY zU}xU>-ZOi<7S;0qBm2M6H|=`Z* zg8Sv*JZhYPm!jh`E}$M$_LLJp5D0#g8OjO_d=?Uvl^ke6ZbV1)9r9U3kW9J@ADjtc zfqW81zvKiREXkPhS;{7xOd#qkhbp;yF@ON}5|D09qfq9igz{|5vc0~i*}Hn**_=$Jxg+#q)!gNt^7ins!obqEoFpGsaB)yc zh`C@+mUBHurHZFSunFmTSd~8E6L^O5T-D2FIKRbY&Hv<988?XZlJX&`+Jh59UHbezK1@L=T*V?vNP z`qVba8DnAVD3ZrD?(P#R>{X${@riSO#xB>7hX1^K|6#vPP#RC#6#f8CHYJ=zsTogzzca6JB zwj)*m%}*;ICsuCS_uc+!E;VW!Keb-vRB@Y~om*_aT*q@P9e8RilbRd13-7Wa)uQG~SlO7CSB9%X*2HvN%eD7WGY6Z!(~g|X5?WKKoZ_Pd zzlbNTeNrOe%WWMbOFE=cOPjz#1J{HubX^j=p%$vTBX&v$Uq${n*dCX&#^}JalcU-c zu}1Ym)^E2`tKVd$)?KW0U3*)-PrNj^#Yx!)4Q_D^SlemzgR$Sda=6fi`g;V2m5?f7q5?J^$7Fw*GYf`xwoMd5F|M zkn$SVUj&w5ClFQk5RRfSfVvElTra(pg;X`U#FZTN{(X3Sg5cf75~Jr=3Q&N`wf9jF zE_#%OJqJqhUVdxh`XRZ^(SzW?V(t);-#F-1FP*5#B@b*plft2k#F~4px0qW8??9!SU3d+EaUK>$U#?{@$sV04@Lk DJFiW$ literal 0 HcmV?d00001 From 05beb6ca793f701de0c6979ab5d0a793b125fcd1 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 15 Feb 2021 17:28:14 +0000 Subject: [PATCH 07/16] Add unit tests for externalModules --- .../@node-red/registry/lib/externalModules.js | 9 +- .../registry/lib/externalModules_spec.js | 292 +++++++++++++++++- 2 files changed, 292 insertions(+), 9 deletions(-) diff --git a/packages/node_modules/@node-red/registry/lib/externalModules.js b/packages/node_modules/@node-red/registry/lib/externalModules.js index 1e9ec4ebe..ddbac24ae 100644 --- a/packages/node_modules/@node-red/registry/lib/externalModules.js +++ b/packages/node_modules/@node-red/registry/lib/externalModules.js @@ -41,8 +41,8 @@ async function refreshExternalModules() { function init(_settings) { settings = _settings; - path.resolve(settings.userDir || process.env.NODE_RED_HOME || "."); - + knownExternalModules = {}; + installEnabled = true; if (settings.externalModules && settings.externalModules.modules) { if (settings.externalModules.modules.allowList || settings.externalModules.modules.denyList) { installAllowList = settings.externalModules.modules.allowList; @@ -197,7 +197,6 @@ async function installModule(moduleDetails) { }).catch(result => { var output = result.stderr; var e; - var lookForVersionNotFound = new RegExp("version not found: ","m"); if (/E404/.test(output) || /ETARGET/.test(output)) { log.error(log._("server.install.install-failed-not-found",{name:installSpec})); e = new Error("Module not found"); @@ -208,7 +207,9 @@ async function installModule(moduleDetails) { log.error("------------------------------------------"); log.error(output); log.error("------------------------------------------"); - throw new Error(log._("server.install.install-failed")); + e = new Error(log._("server.install.install-failed")); + e.code = "unexpected_error"; + throw e; } }) } diff --git a/test/unit/@node-red/registry/lib/externalModules_spec.js b/test/unit/@node-red/registry/lib/externalModules_spec.js index a08c30ed8..17f3cc41d 100644 --- a/test/unit/@node-red/registry/lib/externalModules_spec.js +++ b/test/unit/@node-red/registry/lib/externalModules_spec.js @@ -1,20 +1,302 @@ // init: init, // register: register, + // registerSubflow: registerSubflow, // checkFlowDependencies: checkFlowDependencies, // require: requireModule // const should = require("should"); - -const fs = require("fs"); +const sinon = require("sinon"); +const fs = require("fs-extra"); const path = require("path"); +const os = require("os"); -var NR_TEST_UTILS = require("nr-test-utils"); +const NR_TEST_UTILS = require("nr-test-utils"); +const externalModules = NR_TEST_UTILS.require("@node-red/registry/lib/externalModules"); +const exec = NR_TEST_UTILS.require("@node-red/util/lib/exec"); -var externalModules = NR_TEST_UTILS.require("@node-red/registry/lib/externalModules"); +let homeDir; +async function createUserDir() { + if (!homeDir) { + homeDir = path.join(os.tmpdir(),"nr-test-"+Math.floor(Math.random()*100000)); + } + await fs.ensureDir(homeDir); +} + +async function setupExternalModulesPackage(dependencies) { + await fs.ensureDir(path.join(homeDir,"externalModules")) + await fs.writeFile(path.join(homeDir,"externalModules","package.json"),`{ +"name": "Node-RED-External-Modules", +"description": "These modules are automatically installed by Node-RED to use in Function nodes.", +"version": "1.0.0", +"private": true, +"dependencies": ${JSON.stringify(dependencies)} +}`) +} describe("externalModules api", function() { - + beforeEach(async function() { + await createUserDir() + }) + afterEach(async function() { + await fs.remove(homeDir); + }) + describe("checkFlowDependencies", function() { + beforeEach(function() { + sinon.stub(exec,"run", async function(cmd, args, options) { + let error; + if (args[1] === "moduleNotFound") { + error = new Error(); + error.stderr = "E404"; + } else if (args[1] === "moduleVersionNotFound") { + error = new Error(); + error.stderr = "ETARGET"; + } else if (args[1] === "moduleFail") { + error = new Error(); + error.stderr = "Some unexpected install error"; + } + if (error) { + throw error; + } + }) + }) + afterEach(function() { + exec.run.restore(); + }) + it("does nothing when no types are registered",async function() { + externalModules.init({userDir: homeDir}); + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "foo"}]} + ]) + exec.run.called.should.be.false(); + }); + it("skips install for modules already installed", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + await setupExternalModulesPackage({"foo": "1.2.3", "bar":"2.3.4"}); + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "foo"}]} + ]) + exec.run.called.should.be.false(); + }) + + it("skips install for built-in modules", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "fs"}]} + ]) + exec.run.called.should.be.false(); + }) + + it("installs missing modules", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + fs.existsSync(path.join(homeDir,"externalModuels")).should.be.false(); + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "foo"}]} + ]) + exec.run.called.should.be.true(); + fs.existsSync(path.join(homeDir,"externalModules")).should.be.true(); + }) + + it("installs missing modules from inside subflow module", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + externalModules.registerSubflow("sf", {"flow":[{type: "function", libs:[{module: "foo"}]}]}); + await externalModules.checkFlowDependencies([ + {type: "sf"} + ]) + exec.run.called.should.be.true(); + }) + + it("reports install fail - 404", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "moduleNotFound"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.called.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","moduleNotFound"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code",404); + + } + }) + it("reports install fail - target", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "moduleVersionNotFound"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.called.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","moduleVersionNotFound"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code",404); + } + }) + + it("reports install fail - unexpected", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "moduleFail"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.called.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","moduleFail"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code","unexpected_error"); + } + }) + it("reports install fail - multiple", async function() { + externalModules.init({userDir: homeDir}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "moduleNotFound"},{module: "moduleFail"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.called.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(2); + // Sort the array so we know the order to test for + err.sort(function(A,B) { + return A.module.module.localeCompare(B.module.module); + }) + err[1].should.have.property("module"); + err[1].module.should.have.property("module","moduleNotFound"); + err[1].should.have.property("error"); + err[1].error.should.have.property("code",404); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","moduleFail"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code","unexpected_error"); + + } + }) + it("reports install fail - install disabled", async function() { + externalModules.init({userDir: homeDir, externalModules: { + modules: { + allowInstall: false + } + }}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + {type: "function", libs:[{module: "foo"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + // Should not try to install + exec.run.called.should.be.false(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","foo"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code","install_not_allowed"); + } + }) + + it("reports install fail - module disallowed", async function() { + externalModules.init({userDir: homeDir, externalModules: { + modules: { + denyList: ['foo'] + } + }}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + // foo disallowed + // bar allowed + {type: "function", libs:[{module: "foo"},{module: "bar"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.calledOnce.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","foo"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code","install_not_allowed"); + } + }) + + it("reports install fail - built-in module disallowed", async function() { + externalModules.init({userDir: homeDir, externalModules: { + modules: { + denyList: ['fs'] + } + }}); + externalModules.register("function", "libs"); + try { + await externalModules.checkFlowDependencies([ + // foo disallowed + // bar allowed + {type: "function", libs:[{module: "fs"},{module: "bar"}]} + ]) + throw new Error("checkFlowDependencies did not reject after install fail") + } catch(err) { + exec.run.calledOnce.should.be.true(); + Array.isArray(err).should.be.true(); + err.should.have.length(1); + err[0].should.have.property("module"); + err[0].module.should.have.property("module","fs"); + err[0].should.have.property("error"); + err[0].error.should.have.property("code","module_not_allowed"); + } + }) + }) + describe("require", async function() { + it("requires built-in modules", async function() { + externalModules.init({userDir: homeDir}); + const result = externalModules.require("fs") + result.should.eql(require("fs")); + }) + it("rejects unknown modules", async function() { + externalModules.init({userDir: homeDir}); + try { + externalModules.require("foo") + throw new Error("require did not reject after fail") + } catch(err) { + err.should.have.property("code","module_not_allowed"); + } + }) + + it("rejects disallowed modules", async function() { + externalModules.init({userDir: homeDir, externalModules: { + modules: { + denyList: ['fs'] + } + }}); + try { + externalModules.require("fs") + throw new Error("require did not reject after fail") + } catch(err) { + err.should.have.property("code","module_not_allowed"); + } + }) + }) }); \ No newline at end of file From 9d34abf603054343dbae37986f2cff537a2bf28b Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 15 Feb 2021 20:59:37 +0000 Subject: [PATCH 08/16] Function node: test modules identified in libs are added to sandbox --- test/nodes/core/function/10-function_spec.js | 55 ++++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/test/nodes/core/function/10-function_spec.js b/test/nodes/core/function/10-function_spec.js index d737875cc..95bca5fb4 100644 --- a/test/nodes/core/function/10-function_spec.js +++ b/test/nodes/core/function/10-function_spec.js @@ -93,9 +93,6 @@ describe('function node', function() { - -/* - it('should be loaded', function(done) { var flow = [{id:"n1", type:"function", name: "function" }]; helper.load(functionNode, flow, function() { @@ -1417,6 +1414,57 @@ describe('function node', function() { }); }); + describe('externalModules', function() { + + it('should fail if using OS module without it listed in libs', function(done) { + var flow = [ + {id:"n1",type:"function",wires:[["n2"]],func:"msg.payload = os.type(); return msg;"}, + {id:"n2", type:"helper"} + ]; + helper.load(functionNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + var messageReceived = false; + n2.on("input", function(msg) { + messageReceived = true; + }); + n1.receive({payload:"foo",topic: "bar"}); + setTimeout(function() { + try { + messageReceived.should.be.false(); + done(); + } catch(err) { + done(err); + } + },20); + }).catch(err => done(err)); + + }) + it('should require the OS module', function(done) { + var flow = [ + {id:"n1",type:"function",wires:[["n2"]],func:"msg.payload = os.type(); return msg;", "libs": [{var:"os", module:"os"}]}, + {id:"n2", type:"helper"} + ]; + helper.load(functionNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + try { + msg.should.have.property('topic', 'bar'); + msg.should.have.property('payload', require('os').type()); + done(); + } catch(err) { + done(err); + } + }); + n1.receive({payload:"foo",topic: "bar"}); + }).catch(err => done(err)); + + }) + + }) + + describe('Logger', function () { function testLog(initCode,funcCode,expectedLevel, done) { @@ -1603,5 +1651,4 @@ describe('function node', function() { }); }) - */ }); From 785c349adc415f637371a8ddcac4eee5ea1df8fa Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Tue, 16 Feb 2021 13:58:59 +0000 Subject: [PATCH 09/16] Prevent function module overwriting built-in sandbox properties --- .../src/js/ui/common/typedInput.js | 10 +++ .../nodes/core/function/10-function.html | 69 +++++++++++++++---- .../nodes/core/function/10-function.js | 51 ++++++++------ .../nodes/locales/en-US/messages.json | 10 ++- test/nodes/core/function/10-function_spec.js | 14 +++- .../registry/lib/externalModules_spec.js | 2 +- 6 files changed, 115 insertions(+), 41 deletions(-) diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/common/typedInput.js b/packages/node_modules/@node-red/editor-client/src/js/ui/common/typedInput.js index 332f41fa9..e53372a52 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/common/typedInput.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/common/typedInput.js @@ -344,6 +344,16 @@ that.element.val(that.value()); that.element.trigger('change',that.propertyType,that.value()); }); + this.input.on('keyup', function(evt) { + that.validate(); + that.element.val(that.value()); + that.element.trigger('keyup',evt); + }); + this.input.on('paste', function(evt) { + that.validate(); + that.element.val(that.value()); + that.element.trigger('paste',evt); + }); this.input.on('keydown', function(evt) { if (evt.keyCode >= 37 && evt.keyCode <= 40) { evt.stopPropagation(); diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index 5b0e8da44..27eb588b9 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -23,6 +23,11 @@ border-left: none; border-top: none; border-right: none; + padding-top: 2px; + padding-bottom: 2px; + margin-top: 4px; + margin-bottom: 2px; + height: 26px; } .node-libs-entry > span > i { @@ -89,6 +94,25 @@ (function() { + var invalidModuleVNames = [ + 'console', + 'util', + 'Buffer', + 'Date', + 'RED', + 'node', + '__node__', + 'context', + 'flow', + 'global', + 'env', + 'setTimeout', + 'clearTimeout', + 'setInterval', + 'clearInterval', + 'promisify' + ] + var knownFunctionNodes = {}; RED.events.on("nodes:add", function(n) { if (n.type === "function") { @@ -193,6 +217,16 @@ width: "120px", "margin-left": "5px" }).appendTo(row0).val(opt.var); + var vnameWarning = $('').appendTo(row0); + RED.popover.tooltip(vnameWarning.find("i"),function() { + var val = fvar.val(); + if (invalidModuleVNames.indexOf(val) !== -1) { + return RED._("node-red:function.error.moduleNameReserved",{name:val}) + } else { + return RED._("node-red:function.error.moduleNameError",{name:val}) + } + }) + $(' = require(').appendTo(row0); var fmodule = $("", { class: "node-input-libs-val", @@ -210,29 +244,33 @@ $(')').appendTo(row0); - var warning = $('').appendTo(row0); - RED.popover.tooltip(warning.find("i"),function() { + var moduleWarning = $('').appendTo(row0); + RED.popover.tooltip(moduleWarning.find("i"),function() { var val = fmodule.typedInput("type"); if (val === "_custom_") { val = fmodule.val(); } + var errors = []; + if (!RED.utils.checkModuleAllowed(val,null,installAllowList,installDenyList)) { - return "Module not allowed" + return RED._("node-red:function.error.moduleNotAllowed",{module:val}); } else { - return "Module not installed: "+missingModuleReasons[val] + return RED._("node-red:function.error.moduleLoadError",{module:val,error:missingModuleReasons[val]}); } }) - fvar.on("change", function (e) { + fvar.on("change keyup paste", function (e) { var v = $(this).val().trim(); - if (v === "" || / /.test(v)) { + if (v === "" || / /.test(v) || invalidModuleVNames.indexOf(v) !== -1) { fvar.addClass("input-error"); + vnameWarning.addClass("input-error"); } else { fvar.removeClass("input-error"); + vnameWarning.removeClass("input-error"); } }); - fmodule.on("change", function (e) { + fmodule.on("change keyup paste", function (e) { var val = $(this).typedInput("type"); if (val === "_custom_") { val = $(this).val(); @@ -243,18 +281,18 @@ if (RED.utils.checkModuleAllowed(val,null,installAllowList,installDenyList) && (missingModules.indexOf(val) === -1)) { fmodule.removeClass("input-error"); - warning.removeClass("input-error"); + moduleWarning.removeClass("input-error"); } else { fmodule.addClass("input-error"); - warning.addClass("input-error"); + moduleWarning.addClass("input-error"); } }); if (RED.utils.checkModuleAllowed(opt.module,null,installAllowList,installDenyList) && (missingModules.indexOf(opt.module) === -1)) { fmodule.removeClass("input-error"); - warning.removeClass("input-error"); + moduleWarning.removeClass("input-error"); } else { fmodule.addClass("input-error"); - warning.addClass("input-error"); + moduleWarning.addClass("input-error"); } if (opt.var) { fvar.trigger("change"); @@ -293,6 +331,9 @@ if (missingModules.indexOf(m.module) > -1) { return false; } + if (invalidModuleVNames.indexOf(m.var) !== -1){ + return false; + } } return true; }} @@ -325,15 +366,15 @@ tabs.addTab({ id: "func-tab-init", - label: "On Start", //that._("function.label.initialize") + label: that._("function.label.initialize") }); tabs.addTab({ id: "func-tab-body", - label: "On Message"//that._("function.label.function") + label: that._("function.label.function") }); tabs.addTab({ id: "func-tab-finalize", - label: "On Stop"//that._("function.label.finalize") + label: that._("function.label.finalize") }); tabs.activateTab("func-tab-body"); diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index b9fae8626..d6abde563 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -273,6 +273,37 @@ module.exports = function(RED) { sandbox.promisify = util.promisify; } + if (node.hasOwnProperty("libs")) { + let moduleErrors = false; + var modules = node.libs; + modules.forEach(module => { + var vname = module.hasOwnProperty("var") ? module.var : null; + if (vname && (vname !== "")) { + if (sandbox.hasOwnProperty(vname) || vname === 'node') { + node.error(RED._("function.error.moduleNameError",{name:vname})) + moduleErrors = true; + return; + } + sandbox[vname] = null; + try { + var spec = module.module; + if (spec && (spec !== "")) { + var lib = RED.require(module.module); + sandbox[vname] = lib; + } + } catch (e) { + //TODO: NLS error message + node.error(RED._("function.error.moduleLoadError",{module:module.spec, error:e.toString()})) + moduleErrors = true; + } + } + }); + if (moduleErrors) { + throw new Error("Function node failed to load external modules"); + } + } + + const RESOLVING = 0; const RESOLVED = 1; const ERROR = 2; @@ -289,26 +320,6 @@ module.exports = function(RED) { } }); - if (node.hasOwnProperty("libs")) { - var modules = node.libs; - modules.forEach(module => { - var vname = module.hasOwnProperty("var") ? module.var : null; - if (vname && (vname !== "")) { - sandbox[vname] = null; - try { - var spec = module.module; - if (spec && (spec !== "")) { - var lib = RED.require(module.module); - sandbox[vname] = lib; - } - } - catch (e) { - console.log(e); - node.warn("failed to load library: "+ module.spec); - } - } - }); - } var context = vm.createContext(sandbox); try { var iniScript = null; diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index 20180f33d..5dc17f76d 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -209,9 +209,9 @@ "function": { "function": "", "label": { - "function": "Function", - "initialize": "Setup", - "finalize": "Close", + "function": "On Message", + "initialize": "On Start", + "finalize": "On Stop", "outputs": "Outputs", "require": "Require" }, @@ -224,6 +224,10 @@ "module": "module" }, "error": { + "moduleNotAllowed": "Module __module__ not allowed", + "moduleLoadError": "Failed to load module __module__: __error__", + "moduleNameError": "Invalid module variable name: __name__", + "moduleNameReserved": "Reserved variable name: __name__", "inputListener":"Cannot add listener to 'input' event within Function", "non-message-returned":"Function tried to send a message of type __type__" } diff --git a/test/nodes/core/function/10-function_spec.js b/test/nodes/core/function/10-function_spec.js index 95bca5fb4..7139c0e75 100644 --- a/test/nodes/core/function/10-function_spec.js +++ b/test/nodes/core/function/10-function_spec.js @@ -1438,7 +1438,6 @@ describe('function node', function() { } },20); }).catch(err => done(err)); - }) it('should require the OS module', function(done) { var flow = [ @@ -1459,9 +1458,18 @@ describe('function node', function() { }); n1.receive({payload:"foo",topic: "bar"}); }).catch(err => done(err)); - }) - + it('should fail if module variable name clashes with sandbox builtin', function(done) { + var flow = [ + {id:"n1",type:"function",wires:[["n2"]],func:"msg.payload = os.type(); return msg;", "libs": [{var:"flow", module:"os"}]}, + {id:"n2", type:"helper"} + ]; + helper.load(functionNode, flow, function() { + var n1 = helper.getNode("n1"); + should.not.exist(n1); + done(); + }).catch(err => done(err)); + }) }) diff --git a/test/unit/@node-red/registry/lib/externalModules_spec.js b/test/unit/@node-red/registry/lib/externalModules_spec.js index 17f3cc41d..1cd274561 100644 --- a/test/unit/@node-red/registry/lib/externalModules_spec.js +++ b/test/unit/@node-red/registry/lib/externalModules_spec.js @@ -94,7 +94,7 @@ describe("externalModules api", function() { it("installs missing modules", async function() { externalModules.init({userDir: homeDir}); externalModules.register("function", "libs"); - fs.existsSync(path.join(homeDir,"externalModuels")).should.be.false(); + fs.existsSync(path.join(homeDir,"externalModules")).should.be.false(); await externalModules.checkFlowDependencies([ {type: "function", libs:[{module: "foo"}]} ]) From 45afd060479ddf92690d36c082476456836a20c1 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Wed, 17 Feb 2021 12:03:13 +0000 Subject: [PATCH 10/16] Prevent rogue mouseup on tab from triggering tab change --- .../@node-red/editor-client/src/js/ui/common/tabs.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js index 1d6c75c32..baaac7a8b 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js @@ -29,6 +29,7 @@ RED.tabs = (function() { var currentTabWidth; var currentActiveTabWidth = 0; var collapsibleMenu; + var mousedownTab; var preferredOrder = options.order; var ul = options.element || $("#"+options.id); var wrapper = ul.wrap( "
" ).parent(); @@ -207,6 +208,11 @@ RED.tabs = (function() { if (dragActive) { return } + if (evt.currentTarget !== mousedownTab) { + mousedownTab = null; + return; + } + mousedownTab = null; if (dblClickTime && Date.now()-dblClickTime < 400) { dblClickTime = 0; dblClickArmed = true; @@ -445,6 +451,7 @@ RED.tabs = (function() { } ul.find("li.red-ui-tab a") + .on("mousedown", function(evt) { mousedownTab = evt.currentTarget }) .on("mouseup",onTabClick) .on("click", function(evt) {evt.preventDefault(); }) .on("dblclick", function(evt) {evt.stopPropagation(); evt.preventDefault(); }) @@ -636,6 +643,7 @@ RED.tabs = (function() { } } + link.on("mousedown", function(evt) { mousedownTab = evt.currentTarget }) link.on("mouseup",onTabClick); link.on("click", function(evt) { evt.preventDefault(); }) link.on("dblclick", function(evt) { evt.stopPropagation(); evt.preventDefault(); }) From ea62c1806ec6d7a78264d340813ba6b8aa5b25a5 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Wed, 17 Feb 2021 12:04:20 +0000 Subject: [PATCH 11/16] Give edit dialog a little bit more vertical space --- .../node_modules/@node-red/editor-client/src/sass/editor.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node_modules/@node-red/editor-client/src/sass/editor.scss b/packages/node_modules/@node-red/editor-client/src/sass/editor.scss index aefd617c0..869f8930d 100644 --- a/packages/node_modules/@node-red/editor-client/src/sass/editor.scss +++ b/packages/node_modules/@node-red/editor-client/src/sass/editor.scss @@ -174,8 +174,8 @@ button.red-ui-tray-resize-button { .red-ui-editor .red-ui-tray { .dialog-form, #dialog-form, #node-config-dialog-edit-form { - margin: 20px; - height: calc(100% - 40px); + margin: 10px 20px; + height: calc(100% - 20px); } } From 137fa98903efcb8a1a55c8181c5028e384b49221 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Wed, 17 Feb 2021 12:04:38 +0000 Subject: [PATCH 12/16] Move name field above tab bar in Function node --- .../nodes/core/function/10-function.html | 39 ++++++++++--------- .../nodes/locales/en-US/messages.json | 4 +- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index 27eb588b9..f107dd03f 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -1,5 +1,8 @@