Merge pull request #2873 from node-red/function-modules

Function node external modules

packages/node_modules/@node-red/registry/lib/externalModules.js

@@ -0,0 +1,223 @@
+// This module handles the management of modules required by the runtime and flows.
+// Essentially this means keeping track of what extra modules a flow requires,
+// ensuring those modules are installed and providing a standard way for nodes
+// to require those modules safely.
+
+const fs = require("fs-extra");
+const registryUtil = require("./util");
+const path = require("path");
+const clone = require("clone");
+const exec = require("@node-red/util").exec;
+const log = require("@node-red/util").log;
+
+const BUILTIN_MODULES = require('module').builtinModules;
+const EXTERNAL_MODULES_DIR = "externalModules";
+
+// TODO: outsource running npm to a plugin
+const NPM_COMMAND = (process.platform === "win32") ? "npm.cmd" : "npm";
+
+let registeredTypes = {};
+let subflowTypes = {};
+let settings;
+
+let knownExternalModules = {};
+
+let installEnabled = true;
+let installAllowList = ['*'];
+let installDenyList = [];
+
+function getInstallDir() {
+    return path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "externalModules"));
+}
+
+async function refreshExternalModules() {
+    const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR));
+    try {
+        const pkgFile = JSON.parse(await fs.readFile(path.join(externalModuleDir,"package.json"),"utf-8"));
+        knownExternalModules = pkgFile.dependencies;
+    } catch(err) {
+    }
+}
+
+function init(_settings) {
+    settings = _settings;
+    knownExternalModules = {};
+    installEnabled = true;
+    if (settings.externalModules && settings.externalModules.modules) {
+        if (settings.externalModules.modules.allowList || settings.externalModules.modules.denyList) {
+            installAllowList = settings.externalModules.modules.allowList;
+            installDenyList = settings.externalModules.modules.denyList;
+        }
+        if (settings.externalModules.modules.hasOwnProperty("allowInstall")) {
+            installEnabled = settings.externalModules.modules.allowInstall
+        }
+    }
+    installAllowList = registryUtil.parseModuleList(installAllowList);
+    installDenyList = registryUtil.parseModuleList(installDenyList);
+}
+
+function register(type, dynamicModuleListProperty) {
+    registeredTypes[type] = dynamicModuleListProperty;
+}
+
+function registerSubflow(type, subflowConfig) {
+    subflowTypes[type] = subflowConfig;
+}
+
+function requireModule(module) {
+    if (!registryUtil.checkModuleAllowed( module, null,installAllowList,installDenyList)) {
+        const e = new Error("Module not allowed");
+        e.code = "module_not_allowed";
+        throw e;
+    }
+    if (BUILTIN_MODULES.indexOf(module) !== -1) {
+        return require(module);
+    }
+    if (!knownExternalModules[module]) {
+        const e = new Error("Module not allowed");
+        e.code = "module_not_allowed";
+        throw e;
+    }
+    const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR));
+    const moduleDir = path.join(externalModuleDir,"node_modules",module);
+    return require(moduleDir);
+}
+
+function parseModuleName(module) {
+    var match = /((?:@[^/]+\/)?[^/@]+)(?:@([\s\S]+))?/.exec(module);
+    if (match) {
+        return {
+            spec: module,
+            module: match[1],
+            version: match[2],
+            builtin: BUILTIN_MODULES.indexOf(match[1]) !== -1,
+            known: !!knownExternalModules[match[1]]
+        }
+    }
+    return null;
+}
+
+function isInstalled(moduleDetails) {
+    return moduleDetails.builtin || moduleDetails.known;
+}
+
+
+async function checkFlowDependencies(flowConfig) {
+    let nodes = clone(flowConfig);
+    await refreshExternalModules();
+
+    const checkedModules = {};
+    const promises = [];
+    const errors = [];
+    const checkedSubflows = {};
+    while (nodes.length > 0) {
+        let n = nodes.shift();
+        if (subflowTypes[n.type] && !checkedSubflows[n.type]) {
+            checkedSubflows[n.type] = true;
+            nodes = nodes.concat(subflowTypes[n.type].flow)
+        } else if (registeredTypes[n.type]) {
+            let nodeModules = n[registeredTypes[n.type]] || [];
+            if (!Array.isArray(nodeModules)) {
+                nodeModules = [nodeModules]
+            }
+            nodeModules.forEach(module => {
+                if (typeof module !== 'string') {
+                    module = module.module || "";
+                }
+                if (module) {
+                    let moduleDetails = parseModuleName(module)
+                    if (moduleDetails && checkedModules[moduleDetails.module] === undefined) {
+                        checkedModules[moduleDetails.module] = isInstalled(moduleDetails)
+                        if (!checkedModules[moduleDetails.module]) {
+                            if (installEnabled) {
+                                promises.push(installModule(moduleDetails).catch(err => {
+                                    errors.push({module: moduleDetails,error:err});
+                                }))
+                            } else if (!installEnabled) {
+                                const e = new Error("Module install disabled - externalModules.modules.allowInstall=false");
+                                e.code = "install_not_allowed";
+                                errors.push({module: moduleDetails,error:e});
+                            }
+                        } else if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) {
+                            const e = new Error("Module not allowed");
+                            e.code = "module_not_allowed";
+                            errors.push({module: moduleDetails,error:e});
+                        }
+                    }
+                }
+            })
+        }
+    }
+    return Promise.all(promises).then(refreshExternalModules).then(() => {
+        if (errors.length > 0) {
+            throw errors;
+        }
+    })
+}
+
+
+async function ensureModuleDir() {
+    const installDir = getInstallDir();
+
+    if (!fs.existsSync(installDir)) {
+        await fs.ensureDir(installDir);
+    }
+    const pkgFile = path.join(installDir,"package.json");
+    if (!fs.existsSync(pkgFile)) {
+        await fs.writeFile(path.join(installDir,"package.json"),`{
+    "name": "Node-RED-External-Modules",
+    "description": "These modules are automatically installed by Node-RED to use in Function nodes.",
+    "version": "1.0.0",
+    "private": true,
+    "dependencies": {}
+}`)
+    }
+}
+
+async function installModule(moduleDetails) {
+    let installSpec = moduleDetails.module;
+    if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) {
+        const e = new Error("Install not allowed");
+        e.code = "install_not_allowed";
+        throw e;
+    }
+    if (moduleDetails.version) {
+        installSpec = installSpec+"@"+moduleDetails.version;
+    }
+    log.info(log._("server.install.installing",{name: moduleDetails.module,version: moduleDetails.version||"latest"}));
+    const installDir = getInstallDir();
+
+    await ensureModuleDir();
+
+    var args = ["install", installSpec, "--production"];
+    return exec.run(NPM_COMMAND, args, {
+        cwd: installDir
+    },true).then(result => {
+        log.info("successfully installed: "+installSpec);
+    }).catch(result => {
+        var output = result.stderr;
+        var e;
+        if (/E404/.test(output) || /ETARGET/.test(output)) {
+            log.error(log._("server.install.install-failed-not-found",{name:installSpec}));
+            e = new Error("Module not found");
+            e.code = 404;
+            throw e;
+        } else {
+            log.error(log._("server.install.install-failed-long",{name:installSpec}));
+            log.error("------------------------------------------");
+            log.error(output);
+            log.error("------------------------------------------");
+            e = new Error(log._("server.install.install-failed"));
+            e.code = "unexpected_error";
+            throw e;
+        }
+    })
+}
+
+module.exports = {
+    init: init,
+    register: register,
+    registerSubflow: registerSubflow,
+    checkFlowDependencies: checkFlowDependencies,
+    require: requireModule
+}

packages/node_modules/@node-red/registry/lib/index.js

@@ -28,6 +28,7 @@ var registry = require("./registry");
 var loader = require("./loader");
 var installer = require("./installer");
 var library = require("./library");
+const externalModules = require("./externalModules")
 var plugins = require("./plugins");
 
 /**
@@ -44,6 +45,7 @@ function init(runtime) {
     plugins.init(runtime.settings);
     registry.init(runtime.settings,loader);
     library.init();
+    externalModules.init(runtime.settings);
 }
 
 /**
@@ -299,6 +301,8 @@ module.exports = {
      */
     getNodeExampleFlowPath: library.getExampleFlowPath,
 
+    checkFlowDependencies: externalModules.checkFlowDependencies,
+
     registerPlugin: plugins.registerPlugin,
     getPlugin: plugins.getPlugin,
     getPluginsByType: plugins.getPluginsByType,

packages/node_modules/@node-red/registry/lib/registry.js

@@ -21,6 +21,7 @@ var fs = require("fs");
 var library = require("./library");
 const {events} = require("@node-red/util")
 var subflows = require("./subflow");
+var externalModules = require("./externalModules")
 var settings;
 var loader;
 
@@ -28,6 +29,7 @@ var nodeConfigCache = {};
 var moduleConfigs = {};
 var nodeList = [];
 var nodeConstructors = {};
+var nodeOptions = {};
 var subflowModules = {};
 
 var nodeTypeToId = {};
@@ -36,12 +38,7 @@ var moduleNodes = {};
 function init(_settings,_loader) {
     settings = _settings;
     loader = _loader;
-    moduleNodes = {};
-    nodeTypeToId = {};
-    nodeConstructors = {};
-    subflowModules = {};
-    nodeList = [];
-    nodeConfigCache = {};
+    clear();
 }
 
 function load() {
@@ -241,6 +238,7 @@ function removeNode(id) {
         if (typeId === id) {
             delete subflowModules[t];
             delete nodeConstructors[t];
+            delete nodeOptions[t];
             delete nodeTypeToId[t];
         }
     });
@@ -412,7 +410,7 @@ function getCaller(){
     return stack[0].getFileName();
 }
 
-function registerNodeConstructor(nodeSet,type,constructor) {
+function registerNodeConstructor(nodeSet,type,constructor,options) {
     if (nodeConstructors.hasOwnProperty(type)) {
         throw new Error(type+" already registered");
     }
@@ -432,6 +430,12 @@ function registerNodeConstructor(nodeSet,type,constructor) {
     }
 
     nodeConstructors[type] = constructor;
+    nodeOptions[type] = options;
+    if (options) {
+        if (options.dynamicModuleList) {
+            externalModules.register(type,options.dynamicModuleList);
+        }
+    }
     events.emit("type-registered",type);
 }
 
@@ -452,6 +456,9 @@ function registerSubflow(nodeSet, subflow) {
         nodeSetInfo.config = result.config;
     }
     subflowModules[result.type] = result;
+    externalModules.registerSubflow(result.type,subflow);
+
+
     events.emit("type-registered",result.type);
     return result;
 }
@@ -524,6 +531,7 @@ function clear() {
     moduleConfigs = {};
     nodeList = [];
     nodeConstructors = {};
+    nodeOptions = {};
     subflowModules = {};
     nodeTypeToId = {};
 }

packages/node_modules/@node-red/registry/lib/util.js

@@ -17,6 +17,7 @@
 const path = require("path");
 const semver = require("semver");
 const {events,i18n,log} = require("@node-red/util");
+
 var runtime;
 
 function copyObjectProperties(src,dst,copyList,blockList) {
@@ -45,9 +46,8 @@ function requireModule(name) {
         var relPath = path.relative(__dirname, moduleInfo.path);
         return require(relPath);
     } else {
-        var err = new Error(`Cannot find module '${name}'`);
-        err.code = "MODULE_NOT_FOUND";
-        throw err;
+        // Require it here to avoid the circular dependency
+        return require("./externalModules").require(name);
     }
 }
 
@@ -90,7 +90,7 @@ function createNodeApi(node) {
         httpAdmin: runtime.adminApp,
         server: runtime.server
     }
-    copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials" ]);
+    copyObjectProperties(runtime.nodes,red.nodes,["createNode","getNode","eachNode","addCredentials","getCredentials","deleteCredentials"]);
     red.nodes.registerType = function(type,constructor,opts) {
         runtime.nodes.registerType(node.id,type,constructor,opts);
     }
@@ -136,7 +136,6 @@ function checkAgainstList(module,version,list) {
 }
 
 function checkModuleAllowed(module,version,allowList,denyList) {
-    // console.log("checkModuleAllowed",module,version);//,allowList,denyList)
     if (!allowList && !denyList) {
         // Default to allow
         return true;