Rework Function node module integration

packages/node_modules/@node-red/registry/lib/externalModules.js

@@ -0,0 +1,209 @@
+// This module handles the management of modules required by the runtime and flows.
+// Essentially this means keeping track of what extra modules a flow requires,
+// ensuring those modules are installed and providing a standard way for nodes
+// to require those modules safely.
+
+const fs = require("fs-extra");
+const registryUtil = require("./util");
+const path = require("path");
+const exec = require("@node-red/util").exec;
+const log = require("@node-red/util").log;
+
+const BUILTIN_MODULES = require('module').builtinModules;
+const EXTERNAL_MODULES_DIR = "externalModules";
+
+// TODO: outsource running npm to a plugin
+const NPM_COMMAND = (process.platform === "win32") ? "npm.cmd" : "npm";
+
+let registeredTypes = {};
+let settings;
+
+let knownExternalModules = {};
+
+let installEnabled = true;
+let installAllowList = ['*'];
+let installDenyList = [];
+
+function getInstallDir() {
+    return path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "externalModules"));
+}
+
+async function refreshExternalModules() {
+    const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR));
+    try {
+        const pkgFile = JSON.parse(await fs.readFile(path.join(externalModuleDir,"package.json"),"utf-8"));
+        knownExternalModules = pkgFile.dependencies;
+    } catch(err) {
+    }
+}
+
+function init(_settings) {
+    settings = _settings;
+    path.resolve(settings.userDir || process.env.NODE_RED_HOME || ".");
+
+    if (settings.externalModules && settings.externalModules.modules) {
+        if (settings.externalModules.modules.allowList || settings.externalModules.modules.denyList) {
+            installAllowList = settings.externalModules.modules.allowList;
+            installDenyList = settings.externalModules.modules.denyList;
+        }
+        if (settings.externalModules.modules.hasOwnProperty("allowInstall")) {
+            installEnabled = settings.externalModules.modules.allowInstall
+        }
+    }
+    installAllowList = registryUtil.parseModuleList(installAllowList);
+    installDenyList = registryUtil.parseModuleList(installDenyList);
+}
+
+function register(type, dynamicModuleListProperty) {
+    registeredTypes[type] = dynamicModuleListProperty;
+}
+
+function requireModule(module) {
+    if (!registryUtil.checkModuleAllowed( module, null,installAllowList,installDenyList)) {
+        const e = new Error("Module not allowed");
+        e.code = "module_not_allowed";
+        throw e;
+    }
+    if (BUILTIN_MODULES.indexOf(module) !== -1) {
+        return require(module);
+    }
+    if (!knownExternalModules[module]) {
+        const e = new Error("Module not allowed");
+        e.code = "module_not_allowed";
+        throw e;
+    }
+    const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR));
+    const moduleDir = path.join(externalModuleDir,"node_modules",module);
+    return require(moduleDir);
+}
+
+function parseModuleName(module) {
+    var match = /((?:@[^/]+\/)?[^/@]+)(?:@([\s\S]+))?/.exec(module);
+    if (match) {
+        return {
+            spec: module,
+            module: match[1],
+            version: match[2],
+            builtin: BUILTIN_MODULES.indexOf(match[1]) !== -1,
+            known: !!knownExternalModules[match[1]]
+        }
+    }
+    return null;
+}
+
+function isInstalled(moduleDetails) {
+    return moduleDetails.builtin || moduleDetails.known;
+}
+
+async function checkFlowDependencies(flowConfig) {
+    await refreshExternalModules();
+
+    const checkedModules = {};
+    const promises = [];
+    const errors = [];
+
+    flowConfig.forEach(n => {
+        if (registeredTypes[n.type]) {
+            let nodeModules = n[registeredTypes[n.type]] || [];
+            if (!Array.isArray(nodeModules)) {
+                nodeModules = [nodeModules]
+            }
+            nodeModules.forEach(module => {
+                if (typeof module !== 'string') {
+                    module = module.module || "";
+                }
+                if (module) {
+                    let moduleDetails = parseModuleName(module)
+                    if (moduleDetails && checkedModules[moduleDetails.module] === undefined) {
+                        checkedModules[moduleDetails.module] = isInstalled(moduleDetails)
+                        if (!checkedModules[moduleDetails.module]) {
+                            if (installEnabled) {
+                                promises.push(installModule(moduleDetails).catch(err => {
+                                    errors.push({module: moduleDetails,error:err});
+                                }))
+                            } else if (!installEnabled) {
+                                const e = new Error("Module install disabled - externalModules.modules.allowInstall=false");
+                                e.code = "install_not_allowed";
+                                errors.push({module: moduleDetails,error:e});
+                            }
+                        } else if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) {
+                            const e = new Error("Module not allowed");
+                            e.code = "module_not_allowed";
+                            errors.push({module: moduleDetails,error:e});
+                        }
+                    }
+                }
+            })
+        }
+    })
+
+    return Promise.all(promises).then(() => {
+        if (errors.length > 0) {
+            throw errors;
+        }
+    })
+}
+
+
+async function ensureModuleDir() {
+    const installDir = getInstallDir();
+
+    if (!fs.existsSync(installDir)) {
+        await fs.ensureDir(installDir);
+    }
+    const pkgFile = path.join(installDir,"package.json");
+    if (!fs.existsSync(pkgFile)) {
+        await fs.writeFile(path.join(installDir,"package.json"),`{
+    "name": "Node-RED External Modules",
+    "version": "1.0.0",
+    "private": true,
+    "dependencies": {}
+}`)
+    }
+}
+
+async function installModule(moduleDetails) {
+    let installSpec = moduleDetails.module;
+    if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) {
+        const e = new Error("Install not allowed");
+        e.code = "install_not_allowed";
+        throw e;
+    }
+    if (moduleDetails.version) {
+        installSpec = installSpec+"@"+moduleDetails.version;
+    }
+    log.info(log._("server.install.installing",{name: moduleDetails.module,version: moduleDetails.version||"latest"}));
+    const installDir = getInstallDir();
+
+    await ensureModuleDir();
+
+    var args = ["install", installSpec, "--production"];
+    return exec.run(NPM_COMMAND, args, {
+        cwd: installDir
+    },true).then(result => {
+        log.info("successfully installed: "+installSpec);
+    }).catch(result => {
+        var output = result.stderr;
+        var e;
+        var lookForVersionNotFound = new RegExp("version not found: ","m");
+        if (/E404/.test(output) || /ETARGET/.test(output)) {
+            log.error(log._("server.install.install-failed-not-found",{name:installSpec}));
+            e = new Error("Module not found");
+            e.code = 404;
+            throw e;
+        } else {
+            log.error(log._("server.install.install-failed-long",{name:installSpec}));
+            log.error("------------------------------------------");
+            log.error(output);
+            log.error("------------------------------------------");
+            throw new Error(log._("server.install.install-failed"));
+        }
+    })
+}
+
+module.exports = {
+    init: init,
+    register: register,
+    checkFlowDependencies: checkFlowDependencies,
+    require: requireModule
+}

packages/node_modules/@node-red/registry/lib/index.js

@@ -28,6 +28,7 @@ var registry = require("./registry");
 var loader = require("./loader");
 var installer = require("./installer");
 var library = require("./library");
+const externalModules = require("./externalModules")
 
 /**
  * Initialise the registry with a reference to a runtime object
@@ -42,6 +43,7 @@ function init(runtime) {
     loader.init(runtime);
     registry.init(runtime.settings,loader);
     library.init();
+    externalModules.init(runtime.settings);
 }
 
 /**
@@ -297,6 +299,8 @@ module.exports = {
      */
     getNodeExampleFlowPath: library.getExampleFlowPath,
 
+    checkFlowDependencies: externalModules.checkFlowDependencies,
+
     deprecated: require("./deprecated")
 
 };

packages/node_modules/@node-red/registry/lib/registry.js

@@ -21,6 +21,7 @@ var fs = require("fs");
 var library = require("./library");
 const {events} = require("@node-red/util")
 var subflows = require("./subflow");
+var externalModules = require("./externalModules")
 var settings;
 var loader;
 
@@ -28,6 +29,7 @@ var nodeConfigCache = {};
 var moduleConfigs = {};
 var nodeList = [];
 var nodeConstructors = {};
+var nodeOptions = {};
 var subflowModules = {};
 
 var nodeTypeToId = {};
@@ -36,12 +38,7 @@ var moduleNodes = {};
 function init(_settings,_loader) {
     settings = _settings;
     loader = _loader;
-    moduleNodes = {};
-    nodeTypeToId = {};
-    nodeConstructors = {};
-    subflowModules = {};
-    nodeList = [];
-    nodeConfigCache = {};
+    clear();
 }
 
 function load() {
@@ -234,6 +231,7 @@ function removeNode(id) {
         if (typeId === id) {
             delete subflowModules[t];
             delete nodeConstructors[t];
+            delete nodeOptions[t];
             delete nodeTypeToId[t];
         }
     });
@@ -411,7 +409,7 @@ function getCaller(){
     return stack[0].getFileName();
 }
 
-function registerNodeConstructor(nodeSet,type,constructor) {
+function registerNodeConstructor(nodeSet,type,constructor,options) {
     if (nodeConstructors.hasOwnProperty(type)) {
         throw new Error(type+" already registered");
     }
@@ -431,6 +429,12 @@ function registerNodeConstructor(nodeSet,type,constructor) {
     }
 
     nodeConstructors[type] = constructor;
+    nodeOptions[type] = options;
+    if (options) {
+        if (options.dynamicModuleList) {
+            externalModules.register(type,options.dynamicModuleList);
+        }
+    }
     events.emit("type-registered",type);
 }
 
@@ -525,6 +529,7 @@ function clear() {
     moduleConfigs = {};
     nodeList = [];
     nodeConstructors = {};
+    nodeOptions = {};
     subflowModules = {};
     nodeTypeToId = {};
 }

packages/node_modules/@node-red/registry/lib/util.js

@@ -17,6 +17,7 @@
 const path = require("path");
 const semver = require("semver");
 const {events,i18n,log} = require("@node-red/util");
+
 var runtime;
 
 function copyObjectProperties(src,dst,copyList,blockList) {
@@ -45,13 +46,8 @@ function requireModule(name) {
         var relPath = path.relative(__dirname, moduleInfo.path);
         return require(relPath);
     } else {
-        var npm = runtime.nodes.loadNPMModule(name);
-        if (npm) {
-            return npm;
-        }
-        var err = new Error(`Cannot find module '${name}'`);
-        err.code = "MODULE_NOT_FOUND";
-        throw err;
+        // Require it here to avoid the circular dependency
+        return require("./externalModules").require(name);
     }
 }
 
@@ -129,7 +125,6 @@ function checkAgainstList(module,version,list) {
 }
 
 function checkModuleAllowed(module,version,allowList,denyList) {
-    // console.log("checkModuleAllowed",module,version);//,allowList,denyList)
     if (!allowList && !denyList) {
         // Default to allow
         return true;