align functionality of nodesDir with coreNodesDir and userDir

packages/node_modules/@node-red/registry/lib/localfilesystem.js

@@ -88,9 +88,10 @@ function getLocalFile(file) {
 /**
  * Synchronously walks the directory looking for node files.
  * @param dir the directory to search
+ * @param skipValidNodeRedModules a flag to skip lading icons & files if the directory a valid node-red module
  * @return an array of fully-qualified paths to .js files
  */
-function getLocalNodeFiles(dir) {
+function getLocalNodeFiles(dir, skipValidNodeRedModules) {
     dir = path.resolve(dir);
 
     var result = [];
@@ -102,6 +103,14 @@ function getLocalNodeFiles(dir) {
         return {files: [], icons: []};
     }
     files.sort();
+    // when loading local files, if the path is a valid node-red module
+    // dont include it (will be picked up in scanTreeForNodesModules)
+    if(skipValidNodeRedModules && files.indexOf("package.json") >= 0) {
+        const package = getPackageDetails(dir)
+        if(package.isNodeRedModule) {
+            return {files: [], icons: []};
+        }
+    }
     files.forEach(function(fn) {
         var stats = fs.statSync(path.join(dir,fn));
         if (stats.isFile()) {
@@ -114,7 +123,7 @@ function getLocalNodeFiles(dir) {
         } else if (stats.isDirectory()) {
             // Ignore /.dirs/, /lib/ /node_modules/
             if (!/^(\..*|lib|icons|node_modules|test|locales)$/.test(fn)) {
-                var subDirResults = getLocalNodeFiles(path.join(dir,fn));
+                var subDirResults = getLocalNodeFiles(path.join(dir,fn), skipValidNodeRedModules);
                 result = result.concat(subDirResults.files);
                 icons = icons.concat(subDirResults.icons);
             } else if (fn === "icons") {
@@ -126,21 +135,30 @@ function getLocalNodeFiles(dir) {
     return {files: result, icons: icons}
 }
 
-function scanDirForNodesModules(dir,moduleName) {
+function scanDirForNodesModules(dir,moduleName,package) {
-    var results = [];
+    let results = [];
-    var scopeName;
+    let scopeName;
+    let files
     try {
-        var files = fs.readdirSync(dir);
+        let isNodeRedModule = false
-        if (moduleName) {
+        if(package) {
-            var m = /^(?:(@[^/]+)[/])?([^@/]+)/.exec(moduleName);
+            dir = path.join(package.moduleDir,'..')
-            if (m) {
+            files = [path.basename(package.moduleDir)]
-                scopeName = m[1];
+            moduleName =  (package.package ? package.package.name : null) || moduleName
-                moduleName = m[2];
+            isNodeRedModule = package.isNodeRedModule
+        } else {
+            files = fs.readdirSync(dir);
+            if (moduleName) {
+                var m = /^(?:(@[^/]+)[/])?([^@/]+)/.exec(moduleName);
+                if (m) {
+                    scopeName = m[1];
+                    moduleName = m[2];
+                }
             }
         }
-        for (var i=0;i<files.length;i++) {
+        for (let i=0;i<files.length;i++) {
-            var fn = files[i];
+            let fn = files[i];
-            if (/^@/.test(fn)) {
+            if (!isNodeRedModule && /^@/.test(fn)) {
                 if (scopeName && scopeName === fn) {
                     // Looking for a specific scope/module
                     results = results.concat(scanDirForNodesModules(path.join(dir,fn),moduleName));
@@ -149,16 +167,18 @@ function scanDirForNodesModules(dir,moduleName) {
                     results = results.concat(scanDirForNodesModules(path.join(dir,fn),moduleName));
                 }
             } else {
-                if (isIncluded(fn) && !isExcluded(fn) && (!moduleName || fn == moduleName)) {
+                if ((isNodeRedModule || (!moduleName || fn == moduleName)) && (isIncluded(fn) && !isExcluded(fn))) {
-                    var pkgfn = path.join(dir,fn,"package.json");
                     try {
-                        var pkg = require(pkgfn);
+                        const moduleDir = isNodeRedModule ? package.moduleDir : path.join(dir,fn);
-                        if (pkg['node-red']) {
+                        const pkg = package || getPackageDetails(moduleDir)
-                            if (!registryUtil.checkModuleAllowed(pkg.name,pkg.version,loadAllowList,loadDenyList)) {
+                        if(pkg.error) {
-                                log.debug("! Module: "+pkg.name+" "+pkg.version+ " *ignored due to denyList*");
+                            throw pkg.error
+                        }
+                        if (pkg.isNodeRedModule) {
+                            if (!pkg.allowed) {
+                                log.debug("! Module: "+pkg.package.name+" "+pkg.package.version+ " *ignored due to denyList*");
                             } else {
-                                var moduleDir = path.join(dir,fn);
+                                results.push({dir:moduleDir,package:pkg.package});
-                                results.push({dir:moduleDir,package:pkg});
                             }
                         }
                     } catch(err) {
@@ -182,11 +202,14 @@ function scanDirForNodesModules(dir,moduleName) {
  * @param moduleName the name of the module to be found
  * @return a list of node modules: {dir,package}
  */
-function scanTreeForNodesModules(moduleName) {
+ function scanTreeForNodesModules(moduleName) {
-    var dir = settings.coreNodesDir;
+    let coreNodesDir = settings.coreNodesDir;
-    var results = [];
+    let results = [];
-    var userDir;
+    let userDir;
-
+    let nodesDir;
+    if(settings.nodesDir) {
+        nodesDir = Array.isArray(settings.nodesDir) ? settings.nodesDir : [settings.nodesDir]
+    }
     if (settings.userDir) {
         packageList = getPackageList();
         userDir = path.join(settings.userDir,"node_modules");
@@ -201,15 +224,46 @@ function scanTreeForNodesModules(moduleName) {
         });
     }
 
-    if (dir) {
+    if (coreNodesDir) {
-        var up = path.resolve(path.join(dir,".."));
+        var up = path.resolve(path.join(coreNodesDir,".."));
-        while (up !== dir) {
+        while (up !== coreNodesDir) {
-            var pm = path.join(dir,"node_modules");
+            var pm = path.join(coreNodesDir,"node_modules");
             if (pm != userDir) {
                 results = results.concat(scanDirForNodesModules(pm,moduleName));
             }
-            dir = up;
+            coreNodesDir = up;
-            up = path.resolve(path.join(dir,".."));
+            up = path.resolve(path.join(coreNodesDir,".."));
+        }
+    }
+
+    // scan nodesDir for any node-red modules
+    /*
+    1. if !exist(package.json) || !package.json.has(node-red) => look for node_modules
+    2. exist(package.json) && package.json.has(node-red) => load this only
+    3. in original scan of nodesDir, ignore if:(exist(package.json) && package.json.has(node-red)) 
+    */
+    if (nodesDir) {
+        for (let dirIndex = 0; dirIndex < nodesDir.length; dirIndex++) {
+            const nodeDir = nodesDir[dirIndex];
+            const packageDetails = getPackageDetails(nodeDir)
+            if(packageDetails.isNodeRedModule) {
+                //we have found a node-red module, scan it
+                const nrModules = scanDirForNodesModules(nodeDir, packageDetails.package.name, packageDetails);
+                results = results.concat(nrModules);
+
+            } else if (packageDetails.has_node_modules) {
+                //If this dir has a `node_modues` dir, scan it
+                const nodeModulesDir = path.join(nodeDir, 'node_modules')
+                const nrModules = scanDirForNodesModules(nodeModulesDir, moduleName );
+                results = results.concat(nrModules);
+
+            } else {
+                //If this is not a node-red module AND it does NOT have a node_modules dir,
+                //it may be a directory of project directories or a node_modules dir?
+                //scan this instead
+                const nrModules = scanDirForNodesModules(nodeDir, moduleName);
+                results = results.concat(nrModules);
+            }
         }
     }
     return results;
@@ -274,24 +328,26 @@ function getModuleNodeFiles(module) {
 }
 
 function getNodeFiles(disableNodePathScan) {
-    var dir;
     // Find all of the nodes to load
-    var nodeFiles = [];
+    let results;
-    var results;
+    let nodesDir;
-
+    if(settings.nodesDir) {
-    var dir;
+        nodesDir = Array.isArray(settings.nodesDir) ? settings.nodesDir : [settings.nodesDir]
-    var iconList = [];
+    }
+    let dir;
+    let nodeFiles = [];
+    let iconList = [];
     if (settings.coreNodesDir) {
         results = getLocalNodeFiles(path.resolve(settings.coreNodesDir));
         nodeFiles = nodeFiles.concat(results.files);
         iconList = iconList.concat(results.icons);
-        var defaultLocalesPath = path.join(settings.coreNodesDir,"locales");
+        let defaultLocalesPath = path.join(settings.coreNodesDir,"locales");
         i18n.registerMessageCatalog("node-red",defaultLocalesPath,"messages.json");
     }
 
     if (settings.userDir) {
         dir = path.join(settings.userDir,"lib","icons");
-        var icons = scanIconDir(dir);
+        let icons = scanIconDir(dir);
         if (icons.length > 0) {
             iconList.push({path:dir,icons:icons});
         }
@@ -301,13 +357,9 @@ function getNodeFiles(disableNodePathScan) {
         nodeFiles = nodeFiles.concat(results.files);
         iconList = iconList.concat(results.icons);
     }
-    if (settings.nodesDir) {
+    if (nodesDir) {
-        dir = settings.nodesDir;
+        for (let i = 0; i < nodesDir.length; i++) {
-        if (typeof settings.nodesDir == "string") {
+            results = getLocalNodeFiles(nodesDir[i], true);
-            dir = [dir];
-        }
-        for (var i=0;i<dir.length;i++) {
-            results = getLocalNodeFiles(dir[i]);
             nodeFiles = nodeFiles.concat(results.files);
             iconList = iconList.concat(results.icons);
         }
@@ -479,7 +531,52 @@ function getPackageList() {
     }
     return list;
 }
-
+/**
+ * Gets the package json object for the supplied `dir`.  
+ * If there is no package.json or the `node-red` section is missing, `result.isNodeRedModule` will be `false`.
+ * If there is no package.json `isPackage` will be `false`.
+ * If an error occurs, `result.error` will contain the error.
+ * @param {string} dir The directory to inspect
+ */
+ function getPackageDetails(dir) {
+    const result = {
+        /** @type {string} The package directory */ 
+        moduleDir: dir,
+        /** @type {string} The full file path of package.json for this package */
+        packageFile: null,
+        /** @type {boolean} True if this is a valid node-red module */
+        isNodeRedModule: false,
+        /** @type {boolean} True if a package.json file is present */
+        isPackage: false,
+        /** @type {boolean} True if this a node-red module and passes the checks */
+        allowed: false,
+        /** @type {object} The contents of package.json */
+        package: null,
+    }
+    if (!dir) {  return result }
+    try {
+        const packagefile = path.join(dir,'package.json')
+        result.has_node_modules = fs.existsSync(path.join(dir,'node_modules'))
+        if(!fs.existsSync(packagefile)) {
+            return result
+        }
+        result.packageFile = packagefile
+        const pkg = require(packagefile)
+        result.package = pkg
+        if(result.package) {
+            result.allowed = true
+            result.isPackage = true
+            result.isNodeRedModule = typeof result.package['node-red'] === 'object'
+            if(result.isNodeRedModule) {
+                result.isNodeRedModule = true;
+                result.allowed = registryUtil.checkModuleAllowed(pkg.name,pkg.version,loadAllowList,loadDenyList) 
+            }
+        }
+    } catch(err) {
+        result.error = err; // this is not a package we are interested in!
+    }
+    return result || result;
+}
 module.exports = {
     init: init,
     getNodeFiles: getNodeFiles,