Merge branch 'dev' into function-modules

This commit is contained in:
Nick O'Leary
2021-02-13 00:21:27 +00:00
122 changed files with 3732 additions and 545 deletions

View File

@@ -29,6 +29,7 @@ var loader = require("./loader");
var installer = require("./installer");
var library = require("./library");
const externalModules = require("./externalModules")
var plugins = require("./plugins");
/**
* Initialise the registry with a reference to a runtime object
@@ -41,6 +42,7 @@ function init(runtime) {
// the util module it. The Util module is responsible for constructing the
// RED object passed to node modules when they are loaded.
loader.init(runtime);
plugins.init(runtime.settings);
registry.init(runtime.settings,loader);
library.init();
externalModules.init(runtime.settings);
@@ -301,6 +303,12 @@ module.exports = {
checkFlowDependencies: externalModules.checkFlowDependencies,
registerPlugin: plugins.registerPlugin,
getPlugin: plugins.getPlugin,
getPluginsByType: plugins.getPluginsByType,
getPluginList: plugins.getPluginList,
getPluginConfigs: plugins.getPluginConfigs,
deprecated: require("./deprecated")
};

View File

@@ -27,17 +27,19 @@ async function getFlowsFromPath(path) {
var validFiles = [];
return fs.readdir(path).then(files => {
var promises = [];
files.forEach(function(file) {
var fullPath = fspath.join(path,file);
var stats = fs.lstatSync(fullPath);
if (stats.isDirectory()) {
validFiles.push(file);
promises.push(getFlowsFromPath(fullPath));
} else if (/\.json$/.test(file)){
validFiles.push(file);
promises.push(Promise.resolve(file.split(".")[0]))
}
})
if (files) {
files.forEach(function(file) {
var fullPath = fspath.join(path,file);
var stats = fs.lstatSync(fullPath);
if (stats.isDirectory()) {
validFiles.push(file);
promises.push(getFlowsFromPath(fullPath));
} else if (/\.json$/.test(file)){
validFiles.push(file);
promises.push(Promise.resolve(file.split(".")[0]))
}
})
}
return Promise.all(promises)
}).then(results => {
results.forEach(function(r,i) {

View File

@@ -36,78 +36,140 @@ function load(disableNodePathScan) {
// To skip node scan, the following line will use the stored node list.
// We should expose that as an option at some point, although the
// performance gains are minimal.
//return loadNodeFiles(registry.getModuleList());
//return loadModuleFiles(registry.getModuleList());
log.info(log._("server.loading"));
var nodeFiles = localfilesystem.getNodeFiles(disableNodePathScan);
return loadNodeFiles(nodeFiles);
var modules = localfilesystem.getNodeFiles(disableNodePathScan);
return loadModuleFiles(modules);
}
function loadNodeFiles(nodeFiles) {
function loadModuleTypeFiles(module, type) {
const things = module[type];
var first = true;
var promises = [];
var nodes = [];
for (var module in nodeFiles) {
for (var thingName in things) {
/* istanbul ignore else */
if (nodeFiles.hasOwnProperty(module)) {
if (nodeFiles[module].redVersion &&
!semver.satisfies((settings.version||"0.0.0").replace(/(\-[1-9A-Za-z-][0-9A-Za-z-\.]*)?(\+[0-9A-Za-z-\.]+)?$/,""), nodeFiles[module].redVersion)) {
if (things.hasOwnProperty(thingName)) {
if (module.name != "node-red" && first) {
// Check the module directory exists
first = false;
var fn = things[thingName].file;
var parts = fn.split("/");
var i = parts.length-1;
for (;i>=0;i--) {
if (parts[i] == "node_modules") {
break;
}
}
var moduleFn = parts.slice(0,i+2).join("/");
try {
var stat = fs.statSync(moduleFn);
} catch(err) {
// Module not found, don't attempt to load its nodes
break;
}
}
try {
var promise;
if (type === "nodes") {
promise = loadNodeConfig(things[thingName]);
} else if (type === "plugins") {
promise = loadPluginConfig(things[thingName]);
}
promises.push(
promise.then(
(function() {
var m = module.name;
var n = thingName;
return function(nodeSet) {
things[n] = nodeSet;
return nodeSet;
}
})()
).catch(err => {console.log(err)})
);
} catch(err) {
console.log(err)
//
}
}
}
return promises;
}
function loadModuleFiles(modules) {
var pluginPromises = [];
var nodePromises = [];
for (var module in modules) {
/* istanbul ignore else */
if (modules.hasOwnProperty(module)) {
if (modules[module].redVersion &&
!semver.satisfies((settings.version||"0.0.0").replace(/(\-[1-9A-Za-z-][0-9A-Za-z-\.]*)?(\+[0-9A-Za-z-\.]+)?$/,""), modules[module].redVersion)) {
//TODO: log it
log.warn("["+module+"] "+log._("server.node-version-mismatch",{version:nodeFiles[module].redVersion}));
nodeFiles[module].err = "version_mismatch";
log.warn("["+module+"] "+log._("server.node-version-mismatch",{version:modules[module].redVersion}));
modules[module].err = "version_mismatch";
continue;
}
if (module == "node-red" || !registry.getModuleInfo(module)) {
var first = true;
for (var node in nodeFiles[module].nodes) {
/* istanbul ignore else */
if (nodeFiles[module].nodes.hasOwnProperty(node)) {
if (module != "node-red" && first) {
// Check the module directory exists
first = false;
var fn = nodeFiles[module].nodes[node].file;
var parts = fn.split("/");
var i = parts.length-1;
for (;i>=0;i--) {
if (parts[i] == "node_modules") {
break;
}
}
var moduleFn = parts.slice(0,i+2).join("/");
try {
var stat = fs.statSync(moduleFn);
} catch(err) {
// Module not found, don't attempt to load its nodes
break;
}
}
try {
promises.push(loadNodeConfig(nodeFiles[module].nodes[node]).then((function() {
var m = module;
var n = node;
return function(nodeSet) {
nodeFiles[m].nodes[n] = nodeSet;
nodes.push(nodeSet);
}
})()).catch(err => {}));
} catch(err) {
//
}
}
if (modules[module].nodes) {
nodePromises = nodePromises.concat(loadModuleTypeFiles(modules[module], "nodes"));
}
if (modules[module].plugins) {
pluginPromises = pluginPromises.concat(loadModuleTypeFiles(modules[module], "plugins"));
}
}
}
}
return Promise.all(promises).then(function(results) {
for (var module in nodeFiles) {
if (nodeFiles.hasOwnProperty(module)) {
if (!nodeFiles[module].err) {
registry.addModule(nodeFiles[module]);
var pluginList;
var nodeList;
return Promise.all(pluginPromises).then(function(results) {
pluginList = results.filter(r => !!r);
// Initial plugin load has happened. Ensure modules that provide
// plugins are in the registry now.
for (var module in modules) {
if (modules.hasOwnProperty(module)) {
if (modules[module].plugins && Object.keys(modules[module].plugins).length > 0) {
// Add the modules for plugins
if (!modules[module].err) {
registry.addModule(modules[module]);
}
}
}
}
return loadNodeSetList(nodes);
return loadNodeSetList(pluginList);
}).then(function() {
return Promise.all(nodePromises);
}).then(function(results) {
nodeList = results.filter(r => !!r);
// Initial node load has happened. Ensure remaining modules are in the registry
for (var module in modules) {
if (modules.hasOwnProperty(module)) {
if (!modules[module].plugins || Object.keys(modules[module].plugins).length === 0) {
if (!modules[module].err) {
registry.addModule(modules[module]);
}
}
}
}
return loadNodeSetList(nodeList);
});
}
async function loadPluginTemplate(plugin) {
return fs.readFile(plugin.template,'utf8').then(content => {
plugin.config = content;
return plugin;
}).catch(err => {
if (err.code === 'ENOENT') {
plugin.err = "Error: "+plugin.template+" does not exist";
} else {
plugin.err = err.toString();
}
return plugin;
});
}
@@ -175,11 +237,12 @@ async function loadNodeLocales(node) {
node.namespace = node.module;
return node
}
return fs.stat(path.join(path.dirname(node.file),"locales")).then(stat => {
const baseFile = node.file||node.template;
return fs.stat(path.join(path.dirname(baseFile),"locales")).then(stat => {
node.namespace = node.id;
return i18n.registerMessageCatalog(node.id,
path.join(path.dirname(node.file),"locales"),
path.basename(node.file,".js")+".json")
path.join(path.dirname(baseFile),"locales"),
path.basename(baseFile).replace(/\.[^.]+$/,".json"))
.then(() => node);
}).catch(err => {
node.namespace = node.module;
@@ -204,6 +267,7 @@ async function loadNodeConfig(fileInfo) {
}
var node = {
type: "node",
id: id,
module: module,
name: name,
@@ -227,6 +291,58 @@ async function loadNodeConfig(fileInfo) {
return node;
}
async function loadPluginConfig(fileInfo) {
var file = fileInfo.file;
var module = fileInfo.module;
var name = fileInfo.name;
var version = fileInfo.version;
var id = module + "/" + name;
var isEnabled = true;
// TODO: registry.getPluginInfo
// var info = registry.getPluginInfo(id);
// if (info) {
// if (info.hasOwnProperty("loaded")) {
// throw new Error(file+" already loaded");
// }
// isEnabled = info.enabled;
// }
if (!fs.existsSync(jsFile)) {
}
var plugin = {
type: "plugin",
id: id,
module: module,
name: name,
enabled: isEnabled,
loaded:false,
version: version,
local: fileInfo.local,
plugins: [],
config: "",
help: {}
};
var jsFile = file.replace(/\.[^.]+$/,".js");
var htmlFile = file.replace(/\.[^.]+$/,".html");
if (fs.existsSync(jsFile)) {
plugin.file = jsFile;
}
if (fs.existsSync(htmlFile)) {
plugin.template = htmlFile;
}
await loadNodeLocales(plugin)
if (plugin.template && !settings.disableEditor) {
return loadPluginTemplate(plugin);
}
return plugin
}
/**
* Loads the specified node into the runtime
* @param node a node info object - see loadNodeConfig
@@ -236,8 +352,6 @@ async function loadNodeConfig(fileInfo) {
*
*/
function loadNodeSet(node) {
var nodeDir = path.dirname(node.file);
var nodeFn = path.basename(node.file);
if (!node.enabled) {
return Promise.resolve(node);
} else {
@@ -284,11 +398,59 @@ function loadNodeSet(node) {
}
}
async function loadPlugin(plugin) {
if (!plugin.file) {
// No runtime component - nothing to load
return plugin;
}
try {
var r = require(plugin.file);
if (typeof r === "function") {
var red = registryUtil.createNodeApi(plugin);
var promise = r(red);
if (promise != null && typeof promise.then === "function") {
return promise.then(function() {
plugin.enabled = true;
plugin.loaded = true;
return plugin;
}).catch(function(err) {
plugin.err = err;
return plugin;
});
}
}
plugin.enabled = true;
plugin.loaded = true;
return plugin;
} catch(err) {
console.log(err);
plugin.err = err;
var stack = err.stack;
var message;
if (stack) {
var i = stack.indexOf(plugin.file);
if (i > -1) {
var excerpt = stack.substring(i+node.file.length+1,i+plugin.file.length+20);
var m = /^(\d+):(\d+)/.exec(excerpt);
if (m) {
plugin.err = err+" (line:"+m[1]+")";
}
}
}
return plugin;
}
}
function loadNodeSetList(nodes) {
var promises = [];
nodes.forEach(function(node) {
if (!node.err) {
promises.push(loadNodeSet(node).catch(err => {}));
if (node.type === "plugin") {
promises.push(loadPlugin(node).catch(err => {}));
} else {
promises.push(loadNodeSet(node).catch(err => {}));
}
} else {
promises.push(node);
}
@@ -338,7 +500,7 @@ function addModule(module) {
}
}
}
return loadNodeFiles(moduleFiles).then(() => module)
return loadModuleFiles(moduleFiles).then(() => module)
} catch(err) {
return Promise.reject(err);
}

View File

@@ -220,33 +220,41 @@ function getModuleNodeFiles(module) {
var moduleDir = module.dir;
var pkg = module.package;
var nodes = pkg['node-red'].nodes||{};
var results = [];
var iconDirs = [];
var iconList = [];
for (var n in nodes) {
/* istanbul ignore else */
if (nodes.hasOwnProperty(n)) {
var file = path.join(moduleDir,nodes[n]);
results.push({
file: file,
module: pkg.name,
name: n,
version: pkg.version
});
var iconDir = path.join(moduleDir,path.dirname(nodes[n]),"icons");
if (iconDirs.indexOf(iconDir) == -1) {
try {
fs.statSync(iconDir);
var icons = scanIconDir(iconDir);
iconList.push({path:iconDir,icons:icons});
iconDirs.push(iconDir);
} catch(err) {
function scanTypes(types) {
const files = [];
for (var n in types) {
/* istanbul ignore else */
if (types.hasOwnProperty(n)) {
var file = path.join(moduleDir,types[n]);
files.push({
file: file,
module: pkg.name,
name: n,
version: pkg.version
});
var iconDir = path.join(moduleDir,path.dirname(types[n]),"icons");
if (iconDirs.indexOf(iconDir) == -1) {
try {
fs.statSync(iconDir);
var icons = scanIconDir(iconDir);
iconList.push({path:iconDir,icons:icons});
iconDirs.push(iconDir);
} catch(err) {
}
}
}
}
return files;
}
var result = {files:results,icons:iconList};
var result = {
nodeFiles:scanTypes(pkg['node-red'].nodes||{}),
pluginFiles:scanTypes(pkg['node-red'].plugins||{}),
icons:iconList
};
var examplesDir = path.join(moduleDir,"examples");
try {
@@ -396,6 +404,7 @@ function convertModuleFileListToObject(moduleFiles) {
local: moduleFile.local||false,
user: moduleFile.user||false,
nodes: {},
plugins: {},
icons: nodeModuleFiles.icons,
examples: nodeModuleFiles.examples
};
@@ -408,11 +417,14 @@ function convertModuleFileListToObject(moduleFiles) {
if (moduleFile.usedBy) {
nodeList[moduleFile.package.name].usedBy = moduleFile.usedBy;
}
nodeModuleFiles.files.forEach(function(node) {
node.local = moduleFile.local||false;
nodeModuleFiles.nodeFiles.forEach(function(node) {
nodeList[moduleFile.package.name].nodes[node.name] = node;
nodeList[moduleFile.package.name].nodes[node.name].local = moduleFile.local || false;
});
nodeModuleFiles.pluginFiles.forEach(function(plugin) {
nodeList[moduleFile.package.name].plugins[plugin.name] = plugin;
nodeList[moduleFile.package.name].plugins[plugin.name].local = moduleFile.local || false;
});
});
return nodeList;
}

View File

@@ -0,0 +1,103 @@
const registry = require("./registry");
const {events} = require("@node-red/util")
var pluginConfigCache = {};
var pluginToId = {};
var plugins = {};
var pluginsByType = {};
var settings;
function init(_settings) {
settings = _settings;
plugins = {};
pluginConfigCache = {};
pluginToId = {};
pluginsByType = {};
}
function registerPlugin(nodeSetId,id,definition) {
var moduleId = registry.getModuleFromSetId(nodeSetId);
var pluginId = registry.getNodeFromSetId(nodeSetId);
definition.id = id;
definition.module = moduleId;
pluginToId[id] = nodeSetId;
plugins[id] = definition;
var module = registry.getModule(moduleId);
definition.path = module.path;
module.plugins[pluginId].plugins.push(definition);
if (definition.type) {
pluginsByType[definition.type] = pluginsByType[definition.type] || [];
pluginsByType[definition.type].push(definition);
}
if (definition.onadd && typeof definition.onadd === 'function') {
definition.onadd();
}
events.emit("registry:plugin-added",id);
}
function getPlugin(id) {
return plugins[id]
}
function getPluginsByType(type) {
return pluginsByType[type] || [];
}
function getPluginConfigs(lang) {
if (!pluginConfigCache[lang]) {
var result = "";
var script = "";
var moduleConfigs = registry.getModuleList();
for (var module in moduleConfigs) {
/* istanbul ignore else */
if (moduleConfigs.hasOwnProperty(module)) {
var plugins = moduleConfigs[module].plugins;
for (var plugin in plugins) {
if (plugins.hasOwnProperty(plugin)) {
var config = plugins[plugin];
if (config.enabled && !config.err && config.config) {
result += "\n<!-- --- [red-plugin:"+config.id+"] --- -->\n";
result += config.config;
}
}
}
}
}
pluginConfigCache[lang] = result;
}
return pluginConfigCache[lang];
}
function getPluginList() {
var list = [];
var moduleConfigs = registry.getModuleList();
for (var module in moduleConfigs) {
/* istanbul ignore else */
if (moduleConfigs.hasOwnProperty(module)) {
var plugins = moduleConfigs[module].plugins;
for (var plugin in plugins) {
/* istanbul ignore else */
if (plugins.hasOwnProperty(plugin)) {
var pluginInfo = registry.filterNodeInfo(plugins[plugin]);
pluginInfo.version = moduleConfigs[module].version;
// if (moduleConfigs[module].pending_version) {
// nodeInfo.pending_version = moduleConfigs[module].pending_version;
// }
list.push(pluginInfo);
}
}
}
}
return list;
}
module.exports = {
init,
registerPlugin,
getPlugin,
getPluginsByType,
getPluginConfigs,
getPluginList
}

View File

@@ -64,17 +64,24 @@ function filterNodeInfo(n) {
if (n.hasOwnProperty("err")) {
r.err = n.err;
}
if (n.hasOwnProperty("plugins")) {
r.plugins = n.plugins;
}
if (n.type === "plugin") {
r.editor = !!n.template;
r.runtime = !!n.file;
}
return r;
}
function getModule(id) {
function getModuleFromSetId(id) {
var parts = id.split("/");
return parts.slice(0,parts.length-1).join("/");
}
function getNode(id) {
function getNodeFromSetId(id) {
var parts = id.split("/");
return parts[parts.length-1];
}
@@ -217,11 +224,11 @@ function addModule(module) {
function removeNode(id) {
var config = moduleConfigs[getModule(id)].nodes[getNode(id)];
var config = moduleConfigs[getModuleFromSetId(id)].nodes[getNodeFromSetId(id)];
if (!config) {
throw new Error("Unrecognised id: "+id);
}
delete moduleConfigs[getModule(id)].nodes[getNode(id)];
delete moduleConfigs[getModuleFromSetId(id)].nodes[getNodeFromSetId(id)];
var i = nodeList.indexOf(id);
if (i > -1) {
nodeList.splice(i,1);
@@ -292,9 +299,9 @@ function getNodeInfo(typeOrId) {
}
/* istanbul ignore else */
if (id) {
var module = moduleConfigs[getModule(id)];
var module = moduleConfigs[getModuleFromSetId(id)];
if (module) {
var config = module.nodes[getNode(id)];
var config = module.nodes[getNodeFromSetId(id)];
if (config) {
var info = filterNodeInfo(config);
if (config.hasOwnProperty("loaded")) {
@@ -321,9 +328,9 @@ function getFullNodeInfo(typeOrId) {
}
/* istanbul ignore else */
if (id) {
var module = moduleConfigs[getModule(id)];
var module = moduleConfigs[getModuleFromSetId(id)];
if (module) {
return module.nodes[getNode(id)];
return module.nodes[getNodeFromSetId(id)];
}
}
return null;
@@ -357,16 +364,10 @@ function getNodeList(filter) {
}
function getModuleList() {
//var list = [];
//for (var module in moduleNodes) {
// /* istanbul ignore else */
// if (moduleNodes.hasOwnProperty(module)) {
// list.push(registry.getModuleInfo(module));
// }
//}
//return list;
return moduleConfigs;
}
function getModule(id) {
return moduleConfigs[id];
}
function getModuleInfo(module) {
@@ -465,13 +466,11 @@ function getAllNodeConfigs(lang) {
var script = "";
for (var i=0;i<nodeList.length;i++) {
var id = nodeList[i];
var module = moduleConfigs[getModule(id)]
var module = moduleConfigs[getModuleFromSetId(id)]
if (!module.user && (module.usedBy && module.usedBy.length > 0)) {
continue;
}
var config = module.nodes[getNode(id)];
var config = module.nodes[getNodeFromSetId(id)];
if (config.enabled && !config.err) {
result += "\n<!-- --- [red-module:"+id+"] --- -->\n";
result += config.config;
@@ -490,11 +489,11 @@ function getAllNodeConfigs(lang) {
}
function getNodeConfig(id,lang) {
var config = moduleConfigs[getModule(id)];
var config = moduleConfigs[getModuleFromSetId(id)];
if (!config) {
return null;
}
config = config.nodes[getNode(id)];
config = config.nodes[getNodeFromSetId(id)];
if (config) {
var result = "<!-- --- [red-module:"+id+"] --- -->\n"+config.config;
result += loader.getNodeHelp(config,lang||"en-US")
@@ -515,7 +514,7 @@ function getNodeConstructor(type) {
if (typeof id === "undefined") {
config = undefined;
} else {
config = moduleConfigs[getModule(id)].nodes[getNode(id)];
config = moduleConfigs[getModuleFromSetId(id)].nodes[getNodeFromSetId(id)];
}
if (!config || (config.enabled && !config.err)) {
@@ -553,7 +552,7 @@ function enableNodeSet(typeOrId) {
}
var config;
try {
config = moduleConfigs[getModule(id)].nodes[getNode(id)];
config = moduleConfigs[getModuleFromSetId(id)].nodes[getNodeFromSetId(id)];
delete config.err;
config.enabled = true;
nodeConfigCache = {};
@@ -576,7 +575,7 @@ function disableNodeSet(typeOrId) {
}
var config;
try {
config = moduleConfigs[getModule(id)].nodes[getNode(id)];
config = moduleConfigs[getModuleFromSetId(id)].nodes[getNodeFromSetId(id)];
// TODO: persist setting
config.enabled = false;
nodeConfigCache = {};
@@ -715,6 +714,7 @@ var registry = module.exports = {
getFullNodeInfo: getFullNodeInfo,
getNodeList: getNodeList,
getModuleList: getModuleList,
getModule: getModule,
getModuleInfo: getModuleInfo,
getNodeIconPath: getNodeIconPath,
@@ -730,5 +730,8 @@ var registry = module.exports = {
saveNodeList: saveNodeList,
cleanModuleList: cleanModuleList
cleanModuleList: cleanModuleList,
getModuleFromSetId: getModuleFromSetId,
getNodeFromSetId: getNodeFromSetId,
filterNodeInfo: filterNodeInfo
};

View File

@@ -70,6 +70,17 @@ function createNodeApi(node) {
})
}
},
plugins: {
registerPlugin: function(id,definition) {
return runtime.plugins.registerPlugin(node.id,id,definition);
},
get: function(id) {
return runtime.plugins.getPlugin(id);
},
getByType: function(type) {
return runtime.plugins.getPluginsByType(type);
}
},
library: {
register: function(type) {
return runtime.library.register(node.id,type);