Retry auto-install of modules that fail

- introduces autoInstallModulesRetry - default 30000
 - backs off interval if repeated failures
 - fixes notification to the editor of an auto-reinstall

editor/js/i18n.js

@@ -37,7 +37,25 @@ RED.i18n = (function() {
 
         },
         loadCatalog: function(namespace,done) {
-            i18n.loadNamespace(namespace,done);
+            var languageList = i18n.functions.toLanguages(i18n.detectLanguage());
+            var toLoad = languageList.length;
+            languageList.forEach(function(lang) {
+                $.ajax({
+                    headers: {
+                        "Accept":"application/json"
+                    },
+                    cache: false,
+                    url: 'locales/'+namespace+'?lng='+lang,
+                    success: function(data) {
+                        i18n.addResourceBundle(lang,namespace,data);
+                        toLoad--;
+                        if (toLoad === 0) {
+                            done();
+                        }
+                    }
+                });
+            })
+
         },
 
         loadNodeCatalogs: function(done) {

editor/js/main.js

@@ -72,6 +72,10 @@
                         // handled in ui/deploy.js
                         return;
                     }
+                    if (notificationId === "node") {
+                        // handled below
+                        return;
+                    }
                     if (msg.text) {
                         var text = RED._(msg.text,{default:msg.text});
                         if (!persistentNotifications.hasOwnProperty(notificationId)) {
@@ -98,11 +102,11 @@
                         RED.view.redraw();
                     }
                 });
-                RED.comms.subscribe("node/#",function(topic,msg) {
+                RED.comms.subscribe("notification/node/#",function(topic,msg) {
                     var i,m;
                     var typeList;
                     var info;
-                    if (topic == "node/added") {
+                    if (topic == "notification/node/added") {
                         var addedTypes = [];
                         msg.forEach(function(m) {
                             var id = m.id;
@@ -118,7 +122,7 @@
                             typeList = "<ul><li>"+addedTypes.join("</li><li>")+"</li></ul>";
                             RED.notify(RED._("palette.event.nodeAdded", {count:addedTypes.length})+typeList,"success");
                         }
-                    } else if (topic == "node/removed") {
+                    } else if (topic == "notification/node/removed") {
                         for (i=0;i<msg.length;i++) {
                             m = msg[i];
                             info = RED.nodes.removeNodeSet(m.id);
@@ -127,7 +131,7 @@
                                 RED.notify(RED._("palette.event.nodeRemoved", {count:m.types.length})+typeList,"success");
                             }
                         }
-                    } else if (topic == "node/enabled") {
+                    } else if (topic == "notification/node/enabled") {
                         if (msg.types) {
                             info = RED.nodes.getNodeSet(msg.id);
                             if (info.added) {
@@ -142,7 +146,7 @@
                                 });
                             }
                         }
-                    } else if (topic == "node/disabled") {
+                    } else if (topic == "notification/node/disabled") {
                         if (msg.types) {
                             RED.nodes.disableNodeSet(msg.id);
                             typeList = "<ul><li>"+msg.types.join("</li><li>")+"</li></ul>";

red/api/comms.js

@@ -33,7 +33,7 @@ function handleStatus(event) {
     publish("status/"+event.id,event.status,true);
 }
 function handleRuntimeEvent(event) {
-    publish("notification/"+event.id,event,true);
+    publish("notification/"+event.id,event.payload||{},event.retain);
 }
 function init(_server,runtime) {
     server = _server;

red/api/nodes.js

@@ -15,12 +15,12 @@
  **/
 
 var when = require("when");
-var comms = require("./comms");
 var locales = require("./locales");
 var redNodes;
 var log;
 var i18n;
 var settings;
+var events;
 
 module.exports = {
     init: function(runtime) {
@@ -28,6 +28,7 @@ module.exports = {
         log = runtime.log;
         i18n = runtime.i18n;
         settings = runtime.settings;
+        events = runtime.events;
     },
     getAll: function(req,res) {
         if (req.get("accept") == "application/json") {
@@ -72,9 +73,9 @@ module.exports = {
         }
         promise.then(function(info) {
             if (isUpgrade) {
-                comms.publish("node/upgraded",{module:node.module,version:node.version},false);
+                events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:node.module,version:node.version}});
             } else {
-                comms.publish("node/added",info.nodes,false);
+                events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes});
             }
             if (node.module) {
                 log.audit({event: "nodes.install",module:node.module,version:node.version},req);
@@ -113,7 +114,7 @@ module.exports = {
             }
 
             promise.then(function(list) {
-                comms.publish("node/removed",list,false);
+                events.emit("runtime-event",{id:"node/removed",retain:false,payload:list});
                 log.audit({event: "nodes.remove",module:mod},req);
                 res.status(204).end();
             }).otherwise(function(err) {
@@ -245,7 +246,7 @@ function putNode(node, enabled) {
 
         return promise.then(function(info) {
             if (info.enabled === enabled && !info.err) {
-                comms.publish("node/"+(enabled?"enabled":"disabled"),info,false);
+                events.emit("runtime-event",{id:"node/"+(enabled?"enabled":"disabled"),retain:false,payload:info});
                 log.info(" "+log._("api.nodes."+(enabled?"enabled":"disabled")));
                 for (var i=0;i<info.types.length;i++) {
                     log.info(" - "+info.types[i]);

red/runtime/index.js

@@ -106,7 +106,7 @@ function start() {
                 log.error("*****************************************************************");
                 log.error("* "+log._("runtime.unsupported_version",{component:"Node.js",version:process.version,requires: ">=4"})+" *");
                 log.error("*****************************************************************");
-                events.emit("runtime-event",{id:"runtime-unsupported-version",type:"error",text:"notification.errors.unsupportedVersion"});
+                events.emit("runtime-event",{id:"runtime-unsupported-version",payload:{type:"error",text:"notification.errors.unsupportedVersion"},retain:true});
             }
             log.info(os.type()+" "+os.release()+" "+os.arch()+" "+os.endianness());
             return redNodes.load().then(function() {
@@ -133,21 +133,23 @@ function start() {
                         }
                         missingModules[missing.module].types = missingModules[missing.module].types.concat(missing.types);
                     }
+                    var moduleList = [];
                     var promises = [];
+                    var installingModules = [];
                     for (i in missingModules) {
                         if (missingModules.hasOwnProperty(i)) {
                             log.warn(" - "+i+" ("+missingModules[i].version+"): "+missingModules[i].types.join(", "));
                             if (settings.autoInstallModules && i != "node-red") {
-                                redNodes.installModule(i,missingModules[i].version).otherwise(function(err) {
+                                installingModules.push({id:i,version:missingModules[i].version});
-                                    // Error already reported. Need the otherwise handler
-                                    // to stop the error propagating any further
-                                });
                             }
                         }
                     }
                     if (!settings.autoInstallModules) {
                         log.info(log._("server.removing-modules"));
                         redNodes.cleanModuleList();
+                    } else if (installingModules.length > 0) {
+                        reinstallAttempts = 0;
+                        reinstallModules(installingModules);
                     }
                 }
                 if (settings.settingsFile) {
@@ -161,6 +163,36 @@ function start() {
         });
 }
 
+var reinstallAttempts;
+var reinstallTimeout;
+function reinstallModules(moduleList) {
+    var promises = [];
+    var failedModules = [];
+    for (var i=0;i<moduleList.length;i++) {
+        if (settings.autoInstallModules && i != "node-red") {
+            promises.push(redNodes.installModule(moduleList[i].id,moduleList[i].version));
+        }
+    }
+    when.settle(promises).then(function(results) {
+        var reinstallList = [];
+        for (var i=0;i<results.length;i++) {
+            if (results[i].state === 'rejected') {
+                reinstallList.push(moduleList[i]);
+            } else {
+                events.emit("runtime-event",{id:"node/added",retain:false,payload:results[i].value.nodes});
+            }
+        }
+        if (reinstallList.length > 0) {
+            reinstallAttempts++;
+            // First 5 at 1x timeout, next 5 at 2x, next 5 at 4x, then 8x
+            var timeout = (settings.autoInstallModulesRetry||30000) * Math.pow(2,Math.min(Math.floor(reinstallAttempts/5),3));
+            reinstallTimeout = setTimeout(function() {
+                reinstallModules(reinstallList);
+            },timeout);
+        }
+    });
+}
+
 function reportMetrics() {
     var memUsage = process.memoryUsage();
 
@@ -186,6 +218,9 @@ function stop() {
         clearInterval(runtimeMetricInterval);
         runtimeMetricInterval = null;
     }
+    if (reinstallTimeout) {
+        clearTimeout(reinstallTimeout);
+    }
     started = false;
     return redNodes.stopFlows();
 }

red/runtime/nodes/flows/index.js

@@ -58,7 +58,7 @@ function init(runtime) {
                     log.info(log._("nodes.flows.registered-missing", {type:type}));
                     activeFlowConfig.missingTypes.splice(i,1);
                     if (activeFlowConfig.missingTypes.length === 0 && started) {
-                        events.emit("runtime-event",{id:"runtime-state"});
+                        events.emit("runtime-event",{id:"runtime-state",retain: true});
                         start();
                     }
                 }
@@ -135,13 +135,13 @@ function setFlows(_config,type,muteLog) {
                 return stop(type,diff,muteLog).then(function() {
                     context.clean(activeFlowConfig);
                     start(type,diff,muteLog).then(function() {
-                        events.emit("runtime-event",{id:"runtime-deploy",revision:flowRevision});
+                        events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true});
                     });
                     return flowRevision;
                 }).otherwise(function(err) {
                 })
             } else {
-                events.emit("runtime-event",{id:"runtime-deploy",revision:flowRevision});
+                events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true});
             }
         });
 }
@@ -251,7 +251,7 @@ function start(type,diff,muteLog) {
             log.info(log._("nodes.flows.missing-type-install-2"));
             log.info("  "+settings.userDir);
         }
-        events.emit("runtime-event",{id:"runtime-state",type:"warning",text:"notification.warnings.missing-types"});
+        events.emit("runtime-event",{id:"runtime-state",payload:{type:"warning",text:"notification.warnings.missing-types"},retain:true});
         return when.resolve();
     }
     if (!muteLog) {
@@ -310,7 +310,7 @@ function start(type,diff,muteLog) {
         }
     }
     events.emit("nodes-started");
-    events.emit("runtime-event",{id:"runtime-state"});
+    events.emit("runtime-event",{id:"runtime-state",retain:true});
 
     if (!muteLog) {
         if (type !== "full") {

red/runtime/nodes/index.js

@@ -30,8 +30,6 @@ var library = require("./library");
 
 var events = require("../events");
 
-var child_process = require('child_process');
-
 var settings;
 
 /**

red/runtime/nodes/registry/installer.js

@@ -131,7 +131,7 @@ function installModule(module,version) {
                         resolve(require("./index").addModule(module).then(reportAddedModules));
                     } else {
                         log.info(log._("server.install.upgraded",{name:module, version:version}));
-                        events.emit("runtime-event",{id:"restart-required",type:"warning",text:"notification.warnings.restartRequired"});
+                        events.emit("runtime-event",{id:"restart-required",payload:{type:"warning",text:"notification.warnings.restartRequired"},retain:true});
                         resolve(require("./registry").setModulePendingUpdated(module,version));
                     }
                 }

red/runtime/nodes/registry/registry.js

@@ -128,7 +128,7 @@ function saveNodeList() {
         }
     }
     if (hadPending && !hasPending) {
-        events.emit("runtime-event",{id:"restart-required"});
+        events.emit("runtime-event",{id:"restart-required",retain: true});
     }
     if (settings.available()) {
         return settings.set("nodes",moduleList);

test/red/api/nodes_spec.js

@@ -22,7 +22,6 @@ var sinon = require('sinon');
 var when = require('when');
 
 var nodes = require("../../../red/api/nodes");
-var comms = require("../../../red/api/comms");
 var locales = require("../../../red/api/locales");
 
 describe("nodes api", function() {
@@ -35,6 +34,9 @@ describe("nodes api", function() {
             info: function(){},
             warn: function(){}
         }
+        runtime.events = {
+            emit: function(){}
+        }
         nodes.init(runtime);
 
     }
@@ -49,15 +51,11 @@ describe("nodes api", function() {
         app.put(/\/nodes\/((@[^\/]+\/)?[^\/]+)$/,nodes.putModule);
         app.put(/\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,nodes.putSet);
         app.delete("/nodes/:id",nodes.delete);
-        sinon.stub(comms,"publish");
         sinon.stub(locales,"determineLangFromHeaders", function() {
             return "en-US";
         });
     });
 
-    after(function() {
-        comms.publish.restore();
-    });
 
     describe('get nodes', function() {
         it('returns node list', function(done) {

test/red/runtime/index_spec.js

@@ -141,7 +141,7 @@ describe("runtime", function() {
                     {  module:"node-red",enabled:true,loaded:false,types:["typeC","typeD"]} // missing
                 ].filter(cb);
             });
-            var serverInstallModule = sinon.stub(redNodes,"installModule",function(name) { return when.resolve();});
+            var serverInstallModule = sinon.stub(redNodes,"installModule",function(name) { return when.resolve({nodes:[]});});
             runtime.init({testSettings: true, autoInstallModules:true, httpAdminRoot:"/", load:function() { return when.resolve();}});
             sinon.stub(console,"log");
             runtime.start().then(function() {