Retry auto-install of modules that fail

- introduces autoInstallModulesRetry - default 30000
 - backs off interval if repeated failures
 - fixes notification to the editor of an auto-reinstall

red/runtime/index.js

@@ -106,7 +106,7 @@ function start() {
                 log.error("*****************************************************************");
                 log.error("* "+log._("runtime.unsupported_version",{component:"Node.js",version:process.version,requires: ">=4"})+" *");
                 log.error("*****************************************************************");
-                events.emit("runtime-event",{id:"runtime-unsupported-version",type:"error",text:"notification.errors.unsupportedVersion"});
+                events.emit("runtime-event",{id:"runtime-unsupported-version",payload:{type:"error",text:"notification.errors.unsupportedVersion"},retain:true});
             }
             log.info(os.type()+" "+os.release()+" "+os.arch()+" "+os.endianness());
             return redNodes.load().then(function() {
@@ -133,21 +133,23 @@ function start() {
                         }
                         missingModules[missing.module].types = missingModules[missing.module].types.concat(missing.types);
                     }
+                    var moduleList = [];
                     var promises = [];
+                    var installingModules = [];
                     for (i in missingModules) {
                         if (missingModules.hasOwnProperty(i)) {
                             log.warn(" - "+i+" ("+missingModules[i].version+"): "+missingModules[i].types.join(", "));
                             if (settings.autoInstallModules && i != "node-red") {
-                                redNodes.installModule(i,missingModules[i].version).otherwise(function(err) {
-                                    // Error already reported. Need the otherwise handler
-                                    // to stop the error propagating any further
-                                });
+                                installingModules.push({id:i,version:missingModules[i].version});
                             }
                         }
                     }
                     if (!settings.autoInstallModules) {
                         log.info(log._("server.removing-modules"));
                         redNodes.cleanModuleList();
+                    } else if (installingModules.length > 0) {
+                        reinstallAttempts = 0;
+                        reinstallModules(installingModules);
                     }
                 }
                 if (settings.settingsFile) {
@@ -161,6 +163,36 @@ function start() {
         });
 }
 
+var reinstallAttempts;
+var reinstallTimeout;
+function reinstallModules(moduleList) {
+    var promises = [];
+    var failedModules = [];
+    for (var i=0;i<moduleList.length;i++) {
+        if (settings.autoInstallModules && i != "node-red") {
+            promises.push(redNodes.installModule(moduleList[i].id,moduleList[i].version));
+        }
+    }
+    when.settle(promises).then(function(results) {
+        var reinstallList = [];
+        for (var i=0;i<results.length;i++) {
+            if (results[i].state === 'rejected') {
+                reinstallList.push(moduleList[i]);
+            } else {
+                events.emit("runtime-event",{id:"node/added",retain:false,payload:results[i].value.nodes});
+            }
+        }
+        if (reinstallList.length > 0) {
+            reinstallAttempts++;
+            // First 5 at 1x timeout, next 5 at 2x, next 5 at 4x, then 8x
+            var timeout = (settings.autoInstallModulesRetry||30000) * Math.pow(2,Math.min(Math.floor(reinstallAttempts/5),3));
+            reinstallTimeout = setTimeout(function() {
+                reinstallModules(reinstallList);
+            },timeout);
+        }
+    });
+}
+
 function reportMetrics() {
     var memUsage = process.memoryUsage();
 
@@ -186,6 +218,9 @@ function stop() {
         clearInterval(runtimeMetricInterval);
         runtimeMetricInterval = null;
     }
+    if (reinstallTimeout) {
+        clearTimeout(reinstallTimeout);
+    }
     started = false;
     return redNodes.stopFlows();
 }

red/runtime/nodes/flows/index.js

@@ -58,7 +58,7 @@ function init(runtime) {
                     log.info(log._("nodes.flows.registered-missing", {type:type}));
                     activeFlowConfig.missingTypes.splice(i,1);
                     if (activeFlowConfig.missingTypes.length === 0 && started) {
-                        events.emit("runtime-event",{id:"runtime-state"});
+                        events.emit("runtime-event",{id:"runtime-state",retain: true});
                         start();
                     }
                 }
@@ -135,13 +135,13 @@ function setFlows(_config,type,muteLog) {
                 return stop(type,diff,muteLog).then(function() {
                     context.clean(activeFlowConfig);
                     start(type,diff,muteLog).then(function() {
-                        events.emit("runtime-event",{id:"runtime-deploy",revision:flowRevision});
+                        events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true});
                     });
                     return flowRevision;
                 }).otherwise(function(err) {
                 })
             } else {
-                events.emit("runtime-event",{id:"runtime-deploy",revision:flowRevision});
+                events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true});
             }
         });
 }
@@ -251,7 +251,7 @@ function start(type,diff,muteLog) {
             log.info(log._("nodes.flows.missing-type-install-2"));
             log.info("  "+settings.userDir);
         }
-        events.emit("runtime-event",{id:"runtime-state",type:"warning",text:"notification.warnings.missing-types"});
+        events.emit("runtime-event",{id:"runtime-state",payload:{type:"warning",text:"notification.warnings.missing-types"},retain:true});
         return when.resolve();
     }
     if (!muteLog) {
@@ -310,7 +310,7 @@ function start(type,diff,muteLog) {
         }
     }
     events.emit("nodes-started");
-    events.emit("runtime-event",{id:"runtime-state"});
+    events.emit("runtime-event",{id:"runtime-state",retain:true});
 
     if (!muteLog) {
         if (type !== "full") {

red/runtime/nodes/index.js

@@ -30,8 +30,6 @@ var library = require("./library");
 
 var events = require("../events");
 
-var child_process = require('child_process');
-
 var settings;
 
 /**

red/runtime/nodes/registry/installer.js

@@ -131,7 +131,7 @@ function installModule(module,version) {
                         resolve(require("./index").addModule(module).then(reportAddedModules));
                     } else {
                         log.info(log._("server.install.upgraded",{name:module, version:version}));
-                        events.emit("runtime-event",{id:"restart-required",type:"warning",text:"notification.warnings.restartRequired"});
+                        events.emit("runtime-event",{id:"restart-required",payload:{type:"warning",text:"notification.warnings.restartRequired"},retain:true});
                         resolve(require("./registry").setModulePendingUpdated(module,version));
                     }
                 }

red/runtime/nodes/registry/registry.js

@@ -128,7 +128,7 @@ function saveNodeList() {
         }
     }
     if (hadPending && !hasPending) {
-        events.emit("runtime-event",{id:"restart-required"});
+        events.emit("runtime-event",{id:"restart-required",retain: true});
     }
     if (settings.available()) {
         return settings.set("nodes",moduleList);