WIP: separate runtime and api components

red/runtime/comms.js

@@ -0,0 +1,207 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var ws = require("ws");
+var log = require("./log");
+
+var server;
+var settings;
+
+var wsServer;
+var pendingConnections = [];
+var activeConnections = [];
+
+var retained = {};
+
+var heartbeatTimer;
+var lastSentTime;
+
+
+function init(_server,_settings) {
+    server = _server;
+    settings = _settings;
+}
+
+
+function start() {
+    var Tokens = require("../api/auth/tokens");
+    var Users = require("../api/auth/users");
+    var Permissions = require("../api/auth/permissions");
+    if (!settings.disableEditor) {
+        Users.default().then(function(anonymousUser) {
+            var webSocketKeepAliveTime = settings.webSocketKeepAliveTime || 15000;
+            var path = settings.httpAdminRoot || "/";
+            path = (path.slice(0,1) != "/" ? "/":"") + path + (path.slice(-1) == "/" ? "":"/") + "comms";
+            wsServer = new ws.Server({server:server,path:path});
+
+            wsServer.on('connection',function(ws) {
+                log.audit({event: "comms.open"});
+                var pendingAuth = (settings.adminAuth != null);
+                if (!pendingAuth) {
+                    activeConnections.push(ws);
+                } else {
+                    pendingConnections.push(ws);
+                }
+                ws.on('close',function() {
+                    log.audit({event: "comms.close",user:ws.user});
+                    removeActiveConnection(ws);
+                    removePendingConnection(ws);
+                });
+                ws.on('message', function(data,flags) {
+                    var msg = null;
+                    try {
+                        msg = JSON.parse(data);
+                    } catch(err) {
+                        log.trace("comms received malformed message : "+err.toString());
+                        return;
+                    }
+                    if (!pendingAuth) {
+                        if (msg.subscribe) {
+                            handleRemoteSubscription(ws,msg.subscribe);
+                        }
+                    } else {
+                        var completeConnection = function(userScope,sendAck) {
+                            if (!userScope || !Permissions.hasPermission(userScope,"status.read")) {
+                                ws.close();
+                            } else {
+                                pendingAuth = false;
+                                removePendingConnection(ws);
+                                activeConnections.push(ws);
+                                if (sendAck) {
+                                    ws.send(JSON.stringify({auth:"ok"}));
+                                }
+                            }
+                        }
+                        if (msg.auth) {
+                            Tokens.get(msg.auth).then(function(client) {
+                                if (client) {
+                                    Users.get(client.user).then(function(user) {
+                                        if (user) {
+                                            ws.user = user;
+                                            log.audit({event: "comms.auth",user:ws.user});
+                                            completeConnection(client.scope,true);
+                                        } else {
+                                            log.audit({event: "comms.auth.fail"});
+                                            completeConnection(null,false);
+                                        }
+                                    });
+                                } else {
+                                    log.audit({event: "comms.auth.fail"});
+                                    completeConnection(null,false);
+                                }
+                            });
+                        } else {
+                            if (anonymousUser) {
+                                log.audit({event: "comms.auth",user:anonymousUser});
+                                completeConnection(anonymousUser.permissions,false);
+                            } else {
+                                log.audit({event: "comms.auth.fail"});
+                                completeConnection(null,false);
+                            }
+                            //TODO: duplicated code - pull non-auth message handling out
+                            if (msg.subscribe) {
+                                handleRemoteSubscription(ws,msg.subscribe);
+                            }
+                        }
+                    }
+                });
+                ws.on('error', function(err) {
+                    log.warn(log._("comms.error",{message:err.toString()}));
+                });
+            });
+
+            wsServer.on('error', function(err) {
+                log.warn(log._("comms.error-server",{message:err.toString()}));
+            });
+
+            lastSentTime = Date.now();
+
+            heartbeatTimer = setInterval(function() {
+                var now = Date.now();
+                if (now-lastSentTime > webSocketKeepAliveTime) {
+                    publish("hb",lastSentTime);
+                }
+            }, webSocketKeepAliveTime);
+        });
+    }
+}
+
+function stop() {
+    if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = null;
+    }
+    if (wsServer) {
+        wsServer.close();
+        wsServer = null;
+    }
+}
+
+function publish(topic,data,retain) {
+    if (retain) {
+        retained[topic] = data;
+    } else {
+        delete retained[topic];
+    }
+    lastSentTime = Date.now();
+    activeConnections.forEach(function(conn) {
+        publishTo(conn,topic,data);
+    });
+}
+
+function publishTo(ws,topic,data) {
+    var msg = JSON.stringify({topic:topic,data:data});
+    try {
+        ws.send(msg);
+    } catch(err) {
+        removeActiveConnection(ws);
+        removePendingConnection(ws);
+        log.warn(log._("comms.error-send",{message:err.toString()}));
+    }
+}
+
+function handleRemoteSubscription(ws,topic) {
+    var re = new RegExp("^"+topic.replace(/([\[\]\?\(\)\\\\$\^\*\.|])/g,"\\$1").replace(/\+/g,"[^/]+").replace(/\/#$/,"(\/.*)?")+"$");
+    for (var t in retained) {
+        if (re.test(t)) {
+            publishTo(ws,t,retained[t]);
+        }
+    }
+}
+
+function removeActiveConnection(ws) {
+    for (var i=0;i<activeConnections.length;i++) {
+        if (activeConnections[i] === ws) {
+            activeConnections.splice(i,1);
+            break;
+        }
+    }
+}
+function removePendingConnection(ws) {
+    for (var i=0;i<pendingConnections.length;i++) {
+        if (pendingConnections[i] === ws) {
+            pendingConnections.splice(i,1);
+            break;
+        }
+    }
+}
+
+module.exports = {
+    init:init,
+    start:start,
+    stop:stop,
+    publish:publish
+}

red/runtime/events.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright 2013 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var events = require("events"); 
+
+module.exports = new events.EventEmitter(); 

red/runtime/i18n.js

@@ -0,0 +1,168 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var i18n = require("i18next");
+var when = require("when");
+var path = require("path");
+var fs = require("fs");
+
+var defaultLang = "en-US";
+var supportedLangs = [];
+
+var resourceMap = {
+    "runtime":  {
+        basedir: path.resolve(__dirname+"/../../locales"),
+        file:"runtime.json"
+    },
+    "editor": {
+        basedir: path.resolve(__dirname+"/../../locales"),
+        file: "editor.json"
+    }
+}
+var resourceCache = {}
+
+function registerMessageCatalog(namespace,dir,file) {
+    return when.promise(function(resolve,reject) {
+        resourceMap[namespace] = { basedir:dir, file:file};
+        i18n.loadNamespace(namespace,function() {
+            resolve();
+        });
+    });
+}
+
+var initSupportedLangs = function() {
+    return when.promise(function(resolve,reject) {
+        fs.readdir(resourceMap.editor.basedir, function(err,files) {
+            if(err) {
+                reject(err);
+            } else {
+                supportedLangs = files;
+                resolve();
+            }
+        });
+    });
+}
+
+function mergeCatalog(fallback,catalog) {
+    for (var k in fallback) {
+        if (fallback.hasOwnProperty(k)) {
+            if (!catalog[k]) {
+                catalog[k] = fallback[k];
+            } else if (typeof fallback[k] === 'object') {
+                mergeCatalog(fallback[k],catalog[k]);
+            }
+        }
+    }
+}
+
+var MessageFileLoader = {
+    fetchOne: function(lng, ns, callback) {
+        if (resourceMap[ns]) {
+            var file = path.join(resourceMap[ns].basedir,lng,resourceMap[ns].file);
+            //console.log(file);
+            fs.readFile(file,"utf8",function(err,content) {
+                if (err) {
+                    callback(err);
+                } else {
+                    try {
+                        resourceCache[ns] = resourceCache[ns]||{};
+                        resourceCache[ns][lng] = JSON.parse(content.replace(/^\uFEFF/, ''));
+                        //console.log(resourceCache[ns][lng]);
+                        if (lng !== defaultLang) {
+                            mergeCatalog(resourceCache[ns][defaultLang],resourceCache[ns][lng]);
+                        }
+                        callback(null, resourceCache[ns][lng]);
+                    } catch(e) {
+                        callback(e);
+                    }
+                }
+            });
+        } else {
+            callback(new Error("Unrecognised namespace"));
+        }
+    }
+
+}
+
+function init() {
+    return when.promise(function(resolve,reject) {
+        i18n.backend(MessageFileLoader);
+        i18n.init({
+            ns: {
+                namespaces: ["runtime","editor"],
+                defaultNs: "runtime"
+            },
+            fallbackLng: ['en-US']
+        },function() {
+            initSupportedLangs().then(function() {
+                resolve();
+            });
+        });
+    });
+}
+
+function getCatalog(namespace,lang) {
+    //console.log("+",namespace,lang);
+    //console.log(resourceCache[namespace][lang]);
+    var result = null;
+    if (resourceCache.hasOwnProperty(namespace)) {
+        result = resourceCache[namespace][lang];
+        if (!result) {
+            var langParts = lang.split("-");
+            if (langParts.length == 2) {
+                result = resourceCache[namespace][langParts[0]];
+            }
+            if (!result) {
+                return resourceCache[namespace][defaultLang];
+            }
+        }
+    }
+    //console.log(result);
+    return result;
+}
+
+function determineLangFromHeaders(acceptedLanguages){
+    var lang = "en-US";
+    acceptedLanguages = acceptedLanguages || [];
+    for (var i=0;i<acceptedLanguages.length;i++){
+        if (supportedLangs.indexOf(acceptedLanguages[i]) !== -1){
+            lang = acceptedLanguages[i];
+            break;
+        // check the language without the country code
+        } else if (supportedLangs.indexOf(acceptedLanguages[i].split("-")[0]) !== -1) {
+            lang = acceptedLanguages[i].split("-")[0];
+            break;
+        }
+    }
+    return lang;
+}
+
+var obj = module.exports = {
+    init: init,
+    registerMessageCatalog: registerMessageCatalog,
+    catalog: getCatalog,
+    i: i18n,
+    determineLangFromHeaders: determineLangFromHeaders
+}
+
+obj['_'] = function() {
+    //var opts = {};
+    //if (def) {
+    //    opts.defaultValue = def;
+    //}
+    //console.log(arguments);
+    return i18n.t.apply(null,arguments);
+}

red/runtime/index.js

@@ -0,0 +1,159 @@
+/**
+ * Copyright 2013, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require('when');
+
+var redNodes = require("./nodes");
+var comms = require("./comms");
+var storage = require("./storage");
+var log = require("./log");
+var i18n = require("./i18n");
+var events = require("./events");
+var settings = require("./settings");
+var path = require('path');
+var fs = require("fs");
+
+var runtimeMetricInterval = null;
+
+function init(server,userSettings) {
+    userSettings.version = version();
+    log.init(userSettings);
+    settings.init(userSettings);
+    comms.init(server,settings);
+}
+
+function version() {
+    var p = require(path.join(process.env.NODE_RED_HOME,"package.json")).version;
+    /* istanbul ignore else */
+    if (fs.existsSync(path.join(process.env.NODE_RED_HOME,".git"))) {
+        p += "-git";
+    }
+    return p;
+}
+
+function start() {
+    return i18n.init()
+        .then(function() { return storage.init(settings)})
+        .then(function() { return settings.load(storage)})
+        .then(function() {
+
+            if (log.metric()) {
+                runtimeMetricInterval = setInterval(function() {
+                    reportMetrics();
+                }, settings.runtimeMetricInterval||15000);
+            }
+            console.log("\n\n"+log._("runtime.welcome")+"\n===================\n");
+            if (settings.version) {
+                log.info(log._("runtime.version",{component:"Node-RED",version:"v"+settings.version}));
+            }
+            log.info(log._("runtime.version",{component:"Node.js ",version:process.version}));
+            log.info(log._("server.loading"));
+            redNodes.init(settings,storage);
+            return redNodes.load().then(function() {
+
+                var i;
+                var nodeErrors = redNodes.getNodeList(function(n) { return n.err!=null;});
+                var nodeMissing = redNodes.getNodeList(function(n) { return n.module && n.enabled && !n.loaded && !n.err;});
+                if (nodeErrors.length > 0) {
+                    log.warn("------------------------------------------");
+                    if (settings.verbose) {
+                        for (i=0;i<nodeErrors.length;i+=1) {
+                            log.warn("["+nodeErrors[i].name+"] "+nodeErrors[i].err);
+                        }
+                    } else {
+                        log.warn(log._("server.errors",{count:nodeErrors.length}));
+                        log.warn(log._("server.errors-help"));
+                    }
+                    log.warn("------------------------------------------");
+                }
+                if (nodeMissing.length > 0) {
+                    log.warn(log._("server.missing-modules"));
+                    var missingModules = {};
+                    for (i=0;i<nodeMissing.length;i++) {
+                        var missing = nodeMissing[i];
+                        missingModules[missing.module] = (missingModules[missing.module]||[]).concat(missing.types);
+                    }
+                    var promises = [];
+                    for (i in missingModules) {
+                        if (missingModules.hasOwnProperty(i)) {
+                            log.warn(" - "+i+": "+missingModules[i].join(", "));
+                            if (settings.autoInstallModules && i != "node-red") {
+                                redNodes.installModule(i).otherwise(function(err) {
+                                    // Error already reported. Need the otherwise handler
+                                    // to stop the error propagating any further
+                                });
+                            }
+                        }
+                    }
+                    if (!settings.autoInstallModules) {
+                        log.info(log._("server.removing-modules"));
+                        redNodes.cleanModuleList();
+                    }
+                }
+                log.info(log._("runtime.paths.settings",{path:settings.settingsFile}));
+                redNodes.loadFlows().then(redNodes.startFlows);
+                comms.start();
+            }).otherwise(function(err) {
+                console.log(err);
+            });
+    });
+}
+
+function reportMetrics() {
+    var memUsage = process.memoryUsage();
+
+    log.log({
+        level: log.METRIC,
+        event: "runtime.memory.rss",
+        value: memUsage.rss
+    });
+    log.log({
+        level: log.METRIC,
+        event: "runtime.memory.heapTotal",
+        value: memUsage.heapTotal
+    });
+    log.log({
+        level: log.METRIC,
+        event: "runtime.memory.heapUsed",
+        value: memUsage.heapUsed
+    });
+}
+
+function stop() {
+    if (runtimeMetricInterval) {
+        clearInterval(runtimeMetricInterval);
+        runtimeMetricInterval = null;
+    }
+    redNodes.stopFlows();
+    comms.stop();
+}
+
+var runtime = module.exports = {
+    init: init,
+    start: start,
+    stop: stop,
+
+    version: version,
+
+    log: log,
+    i18n: i18n,
+    settings: settings,
+    storage: storage,
+    comms: comms,
+    events: events,
+    api: redNodes,
+    util: require("./util")
+}

red/runtime/log.js

@@ -0,0 +1,149 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var util = require("util");
+var EventEmitter = require("events").EventEmitter;
+
+var i18n = require("./i18n");
+
+var levels = {
+    off:    1,
+    fatal:  10,
+    error:  20,
+    warn:   30,
+    info:   40,
+    debug:  50,
+    trace:  60,
+    audit:  98,
+    metric: 99
+};
+
+var levelNames = {
+    10: "fatal",
+    20: "error",
+    30: "warn",
+    40: "info",
+    50: "debug",
+    60: "trace",
+    98: "audit",
+    99: "metric"
+};
+
+var logHandlers = [];
+
+var metricsEnabled = false;
+
+var LogHandler = function(settings) {
+    this.logLevel  = settings ? levels[settings.level]||levels.info : levels.info;
+    this.metricsOn = settings ? settings.metrics||false : false;
+    this.auditOn = settings ? settings.audit||false : false;
+    
+    metricsEnabled = metricsEnabled || this.metricsOn;
+    
+    this.handler   = (settings && settings.handler) ? settings.handler(settings) : consoleLogger;
+    this.on("log",function(msg) {
+        if (this.shouldReportMessage(msg.level)) {
+            this.handler(msg);
+        }
+    });
+}
+util.inherits(LogHandler, EventEmitter);
+
+LogHandler.prototype.shouldReportMessage = function(msglevel) {
+    return (msglevel == log.METRIC && this.metricsOn) ||
+           (msglevel == log.AUDIT && this.auditOn) ||
+           msglevel <= this.logLevel;
+}
+
+var consoleLogger = function(msg) {
+    if (msg.level == log.METRIC || msg.level == log.AUDIT) {
+        util.log("["+levelNames[msg.level]+"] "+JSON.stringify(msg));
+    } else {
+        util.log("["+levelNames[msg.level]+"] "+(msg.type?"["+msg.type+":"+(msg.name||msg.id)+"] ":"")+msg.msg);
+    }
+}
+
+var log = module.exports = {
+    FATAL:  10,
+    ERROR:  20,
+    WARN:   30,
+    INFO:   40,
+    DEBUG:  50,
+    TRACE:  60,
+    AUDIT:  98,
+    METRIC: 99,
+
+    init: function(settings) {
+        metricsEnabled = false;
+        logHandlers = [];
+        var loggerSettings = {};
+        if (settings.logging) {
+            var keys = Object.keys(settings.logging);
+            if (keys.length === 0) {
+                log.addHandler(new LogHandler());
+            } else {
+                for (var i=0, l=keys.length; i<l; i++) {
+                    var config = settings.logging[keys[i]];
+                    loggerSettings = config || {};
+                    if ((keys[i] === "console") || config.handler) {
+                        log.addHandler(new LogHandler(loggerSettings));
+                    }
+                }
+            }
+        } else {
+            log.addHandler(new LogHandler());
+        }
+    },
+    addHandler: function(func) {
+        logHandlers.push(func);
+    },
+    log: function(msg) {
+        msg.timestamp = Date.now();
+        logHandlers.forEach(function(handler) {
+            handler.emit("log",msg);
+        });
+    },
+    info: function(msg) {
+        log.log({level:log.INFO,msg:msg});
+    },
+    warn: function(msg) {
+        log.log({level:log.WARN,msg:msg});
+    },
+    error: function(msg) {
+        log.log({level:log.ERROR,msg:msg});
+    },
+    trace: function(msg) {
+        log.log({level:log.TRACE,msg:msg});
+    },
+    debug: function(msg) {
+        log.log({level:log.DEBUG,msg:msg});
+    },
+    metric: function() {
+        return metricsEnabled;
+    },
+    
+    audit: function(msg,req) {
+        msg.level = log.AUDIT;
+        if (req) {
+            msg.user = req.user;
+            msg.path = req.path;
+            msg.ip = (req.headers && req.headers['x-forwarded-for']) || (req.connection && req.connection.remoteAddress) || undefined;
+        }
+        log.log(msg);
+    }
+}
+
+log["_"] = i18n._;

red/runtime/nodes/Node.js

@@ -0,0 +1,256 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var util = require("util");
+var EventEmitter = require("events").EventEmitter;
+var when = require("when");
+
+var redUtil = require("../util");
+var Log = require("../log");
+
+var flows = require("./flows");
+var comms = require("../comms");
+
+function Node(n) {
+    this.id = n.id;
+    this.type = n.type;
+    this.z = n.z;
+    this._closeCallbacks = [];
+
+    if (n.name) {
+        this.name = n.name;
+    }
+    if (n._alias) {
+        this._alias = n._alias;
+    }
+    this.updateWires(n.wires);
+}
+
+util.inherits(Node, EventEmitter);
+
+Node.prototype.updateWires = function(wires) {
+    //console.log("UPDATE",this.id);
+    this.wires = wires || [];
+    delete this._wire;
+
+    var wc = 0;
+    this.wires.forEach(function(w) {
+        wc+=w.length;
+    });
+    this._wireCount = wc;
+    if (wc === 0) {
+        // With nothing wired to the node, no-op send
+        this.send = function(msg) {}
+    } else {
+        this.send = Node.prototype.send;
+        if (this.wires.length === 1 && this.wires[0].length === 1) {
+            // Single wire, so we can shortcut the send when
+            // a single message is sent
+            this._wire = this.wires[0][0];
+        }
+    }
+
+}
+
+Node.prototype._on = Node.prototype.on;
+
+Node.prototype.on = function(event, callback) {
+    var node = this;
+    if (event == "close") {
+        this._closeCallbacks.push(callback);
+    } else {
+        this._on(event, callback);
+    }
+};
+
+Node.prototype.close = function() {
+    var promises = [];
+    var node = this;
+    for (var i=0;i<this._closeCallbacks.length;i++) {
+        var callback = this._closeCallbacks[i];
+        if (callback.length == 1) {
+            promises.push(
+                when.promise(function(resolve) {
+                    callback.call(node, function() {
+                        resolve();
+                    });
+                })
+            );
+        } else {
+            callback.call(node);
+        }
+    }
+    if (promises.length > 0) {
+        return when.settle(promises);
+    } else {
+        return;
+    }
+};
+
+Node.prototype.send = function(msg) {
+    var msgSent = false;
+    var node;
+
+    if (msg === null || typeof msg === "undefined") {
+        return;
+    } else if (!util.isArray(msg)) {
+        if (this._wire) {
+            // A single message and a single wire on output 0
+            // TODO: pre-load flows.get calls - cannot do in constructor
+            //       as not all nodes are defined at that point
+            if (!msg._msgid) {
+                msg._msgid = redUtil.generateId();
+            }
+            this.metric("send",msg);
+            node = flows.get(this._wire);
+            /* istanbul ignore else */
+            if (node) {
+                node.receive(msg);
+            }
+            return;
+        } else {
+            msg = [msg];
+        }
+    }
+
+    var numOutputs = this.wires.length;
+
+    // Build a list of send events so that all cloning is done before
+    // any calls to node.receive
+    var sendEvents = [];
+
+    var sentMessageId = null;
+
+    // for each output of node eg. [msgs to output 0, msgs to output 1, ...]
+    for (var i = 0; i < numOutputs; i++) {
+        var wires = this.wires[i]; // wires leaving output i
+        /* istanbul ignore else */
+        if (i < msg.length) {
+            var msgs = msg[i]; // msgs going to output i
+            if (msgs !== null && typeof msgs !== "undefined") {
+                if (!util.isArray(msgs)) {
+                    msgs = [msgs];
+                }
+                var k = 0;
+                // for each recipent node of that output
+                for (var j = 0; j < wires.length; j++) {
+                    node = flows.get(wires[j]); // node at end of wire j
+                    if (node) {
+                        // for each msg to send eg. [[m1, m2, ...], ...]
+                        for (k = 0; k < msgs.length; k++) {
+                            var m = msgs[k];
+                            /* istanbul ignore else */
+                            if (!sentMessageId) {
+                                sentMessageId = m._msgid;
+                            }
+                            if (msgSent) {
+                                var clonedmsg = redUtil.cloneMessage(m);
+                                sendEvents.push({n:node,m:clonedmsg});
+                            } else {
+                                sendEvents.push({n:node,m:m});
+                                msgSent = true;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    /* istanbul ignore else */
+    if (!sentMessageId) {
+        sentMessageId = redUtil.generateId();
+    }
+    this.metric("send",{_msgid:sentMessageId});
+
+    for (i=0;i<sendEvents.length;i++) {
+        var ev = sendEvents[i];
+        /* istanbul ignore else */
+        if (!ev.m._msgid) {
+            ev.m._msgid = sentMessageId;
+        }
+        ev.n.receive(ev.m);
+    }
+};
+
+Node.prototype.receive = function(msg) {
+    if (!msg) {
+        msg = {};
+    }
+    if (!msg._msgid) {
+        msg._msgid = redUtil.generateId();
+    }
+    this.metric("receive",msg);
+    try {
+        this.emit("input", msg);
+    } catch(err) {
+        this.error(err,msg);
+    }
+};
+
+function log_helper(self, level, msg) {
+    var o = {
+        level: level,
+        id: self.id,
+        type: self.type,
+        msg: msg
+    };
+    if (self.name) {
+        o.name = self.name;
+    }
+    Log.log(o);
+}
+
+Node.prototype.log = function(msg) {
+    log_helper(this, Log.INFO, msg);
+};
+
+Node.prototype.warn = function(msg) {
+    log_helper(this, Log.WARN, msg);
+};
+
+Node.prototype.error = function(logMessage,msg) {
+    logMessage = logMessage || "";
+    log_helper(this, Log.ERROR, logMessage);
+    /* istanbul ignore else */
+    if (msg) {
+        flows.handleError(this,logMessage,msg);
+    }
+};
+
+/**
+ * If called with no args, returns whether metric collection is enabled
+ */
+Node.prototype.metric = function(eventname, msg, metricValue) {
+    if (typeof eventname === "undefined") {
+        return Log.metric();
+    }
+    var metrics = {};
+    metrics.level = Log.METRIC;
+    metrics.nodeid = this.id;
+    metrics.event = "node."+this.type+"."+eventname;
+    metrics.msgid = msg._msgid;
+    metrics.value = metricValue;
+    Log.log(metrics);
+}
+
+/**
+ * status: { fill:"red|green", shape:"dot|ring", text:"blah" }
+ */
+Node.prototype.status = function(status) {
+    comms.publish("status/" + this.id, status, true);
+    flows.handleStatus(this,status);
+};
+module.exports = Node;

red/runtime/nodes/credentials.js

@@ -0,0 +1,168 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+
+var log = require("../log");
+
+var credentialCache = {};
+var storage = null;
+var credentialsDef = {};
+
+module.exports = {
+    init: function (_storage) {
+        storage = _storage;
+    },
+
+    /**
+     * Loads the credentials from storage.
+     */
+    load: function () {
+        return storage.getCredentials().then(function (creds) {
+            credentialCache = creds;
+        }).otherwise(function (err) {
+            log.warn(log._("nodes.credentials.error",{message: err}));
+        });
+    },
+
+    /**
+     * Adds a set of credentials for the given node id.
+     * @param id the node id for the credentials
+     * @param creds an object of credential key/value pairs
+     * @return a promise for the saving of credentials to storage
+     */
+    add: function (id, creds) {
+        credentialCache[id] = creds;
+        return storage.saveCredentials(credentialCache);
+    },
+
+    /**
+     * Gets the credentials for the given node id.
+     * @param id the node id for the credentials
+     * @return the credentials
+     */
+    get: function (id) {
+        return credentialCache[id];
+    },
+
+    /**
+     * Deletes the credentials for the given node id.
+     * @param id the node id for the credentials
+     * @return a promise for the saving of credentials to storage
+     */
+    delete: function (id) {
+        delete credentialCache[id];
+        storage.saveCredentials(credentialCache);
+    },
+
+    /**
+     * Deletes any credentials for nodes that no longer exist
+     * @param config a flow config
+     * @return a promise for the saving of credentials to storage
+     */
+    clean: function (config) {
+        var existingIds = {};
+        config.forEach(function(n) {
+            existingIds[n.id] = true;
+        });
+        var deletedCredentials = false;
+        for (var c in credentialCache) {
+            if (credentialCache.hasOwnProperty(c)) {
+                if (!existingIds[c]) {
+                    deletedCredentials = true;
+                    delete credentialCache[c];
+                }
+            }
+        }
+        if (deletedCredentials) {
+            return storage.saveCredentials(credentialCache);
+        } else {
+            return when.resolve();
+        }
+    },
+
+    /**
+     * Registers a node credential definition.
+     * @param type the node type
+     * @param definition the credential definition
+     */
+    register: function (type, definition) {
+        var dashedType = type.replace(/\s+/g, '-');
+        credentialsDef[dashedType] = definition;
+    },
+
+    /**
+     * Extracts and stores any credential updates in the provided node.
+     * The provided node may have a .credentials property that contains
+     * new credentials for the node.
+     * This function loops through the credentials in the definition for
+     * the node-type and applies any of the updates provided in the node.
+     *
+     * This function does not save the credentials to disk as it is expected
+     * to be called multiple times when a new flow is deployed.
+     *
+     * @param node the node to extract credentials from
+     */
+    extract: function(node) {
+        var nodeID = node.id;
+        var nodeType = node.type;
+        var newCreds = node.credentials;
+        if (newCreds) {
+            var savedCredentials = credentialCache[nodeID] || {};
+            var dashedType = nodeType.replace(/\s+/g, '-');
+            var definition = credentialsDef[dashedType];
+            if (!definition) {
+                log.warn(log._("nodes.credentials.not-registered",{type:nodeType}));
+                return;
+            }
+
+            for (var cred in definition) {
+                if (definition.hasOwnProperty(cred)) {
+                    if (newCreds[cred] === undefined) {
+                        continue;
+                    }
+                    if (definition[cred].type == "password" && newCreds[cred] == '__PWRD__') {
+                        continue;
+                    }
+                    if (0 === newCreds[cred].length || /^\s*$/.test(newCreds[cred])) {
+                        delete savedCredentials[cred];
+                        continue;
+                    }
+                    savedCredentials[cred] = newCreds[cred];
+                }
+            }
+            credentialCache[nodeID] = savedCredentials;
+            delete node.credentials;
+        }
+    },
+
+    /**
+     * Saves the credentials to storage
+     * @return a promise for the saving of credentials to storage
+     */
+    save: function () {
+        return storage.saveCredentials(credentialCache);
+    },
+
+    /**
+     * Gets the credential definition for the given node type
+     * @param type the node type
+     * @return the credential definition
+     */
+    getDefinition: function (type) {
+        return credentialsDef[type];
+    }
+}

red/runtime/nodes/flows/Flow.js

@@ -0,0 +1,466 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var clone = require("clone");
+var typeRegistry = require("../registry");
+var Log = require("../../log");
+var redUtil = require("../../util");
+var flowUtil = require("./util");
+
+function Flow(global,flow) {
+    if (typeof flow === 'undefined') {
+        flow = global;
+    }
+    var activeNodes = {};
+    var subflowInstanceNodes = {};
+    var catchNodeMap = {};
+    var statusNodeMap = {};
+
+    this.start = function(diff) {
+        var node;
+        var id;
+        catchNodeMap = {};
+        statusNodeMap = {};
+        for (id in flow.configs) {
+            if (flow.configs.hasOwnProperty(id)) {
+                node = flow.configs[id];
+                if (!activeNodes[id]) {
+                    activeNodes[id] = createNode(node.type,node);
+                }
+            }
+        }
+        if (diff) {
+            for (var j=0;j<diff.rewired.length;j++) {
+                var rewireNode = activeNodes[diff.rewired[j]];
+                if (rewireNode) {
+                    rewireNode.updateWires(flow.nodes[rewireNode.id].wires);
+                }
+            }
+        }
+
+        for (id in flow.nodes) {
+            if (flow.nodes.hasOwnProperty(id)) {
+                node = flow.nodes[id];
+                if (!node.subflow) {
+                    if (!activeNodes[id]) {
+                        activeNodes[id] = createNode(node.type,node);
+                    }
+                } else {
+                    if (!subflowInstanceNodes[id]) {
+                        try {
+                            var nodes = createSubflow(flow.subflows[node.subflow]||global.subflows[node.subflow],node,flow.subflows,global.subflows,activeNodes);
+                            subflowInstanceNodes[id] = nodes.map(function(n) { return n.id});
+                            for (var i=0;i<nodes.length;i++) {
+                                activeNodes[nodes[i].id] = nodes[i];
+                            }
+                        } catch(err) {
+                            console.log(err.stack)
+                        }
+                    }
+                }
+            }
+        }
+
+        for (id in activeNodes) {
+            if (activeNodes.hasOwnProperty(id)) {
+                node = activeNodes[id];
+                if (node.type === "catch") {
+                    catchNodeMap[node.z] = catchNodeMap[node.z] || [];
+                    catchNodeMap[node.z].push(node);
+                } else if (node.type === "status") {
+                    statusNodeMap[node.z] = statusNodeMap[node.z] || [];
+                    statusNodeMap[node.z].push(node);
+                }
+            }
+        }
+    }
+
+    this.stop = function(stopList) {
+        return when.promise(function(resolve) {
+            var i;
+            if (stopList) {
+                for (i=0;i<stopList.length;i++) {
+                    if (subflowInstanceNodes[stopList[i]]) {
+                        // The first in the list is the instance node we already
+                        // know about
+                        stopList = stopList.concat(subflowInstanceNodes[stopList[i]].slice(1))
+                    }
+                }
+            } else {
+                stopList = Object.keys(activeNodes);
+            }
+            var promises = [];
+            for (i=0;i<stopList.length;i++) {
+                var node = activeNodes[stopList[i]];
+                if (node) {
+                    delete activeNodes[stopList[i]];
+                    if (subflowInstanceNodes[stopList[i]]) {
+                        delete subflowInstanceNodes[stopList[i]];
+                    }
+                    try {
+                        var p = node.close();
+                        if (p) {
+                            promises.push(p);
+                        }
+                    } catch(err) {
+                        node.error(err);
+                    }
+                }
+            }
+            when.settle(promises).then(function() {
+                resolve();
+            });
+        });
+    }
+
+    this.update = function(_global,_flow) {
+        global = _global;
+        flow = _flow;
+    }
+
+    this.getNode = function(id) {
+        return activeNodes[id];
+    }
+
+    this.getActiveNodes = function() {
+        return activeNodes;
+    }
+
+    this.handleStatus = function(node,statusMessage) {
+        var targetStatusNodes = null;
+        var reportingNode = node;
+        var handled = false;
+        while(reportingNode && !handled) {
+            targetStatusNodes = statusNodeMap[reportingNode.z];
+            if (targetStatusNodes) {
+                targetStatusNodes.forEach(function(targetStatusNode) {
+                    if (targetStatusNode.scope && targetStatusNode.scope.indexOf(node.id) === -1) {
+                        return;
+                    }
+                    var message = {
+                        status: {
+                            text: "",
+                            source: {
+                                id: node.id,
+                                type: node.type,
+                                name: node.name
+                            }
+                        }
+                    };
+                    if (statusMessage.text) {
+                        message.status.text = statusMessage.text;
+                    }
+                    targetStatusNode.receive(message);
+                    handled = true;
+                });
+            }
+            if (!handled) {
+                reportingNode = activeNodes[reportingNode.z];
+            }
+        }
+    }
+
+    this.handleError = function(node,logMessage,msg) {
+        var count = 1;
+        if (msg && msg.hasOwnProperty("error")) {
+            if (msg.error.hasOwnProperty("source")) {
+                if (msg.error.source.id === node.id) {
+                    count = msg.error.source.count+1;
+                    if (count === 10) {
+                        node.warn(Log._("nodes.flow.error-loop"));
+                        return;
+                    }
+                }
+            }
+        }
+        var targetCatchNodes = null;
+        var throwingNode = node;
+        var handled = false;
+        while (throwingNode && !handled) {
+            targetCatchNodes = catchNodeMap[throwingNode.z];
+            if (targetCatchNodes) {
+                targetCatchNodes.forEach(function(targetCatchNode) {
+                    if (targetCatchNode.scope && targetCatchNode.scope.indexOf(throwingNode.id) === -1) {
+                        return;
+                    }
+                    var errorMessage;
+                    if (msg) {
+                        errorMessage = redUtil.cloneMessage(msg);
+                    } else {
+                        errorMessage = {};
+                    }
+                    if (errorMessage.hasOwnProperty("error")) {
+                        errorMessage._error = errorMessage.error;
+                    }
+                    errorMessage.error = {
+                        message: logMessage.toString(),
+                        source: {
+                            id: node.id,
+                            type: node.type,
+                            name: node.name,
+                            count: count
+                        }
+                    };
+                    targetCatchNode.receive(errorMessage);
+                    handled = true;
+                });
+            }
+            if (!handled) {
+                throwingNode = activeNodes[throwingNode.z];
+            }
+        }
+    }
+
+}
+
+
+function getID() {
+    return (1+Math.random()*4294967295).toString(16);
+}
+
+var EnvVarPropertyRE = /^\$\((\S+)\)$/;
+
+function mapEnvVarProperties(obj,prop) {
+    if (Buffer.isBuffer(obj[prop])) {
+        return;
+    } else if (Array.isArray(obj[prop])) {
+        for (var i=0;i<obj[prop].length;i++) {
+            mapEnvVarProperties(obj[prop],i);
+        }
+    } else if (typeof obj[prop] === 'string') {
+        var m;
+        if ( (m = EnvVarPropertyRE.exec(obj[prop])) !== null) {
+            if (process.env.hasOwnProperty(m[1])) {
+                obj[prop] = process.env[m[1]];
+            }
+        }
+    } else {
+        for (var p in obj[prop]) {
+            if (obj[prop].hasOwnProperty) {
+                mapEnvVarProperties(obj[prop],p);
+            }
+        }
+    }
+}
+
+function createNode(type,config) {
+    // console.log("CREATE",type,config.id);
+    var nn = null;
+    var nt = typeRegistry.get(type);
+    if (nt) {
+        var conf = clone(config);
+        delete conf.credentials;
+        for (var p in conf) {
+            if (conf.hasOwnProperty(p)) {
+                mapEnvVarProperties(conf,p);
+            }
+        }
+        try {
+            nn = new nt(conf);
+        }
+        catch (err) {
+            Log.log({
+                level: Log.ERROR,
+                id:conf.id,
+                type: type,
+                msg: err
+            });
+        }
+    } else {
+        Log.error(Log._("nodes.flow.unknown-type", {type:type}));
+    }
+    return nn;
+}
+
+function createSubflow(sf,sfn,subflows,globalSubflows,activeNodes) {
+    //console.log("CREATE SUBFLOW",sf.id,sfn.id);
+    var nodes = [];
+    var node_map = {};
+    var newNodes = [];
+    var node;
+    var wires;
+    var i,j,k;
+
+    var createNodeInSubflow = function(def) {
+        node = clone(def);
+        var nid = getID();
+        node_map[node.id] = node;
+        node._alias = node.id;
+        node.id = nid;
+        node.z = sfn.id;
+        newNodes.push(node);
+    }
+
+    // Clone all of the subflow node definitions and give them new IDs
+    for (i in sf.configs) {
+        if (sf.configs.hasOwnProperty(i)) {
+            createNodeInSubflow(sf.configs[i]);
+        }
+    }
+    // Clone all of the subflow node definitions and give them new IDs
+    for (i in sf.nodes) {
+        if (sf.nodes.hasOwnProperty(i)) {
+            createNodeInSubflow(sf.nodes[i]);
+        }
+    }
+
+    // Look for any catch/status nodes and update their scope ids
+    // Update all subflow interior wiring to reflect new node IDs
+    for (i=0;i<newNodes.length;i++) {
+        node = newNodes[i];
+        if (node.wires) {
+            var outputs = node.wires;
+            for (j=0;j<outputs.length;j++) {
+                wires = outputs[j];
+                for (k=0;k<wires.length;k++) {
+                    outputs[j][k] = node_map[outputs[j][k]].id
+                }
+            }
+            if ((node.type === 'catch' || node.type === 'status') && node.scope) {
+                node.scope = node.scope.map(function(id) {
+                    return node_map[id]?node_map[id].id:""
+                })
+            } else {
+                for (var prop in node) {
+                    if (node.hasOwnProperty(prop) && prop !== '_alias') {
+                        if (node_map[node[prop]]) {
+                            //console.log("Mapped",node.type,node.id,prop,node_map[node[prop]].id);
+                            node[prop] = node_map[node[prop]].id;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Create a subflow node to accept inbound messages and route appropriately
+    var Node = require("../Node");
+    var subflowInstance = {
+        id: sfn.id,
+        type: sfn.type,
+        z: sfn.z,
+        name: sfn.name,
+        wires: []
+    }
+    if (sf.in) {
+        subflowInstance.wires = sf.in.map(function(n) { return n.wires.map(function(w) { return node_map[w.id].id;})})
+        subflowInstance._originalWires = clone(subflowInstance.wires);
+    }
+    var subflowNode = new Node(subflowInstance);
+
+    subflowNode.on("input", function(msg) { this.send(msg);});
+
+
+    subflowNode._updateWires = subflowNode.updateWires;
+
+    subflowNode.updateWires = function(newWires) {
+        // Wire the subflow outputs
+        if (sf.out) {
+            var node,wires,i,j;
+            // Restore the original wiring to the internal nodes
+            subflowInstance.wires = clone(subflowInstance._originalWires);
+            for (i=0;i<sf.out.length;i++) {
+                wires = sf.out[i].wires;
+                for (j=0;j<wires.length;j++) {
+                    if (wires[j].id != sf.id) {
+                        node = node_map[wires[j].id];
+                        if (node._originalWires) {
+                            node.wires = clone(node._originalWires);
+                        }
+                    }
+                }
+            }
+
+            var modifiedNodes = {};
+            var subflowInstanceModified = false;
+
+            for (i=0;i<sf.out.length;i++) {
+                wires = sf.out[i].wires;
+                for (j=0;j<wires.length;j++) {
+                    if (wires[j].id === sf.id) {
+                        subflowInstance.wires[wires[j].port] = subflowInstance.wires[wires[j].port].concat(newWires[i]);
+                        subflowInstanceModified = true;
+                    } else {
+                        node = node_map[wires[j].id];
+                        node.wires[wires[j].port] = node.wires[wires[j].port].concat(newWires[i]);
+                        modifiedNodes[node.id] = node;
+                    }
+                }
+            }
+            Object.keys(modifiedNodes).forEach(function(id) {
+                var node = modifiedNodes[id];
+                subflowNode.instanceNodes[id].updateWires(node.wires);
+            });
+            if (subflowInstanceModified) {
+                subflowNode._updateWires(subflowInstance.wires);
+            }
+        }
+    }
+
+    nodes.push(subflowNode);
+
+    // Wire the subflow outputs
+    if (sf.out) {
+        var modifiedNodes = {};
+        for (i=0;i<sf.out.length;i++) {
+            wires = sf.out[i].wires;
+            for (j=0;j<wires.length;j++) {
+                if (wires[j].id === sf.id) {
+                    // A subflow input wired straight to a subflow output
+                    subflowInstance.wires[wires[j].port] = subflowInstance.wires[wires[j].port].concat(sfn.wires[i])
+                    subflowNode._updateWires(subflowInstance.wires);
+                } else {
+                    node = node_map[wires[j].id];
+                    modifiedNodes[node.id] = node;
+                    if (!node._originalWires) {
+                        node._originalWires = clone(node.wires);
+                    }
+                    node.wires[wires[j].port] = (node.wires[wires[j].port]||[]).concat(sfn.wires[i]);
+                }
+            }
+        }
+    }
+
+    // Instantiate the nodes
+    for (i=0;i<newNodes.length;i++) {
+        node = newNodes[i];
+        var type = node.type;
+
+        var m = /^subflow:(.+)$/.exec(type);
+        if (!m) {
+            var newNode = createNode(type,node);
+            activeNodes[node.id] = newNode;
+            nodes.push(newNode);
+        } else {
+            var subflowId = m[1];
+            nodes = nodes.concat(createSubflow(subflows[subflowId]||globalSubflows[subflowId],node,subflows,globalSubflows,activeNodes));
+        }
+    }
+
+    subflowNode.instanceNodes = {};
+
+    nodes.forEach(function(node) {
+       subflowNode.instanceNodes[node.id] = node;
+    });
+    return nodes;
+}
+
+module.exports = {
+    create: function(global,conf) {
+        return new Flow(global,conf);
+    }
+}

red/runtime/nodes/flows/index.js

@@ -0,0 +1,392 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var clone = require("clone");
+var when = require("when");
+
+var Flow = require('./Flow');
+
+var typeRegistry = require("../registry");
+var credentials = require("../credentials");
+
+var flowUtil = require("./util");
+var log = require("../../log");
+var events = require("../../events");
+var redUtil = require("../../util");
+var deprecated = require("../registry/deprecated");
+
+var storage = null;
+var settings = null;
+
+var activeConfig = null;
+var activeFlowConfig = null;
+
+var activeFlows = {};
+var started = false;
+
+var activeNodesToFlow = {};
+var subflowInstanceNodeMap = {};
+
+var typeEventRegistered = false;
+
+function init(_settings, _storage) {
+    if (started) {
+        throw new Error("Cannot init without a stop");
+    }
+    settings = _settings;
+    storage = _storage;
+    started = false;
+    if (!typeEventRegistered) {
+        events.on('type-registered',function(type) {
+            if (activeFlowConfig && activeFlowConfig.missingTypes.length > 0) {
+                var i = activeFlowConfig.missingTypes.indexOf(type);
+                if (i != -1) {
+                    log.info(log._("nodes.flows.registered-missing", {type:type}));
+                    activeFlowConfig.missingTypes.splice(i,1);
+                    if (activeFlowConfig.missingTypes.length === 0 && started) {
+                        start();
+                    }
+                }
+            }
+        });
+        typeEventRegistered = true;
+    }
+}
+function load() {
+    return storage.getFlows().then(function(flows) {
+        return credentials.load().then(function() {
+            return setConfig(flows,"load");
+        });
+    }).otherwise(function(err) {
+        log.warn(log._("nodes.flows.error",{message:err.toString()}));
+        console.log(err.stack);
+    });
+}
+
+function setConfig(_config,type) {
+    var config = clone(_config);
+    type = type||"full";
+
+    var credentialsChanged = false;
+    var credentialSavePromise = null;
+    var configSavePromise = null;
+
+    var diff;
+    var newFlowConfig = flowUtil.parseConfig(clone(config));
+    if (type !== 'full' && type !== 'load') {
+        diff = flowUtil.diffConfigs(activeFlowConfig,newFlowConfig);
+    }
+    config.forEach(function(node) {
+        if (node.credentials) {
+            credentials.extract(node);
+            credentialsChanged = true;
+        }
+    });
+    if (credentialsChanged) {
+        credentialSavePromise = credentials.save();
+    } else {
+        credentialSavePromise = when.resolve();
+    }
+    if (type === 'load') {
+        configSavePromise = credentialSavePromise;
+        type = 'full';
+    } else {
+        configSavePromise = credentialSavePromise.then(function() {
+            return storage.saveFlows(config);
+        });
+    }
+
+    return configSavePromise
+        .then(function() {
+            activeConfig = config;
+            activeFlowConfig = newFlowConfig;
+            return credentials.clean(activeConfig).then(function() {
+                if (started) {
+                    return stop(type,diff).then(function() {
+                        start(type,diff);
+                    }).otherwise(function(err) {
+                    })
+                }
+            });
+        });
+}
+
+function getNode(id) {
+    var node;
+    if (activeNodesToFlow[id]) {
+        return activeFlows[activeNodesToFlow[id]].getNode(id);
+    }
+    for (var flowId in activeFlows) {
+        if (activeFlows.hasOwnProperty(flowId)) {
+            node = activeFlows[flowId].getNode(id);
+            if (node) {
+                return node;
+            }
+        }
+    }
+    return null;
+}
+
+function eachNode(cb) {
+    for (var id in activeFlowConfig.allNodes) {
+        if (activeFlowConfig.allNodes.hasOwnProperty(id)) {
+            cb(activeFlowConfig.allNodes[id]);
+        }
+    }
+}
+
+function getConfig() {
+    return activeConfig;
+}
+
+function delegateError(node,logMessage,msg) {
+    if (activeFlows[node.z]) {
+        activeFlows[node.z].handleError(node,logMessage,msg);
+    } else if (activeNodesToFlow[node.z]) {
+        activeFlows[activeNodesToFlow[node.z]].handleError(node,logMessage,msg);
+    } else if (activeFlowConfig.subflows[node.z]) {
+        subflowInstanceNodeMap[node.id].forEach(function(n) {
+            delegateError(getNode(n),logMessage,msg);
+        });
+    }
+}
+function handleError(node,logMessage,msg) {
+    if (node.z) {
+        delegateError(node,logMessage,msg);
+    } else {
+        if (activeFlowConfig.configs[node.id]) {
+            activeFlowConfig.configs[node.id]._users.forEach(function(id) {
+                var userNode = activeFlowConfig.allNodes[id];
+                delegateError(userNode,logMessage,msg);
+            })
+        }
+    }
+}
+
+function delegateStatus(node,statusMessage) {
+    if (activeFlows[node.z]) {
+        activeFlows[node.z].handleStatus(node,statusMessage);
+    }
+}
+function handleStatus(node,statusMessage) {
+    if (node.z) {
+        delegateStatus(node,statusMessage);
+    } else {
+        if (activeFlowConfig.configs[node.id]) {
+            activeFlowConfig.configs[node.id]._users.forEach(function(id) {
+                var userNode = activeFlowConfig.allNodes[id];
+                delegateStatus(userNode,statusMessage);
+            })
+        }
+    }
+}
+
+
+function start(type,diff) {
+    type = type||"full";
+    started = true;
+    var i;
+    if (activeFlowConfig.missingTypes.length > 0) {
+        log.info(log._("nodes.flows.missing-types"));
+        var knownUnknowns = 0;
+        for (i=0;i<activeFlowConfig.missingTypes.length;i++) {
+            var nodeType = activeFlowConfig.missingTypes[i];
+            var info = deprecated.get(nodeType);
+            if (info) {
+                log.info(log._("nodes.flows.missing-type-provided",{type:activeFlowConfig.missingTypes[i],module:info.module}));
+                knownUnknowns += 1;
+            } else {
+                log.info(" - "+activeFlowConfig.missingTypes[i]);
+            }
+        }
+        if (knownUnknowns > 0) {
+            log.info(log._("nodes.flows.missing-type-install-1"));
+            log.info("  npm install <module name>");
+            log.info(log._("nodes.flows.missing-type-install-2"));
+            log.info("  "+settings.userDir);
+        }
+        return;
+    }
+    if (diff) {
+        log.info(log._("nodes.flows.starting-modified-"+type));
+    } else {
+        log.info(log._("nodes.flows.starting-flows"));
+    }
+    var id;
+    if (!diff) {
+        activeFlows['_GLOBAL_'] = Flow.create(activeFlowConfig);
+        for (id in activeFlowConfig.flows) {
+            if (activeFlowConfig.flows.hasOwnProperty(id)) {
+                activeFlows[id] = Flow.create(activeFlowConfig,activeFlowConfig.flows[id]);
+            }
+        }
+    } else {
+        activeFlows['_GLOBAL_'].update(activeFlowConfig,activeFlowConfig);
+        for (id in activeFlowConfig.flows) {
+            if (activeFlowConfig.flows.hasOwnProperty(id)) {
+                if (activeFlows[id]) {
+                    activeFlows[id].update(activeFlowConfig,activeFlowConfig.flows[id]);
+                } else {
+                    activeFlows[id] = Flow.create(activeFlowConfig,activeFlowConfig.flows[id]);
+                }
+            }
+        }
+    }
+
+    for (id in activeFlows) {
+        if (activeFlows.hasOwnProperty(id)) {
+            activeFlows[id].start(diff);
+            var activeNodes = activeFlows[id].getActiveNodes();
+            Object.keys(activeNodes).forEach(function(nid) {
+                activeNodesToFlow[nid] = id;
+                if (activeNodes[nid]._alias) {
+                    subflowInstanceNodeMap[activeNodes[nid]._alias] = subflowInstanceNodeMap[activeNodes[nid]._alias] || [];
+                    subflowInstanceNodeMap[activeNodes[nid]._alias].push(nid);
+                }
+            });
+
+        }
+    }
+    events.emit("nodes-started");
+    if (diff) {
+        log.info(log._("nodes.flows.started-modified-"+type));
+    } else {
+        log.info(log._("nodes.flows.started-flows"));
+    }
+}
+
+function stop(type,diff) {
+    type = type||"full";
+    if (diff) {
+        log.info(log._("nodes.flows.stopping-modified-"+type));
+    } else {
+        log.info(log._("nodes.flows.stopping-flows"));
+    }
+    started = false;
+    var promises = [];
+    var stopList;
+    if (type === 'nodes') {
+        stopList = diff.changed.concat(diff.removed);
+    } else if (type === 'flows') {
+        stopList = diff.changed.concat(diff.removed).concat(diff.linked);
+    }
+    for (var id in activeFlows) {
+        if (activeFlows.hasOwnProperty(id)) {
+            promises = promises.concat(activeFlows[id].stop(stopList));
+            if (!diff || diff.removed.indexOf(id)!==-1) {
+                delete activeFlows[id];
+            }
+        }
+    }
+
+    return when.promise(function(resolve,reject) {
+        when.settle(promises).then(function() {
+            for (id in activeNodesToFlow) {
+                if (activeNodesToFlow.hasOwnProperty(id)) {
+                    if (!activeFlows[activeNodesToFlow[id]]) {
+                        delete activeNodesToFlow[id];
+                    }
+                }
+            }
+            if (stopList) {
+                stopList.forEach(function(id) {
+                    delete activeNodesToFlow[id];
+                });
+            }
+            // Ideally we'd prune just what got stopped - but mapping stopList
+            // id to the list of subflow instance nodes is something only Flow
+            // can do... so cheat by wiping the map knowing it'll be rebuilt
+            // in start()
+            subflowInstanceNodeMap = {};
+            if (diff) {
+                log.info(log._("nodes.flows.stopped-modified-"+type));
+            } else {
+                log.info(log._("nodes.flows.stopped-flows"));
+            }
+            resolve();
+        });
+    });
+}
+
+
+function checkTypeInUse(id) {
+    var nodeInfo = typeRegistry.getNodeInfo(id);
+    if (!nodeInfo) {
+        throw new Error(log._("nodes.index.unrecognised-id", {id:id}));
+    } else {
+        var inUse = {};
+        var config = getConfig();
+        config.forEach(function(n) {
+            inUse[n.type] = (inUse[n.type]||0)+1;
+        });
+        var nodesInUse = [];
+        nodeInfo.types.forEach(function(t) {
+            if (inUse[t]) {
+                nodesInUse.push(t);
+            }
+        });
+        if (nodesInUse.length > 0) {
+            var msg = nodesInUse.join(", ");
+            var err = new Error(log._("nodes.index.type-in-use", {msg:msg}));
+            err.code = "type_in_use";
+            throw err;
+        }
+    }
+}
+
+module.exports = {
+    init: init,
+
+    /**
+     * Load the current flow configuration from storage
+     * @return a promise for the loading of the config
+     */
+    load: load,
+
+    get:getNode,
+    eachNode: eachNode,
+
+    /**
+     * Gets the current flow configuration
+     */
+    getFlows: getConfig,
+
+    /**
+     * Sets the current active config.
+     * @param config the configuration to enable
+     * @param type the type of deployment to do: full (default), nodes, flows, load
+     * @return a promise for the saving/starting of the new flow
+     */
+    setFlows: setConfig,
+
+    /**
+     * Starts the current flow configuration
+     */
+    startFlows: start,
+
+    /**
+     * Stops the current flow configuration
+     * @return a promise for the stopping of the flow
+     */
+    stopFlows: stop,
+
+
+    handleError: handleError,
+    handleStatus: handleStatus,
+
+    checkTypeInUse: checkTypeInUse
+
+};

red/runtime/nodes/flows/util.js

@@ -0,0 +1,327 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+var clone = require("clone");
+var redUtil = require("../../util");
+var subflowInstanceRE = /^subflow:(.+)$/;
+var typeRegistry = require("../registry");
+
+function diffNodes(oldNode,newNode) {
+    if (oldNode == null) {
+        return true;
+    }
+    var oldKeys = Object.keys(oldNode).filter(function(p) { return p != "x" && p != "y" && p != "wires" });
+    var newKeys = Object.keys(newNode).filter(function(p) { return p != "x" && p != "y" && p != "wires" });
+    if (oldKeys.length != newKeys.length) {
+        return true;
+    }
+    for (var i=0;i<newKeys.length;i++) {
+        var p = newKeys[i];
+        if (!redUtil.compareObjects(oldNode[p],newNode[p])) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+module.exports = {
+
+    diffNodes: diffNodes,
+
+    parseConfig: function(config) {
+        var flow = {};
+        flow.allNodes = {};
+        flow.subflows = {};
+        flow.configs = {};
+        flow.flows = {};
+        flow.missingTypes = [];
+
+        config.forEach(function(n) {
+            flow.allNodes[n.id] = clone(n);
+            if (n.type === 'tab') {
+                flow.flows[n.id] = n;
+                flow.flows[n.id].subflows = {};
+                flow.flows[n.id].configs = {};
+                flow.flows[n.id].nodes = {};
+            }
+        });
+
+        config.forEach(function(n) {
+            if (n.type === 'subflow') {
+                flow.subflows[n.id] = n;
+                flow.subflows[n.id].configs = {};
+                flow.subflows[n.id].nodes = {};
+                flow.subflows[n.id].instances = [];
+            }
+        });
+
+        config.forEach(function(n) {
+            if (n.type !== 'subflow' && n.type !== 'tab') {
+                var subflowDetails = subflowInstanceRE.exec(n.type);
+
+                if ( (subflowDetails && !flow.subflows[subflowDetails[1]]) || (!subflowDetails && !typeRegistry.get(n.type)) ) {
+                    if (flow.missingTypes.indexOf(n.type) === -1) {
+                        flow.missingTypes.push(n.type);
+                    }
+                } else {
+                    var container = null;
+                    if (flow.flows[n.z]) {
+                        container = flow.flows[n.z];
+                    } else if (flow.subflows[n.z]) {
+                        container = flow.subflows[n.z];
+                    }
+                    if (n.hasOwnProperty('x') && n.hasOwnProperty('y')) {
+                        if (subflowDetails) {
+                            var subflowType = subflowDetails[1]
+                            n.subflow = subflowType;
+                            flow.subflows[subflowType].instances.push(n)
+                        }
+                        if (container) {
+                            container.nodes[n.id] = n;
+                        }
+                    } else {
+                        if (container) {
+                            container.configs[n.id] = n;
+                        } else {
+                            flow.configs[n.id] = n;
+                            flow.configs[n.id]._users = [];
+                        }
+                    }
+                }
+            }
+        });
+        config.forEach(function(n) {
+            if (n.type !== 'subflow' && n.type !== 'tab') {
+                for (var prop in n) {
+                    if (n.hasOwnProperty(prop) && prop !== 'id' && prop !== 'wires' && prop !== '_users' && flow.configs[n[prop]]) {
+                        // This property references a global config node
+                        flow.configs[n[prop]]._users.push(n.id)
+                    }
+                }
+            }
+        });
+
+        return flow;
+    },
+
+    diffConfigs: function(oldConfig, newConfig) {
+        var id;
+        var node;
+        var nn;
+        var wires;
+        var j,k;
+
+        var changedSubflows = {};
+
+        var added = {};
+        var removed = {};
+        var changed = {};
+        var wiringChanged = {};
+
+        var linkMap = {};
+
+        for (id in oldConfig.allNodes) {
+            if (oldConfig.allNodes.hasOwnProperty(id)) {
+                node = oldConfig.allNodes[id];
+                // build the map of what this node was previously wired to
+                if (node.wires) {
+                    linkMap[node.id] = linkMap[node.id] || [];
+                    for (j=0;j<node.wires.length;j++) {
+                        wires = node.wires[j];
+                        for (k=0;k<wires.length;k++) {
+                            linkMap[node.id].push(wires[k]);
+                            nn = oldConfig.allNodes[wires[k]];
+                            if (nn) {
+                                linkMap[nn.id] = linkMap[nn.id] || [];
+                                linkMap[nn.id].push(node.id);
+                            }
+                        }
+                    }
+                }
+                // This node has been removed
+                if (!newConfig.allNodes.hasOwnProperty(id)) {
+                    removed[id] = node;
+                    // Mark the container as changed
+                    if (newConfig.allNodes[removed[id].z]) {
+                        changed[removed[id].z] = newConfig.allNodes[removed[id].z];
+                        if (changed[removed[id].z].type === "subflow") {
+                            changedSubflows[removed[id].z] = changed[removed[id].z];
+                            delete removed[id];
+                        }
+                    }
+                } else {
+                    // This node has a material configuration change
+                    if (diffNodes(node,newConfig.allNodes[id]) || newConfig.allNodes[id].credentials) {
+                        changed[id] = newConfig.allNodes[id];
+                        if (changed[id].type === "subflow") {
+                            changedSubflows[id] = changed[id];
+                        }
+                        // Mark the container as changed
+                        if (newConfig.allNodes[changed[id].z]) {
+                            changed[changed[id].z] = newConfig.allNodes[changed[id].z];
+                            if (changed[changed[id].z].type === "subflow") {
+                                changedSubflows[changed[id].z] = changed[changed[id].z];
+                                delete changed[id];
+                            }
+                        }
+                    }
+                    // This node's wiring has changed
+                    if (!redUtil.compareObjects(node.wires,newConfig.allNodes[id].wires)) {
+                        wiringChanged[id] = newConfig.allNodes[id];
+                        // Mark the container as changed
+                        if (newConfig.allNodes[wiringChanged[id].z]) {
+                            changed[wiringChanged[id].z] = newConfig.allNodes[wiringChanged[id].z];
+                            if (changed[wiringChanged[id].z].type === "subflow") {
+                                changedSubflows[wiringChanged[id].z] = changed[wiringChanged[id].z];
+                                delete wiringChanged[id];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Look for added nodes
+        for (id in newConfig.allNodes) {
+            if (newConfig.allNodes.hasOwnProperty(id)) {
+                node = newConfig.allNodes[id];
+                // build the map of what this node is now wired to
+                if (node.wires) {
+                    linkMap[node.id] = linkMap[node.id] || [];
+                    for (j=0;j<node.wires.length;j++) {
+                        wires = node.wires[j];
+                        for (k=0;k<wires.length;k++) {
+                            if (linkMap[node.id].indexOf(wires[k]) === -1) {
+                                linkMap[node.id].push(wires[k]);
+                            }
+                            nn = newConfig.allNodes[wires[k]];
+                            if (nn) {
+                                linkMap[nn.id] = linkMap[nn.id] || [];
+                                if (linkMap[nn.id].indexOf(node.id) === -1) {
+                                    linkMap[nn.id].push(node.id);
+                                }
+                            }
+                        }
+                    }
+                }
+                // This node has been added
+                if (!oldConfig.allNodes.hasOwnProperty(id)) {
+                    added[id] = node;
+                    // Mark the container as changed
+                    if (newConfig.allNodes[added[id].z]) {
+                        changed[added[id].z] = newConfig.allNodes[added[id].z];
+                        if (changed[added[id].z].type === "subflow") {
+                            changedSubflows[added[id].z] = changed[added[id].z];
+                            delete added[id];
+                        }
+                    }
+                }
+            }
+        }
+
+        for (id in newConfig.allNodes) {
+            if (newConfig.allNodes.hasOwnProperty(id)) {
+                node = newConfig.allNodes[id];
+                for (var prop in node) {
+                    if (node.hasOwnProperty(prop) && prop != "z" && prop != "id" && prop != "wires") {
+                        // This node has a property that references a changed/removed node
+                        // Assume it is a config node change and mark this node as
+                        // changed.
+                        if (changed[node[prop]] || removed[node[prop]]) {
+                            if (!changed[node.id]) {
+                                changed[node.id] = node;
+                                if (newConfig.allNodes[node.z]) {
+                                    changed[node.z] = newConfig.allNodes[node.z];
+                                    if (changed[node.z].type === "subflow") {
+                                        changedSubflows[node.z] = changed[node.z];
+                                        delete changed[node.id];
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        // Recursively mark all instances of changed subflows as changed
+        var changedSubflowStack = Object.keys(changedSubflows);
+        while(changedSubflowStack.length > 0) {
+            var subflowId = changedSubflowStack.pop();
+            for (id in newConfig.allNodes) {
+                if (newConfig.allNodes.hasOwnProperty(id)) {
+                    node = newConfig.allNodes[id];
+                    if (node.type === 'subflow:'+subflowId) {
+                        if (!changed[node.id]) {
+                            changed[node.id] = node;
+                            if (!changed[changed[node.id].z] && newConfig.allNodes[changed[node.id].z]) {
+                                changed[changed[node.id].z] = newConfig.allNodes[changed[node.id].z];
+                                if (newConfig.allNodes[changed[node.id].z].type === "subflow") {
+                                    // This subflow instance is inside a subflow. Add the
+                                    // containing subflow to the stack to mark
+                                    changedSubflowStack.push(changed[node.id].z);
+                                    delete changed[node.id];
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        var diff = {
+            added:Object.keys(added),
+            changed:Object.keys(changed),
+            removed:Object.keys(removed),
+            rewired:Object.keys(wiringChanged),
+            linked:[]
+        }
+
+        // Traverse the links of all modified nodes to mark the connected nodes
+        var modifiedNodes = diff.added.concat(diff.changed).concat(diff.removed).concat(diff.rewired);
+        var visited = {};
+        while(modifiedNodes.length > 0) {
+            node = modifiedNodes.pop();
+            if (!visited[node]) {
+                visited[node] = true;
+                if (linkMap[node]) {
+                    if (!changed[node] && !added[node] && !removed[node] && !wiringChanged[node]) {
+                        diff.linked.push(node);
+                    }
+                    modifiedNodes = modifiedNodes.concat(linkMap[node]);
+                }
+            }
+        }
+        // for (id in newConfig.allNodes) {
+        //     console.log(
+        //         (added[id]?"+":(changed[id]?"!":" "))+(wiringChanged[id]?"w":" ")+(diff.linked.indexOf(id)!==-1?"~":" "),
+        //         id,
+        //         newConfig.allNodes[id].type,
+        //         newConfig.allNodes[id].name||newConfig.allNodes[id].label||""
+        //     );
+        // }
+        // for (id in removed) {
+        //     console.log(
+        //         "- "+(diff.linked.indexOf(id)!==-1?"~":" "),
+        //         id,
+        //         oldConfig.allNodes[id].type,
+        //         oldConfig.allNodes[id].name||oldConfig.allNodes[id].label||""
+        //     );
+        // }
+
+        return diff;
+    }
+}

red/runtime/nodes/index.js

@@ -0,0 +1,134 @@
+/**
+ * Copyright 2013, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var path = require("path");
+var fs = require("fs");
+
+var registry = require("./registry");
+var credentials = require("./credentials");
+var flows = require("./flows");
+var Node = require("./Node");
+var log = require("../log");
+
+var events = require("../events");
+
+var child_process = require('child_process');
+
+var settings;
+
+/**
+ * Registers a node constructor
+ * @param type - the string type name
+ * @param constructor - the constructor function for this node type
+ * @param opts - optional additional options for the node
+ */
+function registerType(type,constructor,opts) {
+    if (opts && opts.credentials) {
+        credentials.register(type,opts.credentials);
+    }
+    registry.registerType(type,constructor);
+}
+
+/**
+ * Called from a Node's constructor function, invokes the super-class
+ * constructor and attaches any credentials to the node.
+ * @param node the node object being created
+ * @param def the instance definition for the node
+ */
+function createNode(node,def) {
+    Node.call(node,def);
+    var id = node.id;
+    if (def._alias) {
+        id = def._alias;
+    }
+    var creds = credentials.get(id);
+    if (creds) {
+        //console.log("Attaching credentials to ",node.id);
+        node.credentials = creds;
+    } else if (credentials.getDefinition(node.type)) {
+        node.credentials = {};
+    }
+}
+
+function init(_settings,storage) {
+    settings = _settings;
+    credentials.init(storage);
+    flows.init(_settings,storage);
+    registry.init(_settings);
+}
+
+function disableNode(id) {
+    flows.checkTypeInUse(id);
+    return registry.disableNode(id);
+}
+
+function uninstallModule(module) {
+    var info = registry.getModuleInfo(module);
+    if (!info) {
+        throw new Error(log._("nodes.index.unrecognised-module", {module:module}));
+    } else {
+        for (var i=0;i<info.nodes.length;i++) {
+            flows.checkTypeInUse(module+"/"+info.nodes[i].name);
+        }
+        return registry.uninstallModule(module);
+    }
+}
+
+module.exports = {
+    // Lifecycle
+    init: init,
+    load: registry.load,
+
+    // Node registry
+    createNode: createNode,
+    getNode: flows.get,
+    eachNode: flows.eachNode,
+
+    installModule: registry.installModule,
+    uninstallModule: uninstallModule,
+
+    enableNode: registry.enableNode,
+    disableNode: disableNode,
+
+    // Node type registry
+    registerType: registerType,
+    getType: registry.get,
+
+    getNodeInfo: registry.getNodeInfo,
+    getNodeList: registry.getNodeList,
+
+    getModuleInfo: registry.getModuleInfo,
+
+    getNodeConfigs: registry.getNodeConfigs,
+    getNodeConfig: registry.getNodeConfig,
+
+    clearRegistry: registry.clear,
+    cleanModuleList: registry.cleanModuleList,
+
+    // Flow handling
+    loadFlows: flows.load,
+    startFlows: flows.startFlows,
+    stopFlows: flows.stopFlows,
+    setFlows: flows.setFlows,
+    getFlows: flows.getFlows,
+
+    // Credentials
+    addCredentials: credentials.add,
+    getCredentials: credentials.get,
+    deleteCredentials: credentials.delete,
+    getCredentialDefinition: credentials.getDefinition
+};

red/runtime/nodes/registry/deprecated.js

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+ 
+var nodes = {
+    "irc in":     {module:"node-red-node-irc"},
+    "irc out":    {module:"node-red-node-irc"},
+    "irc-server": {module:"node-red-node-irc"},
+    
+    "arduino in":    {module:"node-red-node-arduino"},
+    "arduino out":   {module:"node-red-node-arduino"},
+    "arduino-board": {module:"node-red-node-arduino"},
+    
+    "redis out": {module:"node-red-node-redis"},
+    
+    "mongodb": {module:"node-red-node-mongodb"},
+    "mongodb out": {module:"node-red-node-mongodb"},
+    
+    "serial in": {module:"node-red-node-serialport"},
+    "serial out": {module:"node-red-node-serialport"},
+    "serial-port": {module:"node-red-node-serialport"},
+    
+    "twitter-credentials": {module:"node-red-node-twitter"},
+    "twitter in": {module:"node-red-node-twitter"},
+    "twitter out": {module:"node-red-node-twitter"},
+    
+    "e-mail": {module:"node-red-node-email"},
+    "e-mail in": {module:"node-red-node-email"},
+    
+    "feedparse": {module:"node-red-node-feedparser"}
+}
+
+module.exports = {
+    get: function(id) {
+        return nodes[id];
+    }
+}

red/runtime/nodes/registry/index.js

@@ -0,0 +1,85 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var fs = require("fs");
+var path = require("path");
+
+var events = require("../../events");
+var registry = require("./registry");
+var loader = require("./loader");
+var installer = require("./installer");
+
+var settings;
+
+function init(_settings) {
+    settings = _settings;
+    installer.init(settings);
+    loader.init(settings);
+    registry.init(settings,loader);
+}
+//TODO: defaultNodesDir/disableNodePathScan are to make testing easier.
+//      When the tests are componentized to match the new registry structure,
+//      these flags belong on localfilesystem.load, not here.
+function load(defaultNodesDir,disableNodePathScan) {
+    return loader.load(defaultNodesDir,disableNodePathScan);
+}
+
+function addModule(module) {
+    return loader.addModule(module).then(function() {
+        return registry.getModuleInfo(module);
+    });
+}
+
+function enableNodeSet(typeOrId) {
+    return registry.enableNodeSet(typeOrId).then(function() {
+        var nodeSet = registry.getNodeInfo(typeOrId);
+        if (!nodeSet.loaded) {
+            return loader.loadNodeSet(registry.getFullNodeInfo(typeOrId)).then(function() {
+                return registry.getNodeInfo(typeOrId);
+            });
+        }
+        return when.resolve(nodeSet);
+    });
+}
+
+module.exports = {
+    init:init,
+    load:load,
+    clear: registry.clear,
+    registerType: registry.registerNodeConstructor,
+
+    get: registry.getNodeConstructor,
+    getNodeInfo: registry.getNodeInfo,
+    getNodeList: registry.getNodeList,
+
+    getModuleInfo: registry.getModuleInfo,
+    getModuleList: registry.getModuleList,
+
+    getNodeConfigs: registry.getAllNodeConfigs,
+    getNodeConfig: registry.getNodeConfig,
+
+    enableNode: enableNodeSet,
+    disableNode: registry.disableNodeSet,
+
+    addModule: addModule,
+    removeModule: registry.removeModule,
+
+    installModule: installer.installModule,
+    uninstallModule: installer.uninstallModule,
+
+    cleanModuleList: registry.cleanModuleList
+};

red/runtime/nodes/registry/installer.js

@@ -0,0 +1,186 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+
+var when = require("when");
+var path = require("path");
+var fs = require("fs");
+
+var registry = require("./registry");
+var log = require("../../log");
+
+var events = require("../../events");
+
+var child_process = require('child_process');
+
+var settings;
+
+var moduleRe = /^[^/]+$/;
+var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
+
+function init(_settings) {
+    settings = _settings;
+}
+
+function checkModulePath(folder) {
+    var moduleName;
+    var err;
+    var fullPath = path.resolve(folder);
+    var packageFile = path.join(fullPath,'package.json');
+    if (fs.existsSync(packageFile)) {
+        var pkg = require(packageFile);
+        moduleName = pkg.name;
+        if (!pkg['node-red']) {
+            // TODO: nls
+            err = new Error("Invalid Node-RED module");
+            err.code = 'invalid_module';
+            throw err;
+        }
+    } else {
+        err = new Error("Module not found");
+        err.code = 404;
+        throw err;
+    }
+    return moduleName;
+}
+
+function checkExistingModule(module) {
+    if (registry.getModuleInfo(module)) {
+        // TODO: nls
+        var err = new Error("Module already loaded");
+        err.code = "module_already_loaded";
+        throw err;
+    }
+}
+
+function installModule(module) {
+    //TODO: ensure module is 'safe'
+    return when.promise(function(resolve,reject) {
+        var installName = module;
+
+        try {
+            if (moduleRe.test(module)) {
+                // Simple module name - assume it can be npm installed
+            } else if (slashRe.test(module)) {
+                // A path - check if there's a valid package.json
+                installName = module;
+                module = checkModulePath(module);
+            }
+            checkExistingModule(module);
+        } catch(err) {
+            return reject(err);
+        }
+        log.info(log._("server.install.installing",{name: module}));
+
+        var installDir = settings.userDir || process.env.NODE_RED_HOME || ".";
+        var child = child_process.execFile('npm',['install','--production',installName],
+            {
+                cwd: installDir
+            },
+            function(err, stdin, stdout) {
+                if (err) {
+                    var lookFor404 = new RegExp(" 404 .*"+installName+"$","m");
+                    if (lookFor404.test(stdout)) {
+                        log.warn(log._("server.install.install-failed-not-found",{name:module}));
+                        var e = new Error("Module not found");
+                        e.code = 404;
+                        reject(e);
+                    } else {
+                        log.warn(log._("server.install.install-failed-long",{name:module}));
+                        log.warn("------------------------------------------");
+                        log.warn(err.toString());
+                        log.warn("------------------------------------------");
+                        reject(new Error(log._("server.install.install-failed")));
+                    }
+                } else {
+                    log.info(log._("server.install.installed",{name:module}));
+                    resolve(require("./index").addModule(module).then(reportAddedModules));
+                }
+            }
+        );
+    });
+}
+
+
+function reportAddedModules(info) {
+    //comms.publish("node/added",info.nodes,false);
+    if (info.nodes.length > 0) {
+        log.info(log._("server.added-types"));
+        for (var i=0;i<info.nodes.length;i++) {
+            for (var j=0;j<info.nodes[i].types.length;j++) {
+                log.info(" - "+
+                    (info.nodes[i].module?info.nodes[i].module+":":"")+
+                    info.nodes[i].types[j]+
+                    (info.nodes[i].err?" : "+info.nodes[i].err:"")
+                );
+            }
+        }
+    }
+    return info;
+}
+
+function reportRemovedModules(removedNodes) {
+    //comms.publish("node/removed",removedNodes,false);
+    log.info(log._("server.removed-types"));
+    for (var j=0;j<removedNodes.length;j++) {
+        for (var i=0;i<removedNodes[j].types.length;i++) {
+            log.info(" - "+(removedNodes[j].module?removedNodes[j].module+":":"")+removedNodes[j].types[i]);
+        }
+    }
+    return removedNodes;
+}
+
+function uninstallModule(module) {
+    return when.promise(function(resolve,reject) {
+        if (/[\s;]/.test(module)) {
+            reject(new Error(log._("server.install.invalid")));
+            return;
+        }
+        var installDir = settings.userDir || process.env.NODE_RED_HOME || ".";
+        var moduleDir = path.join(installDir,"node_modules",module);
+        if (!fs.existsSync(moduleDir)) {
+            return reject(new Error(log._("server.install.uninstall-failed",{name:module})));
+        }
+
+        var list = registry.removeModule(module);
+        log.info(log._("server.install.uninstalling",{name:module}));
+        var child = child_process.execFile('npm',['remove',module],
+            {
+                cwd: installDir
+            },
+            function(err, stdin, stdout) {
+                if (err) {
+                    log.warn(log._("server.install.uninstall-failed-long",{name:module}));
+                    log.warn("------------------------------------------");
+                    log.warn(err.toString());
+                    log.warn("------------------------------------------");
+                    reject(new Error(log._("server.install.uninstall-failed",{name:module})));
+                } else {
+                    log.info(log._("server.install.uninstalled",{name:module}));
+                    reportRemovedModules(list);
+                    resolve(list);
+                }
+            }
+        );
+    });
+}
+
+module.exports = {
+    init: init,
+
+    installModule: installModule,
+    uninstallModule: uninstallModule
+}

red/runtime/nodes/registry/loader.js

@@ -0,0 +1,360 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var fs = require("fs");
+var path = require("path");
+var semver = require("semver");
+
+var events = require("../../events");
+
+var localfilesystem = require("./localfilesystem");
+var registry = require("./registry");
+
+var RED;
+var settings;
+
+var i18n = require("../../i18n");
+
+events.on("node-locales-dir", function(info) {
+   i18n.registerMessageCatalog(info.namespace,info.dir,info.file);
+});
+
+function init(_settings) {
+    settings = _settings;
+    localfilesystem.init(settings);
+    RED = require('../../../red');
+}
+
+function load(defaultNodesDir,disableNodePathScan) {
+    // To skip node scan, the following line will use the stored node list.
+    // We should expose that as an option at some point, although the
+    // performance gains are minimal.
+    //return loadNodeFiles(registry.getModuleList());
+
+    var nodeFiles = localfilesystem.getNodeFiles(defaultNodesDir,disableNodePathScan);
+    return loadNodeFiles(nodeFiles);
+}
+
+function loadNodeFiles(nodeFiles) {
+    var promises = [];
+    for (var module in nodeFiles) {
+        /* istanbul ignore else */
+        if (nodeFiles.hasOwnProperty(module)) {
+            if (nodeFiles[module].redVersion &&
+                !semver.satisfies(RED.version().replace("-git",""), nodeFiles[module].redVersion)) {
+                //TODO: log it
+                continue;
+            }
+            if (module == "node-red" || !registry.getModuleInfo(module)) {
+                var first = true;
+                for (var node in nodeFiles[module].nodes) {
+                    /* istanbul ignore else */
+                    if (nodeFiles[module].nodes.hasOwnProperty(node)) {
+                        if (module != "node-red" && first) {
+                            // Check the module directory exists
+                            first = false;
+                            var fn = nodeFiles[module].nodes[node].file;
+                            var parts = fn.split("/");
+                            var i = parts.length-1;
+                            for (;i>=0;i--) {
+                                if (parts[i] == "node_modules") {
+                                    break;
+                                }
+                            }
+                            var moduleFn = parts.slice(0,i+2).join("/");
+
+                            try {
+                                var stat = fs.statSync(moduleFn);
+                            } catch(err) {
+                                // Module not found, don't attempt to load its nodes
+                                break;
+                            }
+                        }
+
+                        try {
+                            promises.push(loadNodeConfig(nodeFiles[module].nodes[node]))
+                        } catch(err) {
+                            //
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return when.settle(promises).then(function(results) {
+        var nodes = results.map(function(r) {
+            registry.addNodeSet(r.value.id,r.value,r.value.version);
+            return r.value;
+        });
+        return loadNodeSetList(nodes);
+    });
+}
+
+function loadNodeConfig(fileInfo) {
+    return when.promise(function(resolve) {
+        var file = fileInfo.file;
+        var module = fileInfo.module;
+        var name = fileInfo.name;
+        var version = fileInfo.version;
+
+        var id = module + "/" + name;
+        var info = registry.getNodeInfo(id);
+        var isEnabled = true;
+        if (info) {
+            if (info.hasOwnProperty("loaded")) {
+                throw new Error(file+" already loaded");
+            }
+            isEnabled = info.enabled;
+        }
+
+        var node = {
+            id: id,
+            module: module,
+            name: name,
+            file: file,
+            template: file.replace(/\.js$/,".html"),
+            enabled: isEnabled,
+            loaded:false,
+            version: version
+        };
+        if (fileInfo.hasOwnProperty("types")) {
+            node.types = fileInfo.types;
+        }
+
+        fs.readFile(node.template,'utf8', function(err,content) {
+            if (err) {
+                node.types = [];
+                if (err.code === 'ENOENT') {
+                    if (!node.types) {
+                        node.types = [];
+                    }
+                    node.err = "Error: "+file+" does not exist";
+                } else {
+                    node.types = [];
+                    node.err = err.toString();
+                }
+                resolve(node);
+            } else {
+                var types = [];
+
+                var regExp = /<script ([^>]*)data-template-name=['"]([^'"]*)['"]/gi;
+                var match = null;
+
+                while((match = regExp.exec(content)) !== null) {
+                    types.push(match[2]);
+                }
+                node.types = types;
+
+                var langRegExp = /^<script[^>]* data-lang=['"](.+?)['"]/i;
+                regExp = /(<script[^>]* data-help-name=[\s\S]*?<\/script>)/gi;
+                match = null;
+                var mainContent = "";
+                var helpContent = {};
+                var index = 0;
+                while((match = regExp.exec(content)) !== null) {
+                    mainContent += content.substring(index,regExp.lastIndex-match[1].length);
+                    index = regExp.lastIndex;
+                    var help = content.substring(regExp.lastIndex-match[1].length,regExp.lastIndex);
+
+                    var lang = "en-US";
+                    if ((match = langRegExp.exec(help)) !== null) {
+                        lang = match[1];
+                    }
+                    if (!helpContent.hasOwnProperty(lang)) {
+                        helpContent[lang] = "";
+                    }
+
+                    helpContent[lang] += help;
+                }
+                mainContent += content.substring(index);
+
+                node.config = mainContent;
+                node.help = helpContent;
+                // TODO: parse out the javascript portion of the template
+                //node.script = "";
+                for (var i=0;i<node.types.length;i++) {
+                    if (registry.getTypeId(node.types[i])) {
+                        node.err = node.types[i]+" already registered";
+                        break;
+                    }
+                }
+                fs.stat(path.join(path.dirname(file),"locales"),function(err,stat) {
+                    if (!err) {
+                        node.namespace = node.id;
+                        i18n.registerMessageCatalog(node.id,
+                                path.join(path.dirname(file),"locales"),
+                                path.basename(file,".js")+".json")
+                            .then(function() {
+                                resolve(node);
+                            });
+                    } else {
+                        node.namespace = node.module;
+                        resolve(node);
+                    }
+                });
+            }
+        });
+    });
+}
+
+
+//function getAPIForNode(node) {
+//    var red = {
+//        nodes: RED.nodes,
+//        library: RED.library,
+//        credentials: RED.credentials,
+//        events: RED.events,
+//        log: RED.log,
+//
+//    }
+//
+//}
+
+
+/**
+ * Loads the specified node into the runtime
+ * @param node a node info object - see loadNodeConfig
+ * @return a promise that resolves to an update node info object. The object
+ *         has the following properties added:
+ *            err: any error encountered whilst loading the node
+ *
+ */
+function loadNodeSet(node) {
+    var nodeDir = path.dirname(node.file);
+    var nodeFn = path.basename(node.file);
+    if (!node.enabled) {
+        return when.resolve(node);
+    } else {
+    }
+    try {
+        var loadPromise = null;
+        var r = require(node.file);
+        if (typeof r === "function") {
+
+            var red = {};
+            for (var i in RED) {
+                if (RED.hasOwnProperty(i) && !/^(init|start|stop)$/.test(i)) {
+                    var propDescriptor = Object.getOwnPropertyDescriptor(RED,i);
+                    Object.defineProperty(red,i,propDescriptor);
+                }
+            }
+            red["_"] = function() {
+                var args = Array.prototype.slice.call(arguments, 0);
+                args[0] = node.namespace+":"+args[0];
+                return i18n._.apply(null,args);
+            }
+            var promise = r(red);
+            if (promise != null && typeof promise.then === "function") {
+                loadPromise = promise.then(function() {
+                    node.enabled = true;
+                    node.loaded = true;
+                    return node;
+                }).otherwise(function(err) {
+                    node.err = err;
+                    return node;
+                });
+            }
+        }
+        if (loadPromise == null) {
+            node.enabled = true;
+            node.loaded = true;
+            loadPromise = when.resolve(node);
+        }
+        return loadPromise;
+    } catch(err) {
+        node.err = err;
+        return when.resolve(node);
+    }
+}
+
+function loadNodeSetList(nodes) {
+    var promises = [];
+    nodes.forEach(function(node) {
+        if (!node.err) {
+            promises.push(loadNodeSet(node));
+        } else {
+            promises.push(node);
+        }
+    });
+
+    return when.settle(promises).then(function() {
+        if (settings.available()) {
+            return registry.saveNodeList();
+        } else {
+            return;
+        }
+    });
+}
+
+function addModule(module) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+    var nodes = [];
+    if (registry.getModuleInfo(module)) {
+        // TODO: nls
+        var e = new Error("module_already_loaded");
+        e.code = "module_already_loaded";
+        return when.reject(e);
+    }
+    try {
+        var moduleFiles = localfilesystem.getModuleFiles(module);
+        return loadNodeFiles(moduleFiles);
+    } catch(err) {
+        return when.reject(err);
+    }
+}
+
+function loadNodeHelp(node,lang) {
+    var dir = path.dirname(node.template);
+    var base = path.basename(node.template);
+    var localePath = path.join(dir,"locales",lang,base);
+    try {
+        // TODO: make this async
+        var content = fs.readFileSync(localePath, "utf8")
+        return content;
+    } catch(err) {
+        return null;
+    }
+}
+
+function getNodeHelp(node,lang) {
+    if (!node.help[lang]) {
+        var help = loadNodeHelp(node,lang);
+        if (help == null) {
+            var langParts = lang.split("-");
+            if (langParts.length == 2) {
+                help = loadNodeHelp(node,langParts[0]);
+            }
+        }
+        if (help) {
+            node.help[lang] = help;
+        } else {
+            node.help[lang] = node.help["en-US"];
+        }
+
+    }
+    return node.help[lang];
+}
+
+module.exports = {
+    init: init,
+    load: load,
+    addModule: addModule,
+    loadNodeSet: loadNodeSet,
+    getNodeHelp: getNodeHelp
+}

red/runtime/nodes/registry/localfilesystem.js

@@ -0,0 +1,286 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var fs = require("fs");
+var path = require("path");
+
+var events = require("../../events");
+var log = require("../../log");
+
+var settings;
+var defaultNodesDir = path.resolve(path.join(__dirname,"..","..","..","..","nodes"));
+var disableNodePathScan = false;
+
+function init(_settings,_defaultNodesDir,_disableNodePathScan) {
+    settings = _settings;
+    if (_disableNodePathScan) {
+        disableNodePathScan = _disableNodePathScan;
+    }
+    if (_defaultNodesDir) {
+        defaultNodesDir = path.resolve(_defaultNodesDir);
+    }
+}
+
+function isExcluded(name) {
+     if (settings.nodesExcludes) {
+        for (var i=0;i<settings.nodesExcludes.length;i++) {
+            if (settings.nodesExcludes[i] == name) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+function getLocalFile(file) {
+    if (isExcluded(path.basename(file))) {
+        return null;
+    }
+    if (fs.existsSync(file.replace(/\.js$/,".html"))) {
+        return {
+            file:    file,
+            module:  "node-red",
+            name:    path.basename(file).replace(/^\d+-/,"").replace(/\.js$/,""),
+            version: settings.version
+        };
+    }
+    return null;
+
+}
+
+
+/**
+ * Synchronously walks the directory looking for node files.
+ * Emits 'node-icon-dir' events for an icon dirs found
+ * @param dir the directory to search
+ * @return an array of fully-qualified paths to .js files
+ */
+function getLocalNodeFiles(dir) {
+    var result = [];
+    var files = [];
+    try {
+        files = fs.readdirSync(dir);
+    } catch(err) {
+        return result;
+    }
+    files.sort();
+    files.forEach(function(fn) {
+        var stats = fs.statSync(path.join(dir,fn));
+        if (stats.isFile()) {
+            if (/\.js$/.test(fn)) {
+                var info = getLocalFile(path.join(dir,fn));
+                if (info) {
+                    result.push(info);
+                }
+            }
+        } else if (stats.isDirectory()) {
+            // Ignore /.dirs/, /lib/ /node_modules/
+            if (!/^(\..*|lib|icons|node_modules|test|locales)$/.test(fn)) {
+                result = result.concat(getLocalNodeFiles(path.join(dir,fn)));
+            } else if (fn === "icons") {
+                events.emit("node-icon-dir",path.join(dir,fn));
+            }
+        }
+    });
+    return result;
+}
+
+function scanDirForNodesModules(dir,moduleName) {
+    var results = [];
+    try {
+        var files = fs.readdirSync(dir);
+        for (var i=0;i<files.length;i++) {
+            var fn = files[i];
+            if (!isExcluded(fn) && (!moduleName || fn == moduleName)) {
+                var pkgfn = path.join(dir,fn,"package.json");
+                try {
+                    var pkg = require(pkgfn);
+                    if (pkg['node-red']) {
+                        var moduleDir = path.join(dir,fn);
+                        results.push({dir:moduleDir,package:pkg});
+                    }
+                } catch(err) {
+                    if (err.code != "MODULE_NOT_FOUND") {
+                        // TODO: handle unexpected error
+                    }
+                }
+                if (fn == moduleName) {
+                    break;
+                }
+            }
+        }
+    } catch(err) {
+    }
+    return results;
+}
+
+/**
+ * Scans the node_modules path for nodes
+ * @param moduleName the name of the module to be found
+ * @return a list of node modules: {dir,package}
+ */
+function scanTreeForNodesModules(moduleName) {
+    var dir = __dirname+"/../../../../nodes";
+    var results = [];
+    var userDir;
+
+    if (settings.userDir) {
+        userDir = path.join(settings.userDir,"node_modules");
+        results = results.concat(scanDirForNodesModules(userDir,moduleName));
+    }
+
+    var up = path.resolve(path.join(dir,".."));
+    while (up !== dir) {
+        var pm = path.join(dir,"node_modules");
+        if (pm != userDir) {
+            results = results.concat(scanDirForNodesModules(pm,moduleName));
+        }
+        dir = up;
+        up = path.resolve(path.join(dir,".."));
+    }
+    return results;
+}
+
+function getModuleNodeFiles(module) {
+
+    var moduleDir = module.dir;
+    var pkg = module.package;
+
+    var nodes = pkg['node-red'].nodes||{};
+    var results = [];
+    var iconDirs = [];
+
+    for (var n in nodes) {
+        /* istanbul ignore else */
+        if (nodes.hasOwnProperty(n)) {
+            var file = path.join(moduleDir,nodes[n]);
+            results.push({
+                file:    file,
+                module:  pkg.name,
+                name:    n,
+                version: pkg.version
+            });
+            var iconDir = path.join(moduleDir,path.dirname(nodes[n]),"icons");
+            if (iconDirs.indexOf(iconDir) == -1) {
+                if (fs.existsSync(iconDir)) {
+                    events.emit("node-icon-dir",iconDir);
+                    iconDirs.push(iconDir);
+                }
+            }
+        }
+    }
+    return results;
+}
+
+function getNodeFiles(_defaultNodesDir,disableNodePathScan) {
+
+    if (_defaultNodesDir) {
+        defaultNodesDir = _defaultNodesDir;
+    }
+
+    var dir;
+    // Find all of the nodes to load
+    //console.log(defaultNodesDir);
+    var nodeFiles = getLocalNodeFiles(path.resolve(defaultNodesDir));
+    //console.log(nodeFiles);
+
+    var defaultLocalesPath = path.resolve(path.join(defaultNodesDir,"core","locales"));
+    events.emit("node-locales-dir", {
+        namespace:"node-red",
+        dir: defaultLocalesPath,
+        file: "messages.json"
+    });
+
+    if (settings.userDir) {
+        dir = path.join(settings.userDir,"nodes");
+        nodeFiles = nodeFiles.concat(getLocalNodeFiles(dir));
+    }
+    if (settings.nodesDir) {
+        dir = settings.nodesDir;
+        if (typeof settings.nodesDir == "string") {
+            dir = [dir];
+        }
+        for (var i=0;i<dir.length;i++) {
+            nodeFiles = nodeFiles.concat(getLocalNodeFiles(dir[i]));
+        }
+    }
+
+    var nodeList = {
+        "node-red": {
+            name: "node-red",
+            version: settings.version,
+            nodes: {}
+        }
+    }
+    nodeFiles.forEach(function(node) {
+        nodeList["node-red"].nodes[node.name] = node;
+    });
+
+    if (!disableNodePathScan) {
+        var moduleFiles = scanTreeForNodesModules();
+        moduleFiles.forEach(function(moduleFile) {
+            var nodeModuleFiles = getModuleNodeFiles(moduleFile);
+            nodeList[moduleFile.package.name] = {
+                name: moduleFile.package.name,
+                version: moduleFile.package.version,
+                nodes: {}
+            };
+            if (moduleFile.package['node-red'].version) {
+                nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
+            }
+            nodeModuleFiles.forEach(function(node) {
+                nodeList[moduleFile.package.name].nodes[node.name] = node;
+            });
+            nodeFiles = nodeFiles.concat(nodeModuleFiles);
+        });
+    }
+    return nodeList;
+}
+
+function getModuleFiles(module) {
+    var nodeList = {};
+
+    var moduleFiles = scanTreeForNodesModules(module);
+    if (moduleFiles.length === 0) {
+        var err = new Error(log._("nodes.registry.localfilesystem.module-not-found", {module:module}));
+        err.code = 'MODULE_NOT_FOUND';
+        throw err;
+    }
+
+    moduleFiles.forEach(function(moduleFile) {
+        var nodeModuleFiles = getModuleNodeFiles(moduleFile);
+        nodeList[moduleFile.package.name] = {
+            name: moduleFile.package.name,
+            version: moduleFile.package.version,
+            nodes: {}
+        };
+        if (moduleFile.package['node-red'].version) {
+            nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
+        }
+        nodeModuleFiles.forEach(function(node) {
+            nodeList[moduleFile.package.name].nodes[node.name] = node;
+        });
+    });
+    return nodeList;
+}
+
+
+module.exports = {
+    init: init,
+    getNodeFiles: getNodeFiles,
+    getLocalFile: getLocalFile,
+    getModuleFiles: getModuleFiles
+}

red/runtime/nodes/registry/registry.js

@@ -0,0 +1,514 @@
+/**
+ * Copyright 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+ //var UglifyJS = require("uglify-js");
+var util = require("util");
+var when = require("when");
+var events = require("../../events");
+
+var settings;
+
+var Node;
+
+var loader;
+
+var nodeConfigCache = null;
+var moduleConfigs = {};
+var nodeList = [];
+var nodeConstructors = {};
+var nodeTypeToId = {};
+var moduleNodes = {};
+
+function init(_settings,_loader) {
+    settings = _settings;
+    loader = _loader;
+    if (settings.available()) {
+        moduleConfigs = loadNodeConfigs();
+    } else {
+        moduleConfigs = {};
+    }
+    moduleNodes = {};
+    nodeTypeToId = {};
+    nodeConstructors = {};
+    nodeList = [];
+    nodeConfigCache = null;
+    Node = require("../Node");
+}
+
+function filterNodeInfo(n) {
+    var r = {
+        id: n.id||n.module+"/"+n.name,
+        name: n.name,
+        types: n.types,
+        enabled: n.enabled
+    };
+    if (n.hasOwnProperty("module")) {
+        r.module = n.module;
+    }
+    if (n.hasOwnProperty("err")) {
+        r.err = n.err.toString();
+    }
+    return r;
+}
+
+
+
+function getModule(id) {
+    return id.split("/")[0];
+}
+
+function getNode(id) {
+    return id.split("/")[1];
+}
+
+function saveNodeList() {
+    var moduleList = {};
+
+    for (var module in moduleConfigs) {
+        /* istanbul ignore else */
+        if (moduleConfigs.hasOwnProperty(module)) {
+            if (Object.keys(moduleConfigs[module].nodes).length > 0) {
+                if (!moduleList[module]) {
+                    moduleList[module] = {
+                        name: module,
+                        version: moduleConfigs[module].version,
+                        nodes: {}
+                    };
+                }
+                var nodes = moduleConfigs[module].nodes;
+                for(var node in nodes) {
+                    /* istanbul ignore else */
+                    if (nodes.hasOwnProperty(node)) {
+                        var config = nodes[node];
+                        var n = filterNodeInfo(config);
+                        delete n.err;
+                        delete n.file;
+                        delete n.id;
+                        n.file = config.file;
+                        moduleList[module].nodes[node] = n;
+                    }
+                }
+            }
+        }
+    }
+    if (settings.available()) {
+        return settings.set("nodes",moduleList);
+    } else {
+        return when.reject("Settings unavailable");
+    }
+}
+
+function loadNodeConfigs() {
+    var configs = settings.get("nodes");
+
+    if (!configs) {
+        return {};
+    } else if (configs['node-red']) {
+        return configs;
+    } else {
+        // Migrate from the 0.9.1 format of settings
+        var newConfigs = {};
+        for (var id in configs) {
+            /* istanbul ignore else */
+            if (configs.hasOwnProperty(id)) {
+                var nodeConfig = configs[id];
+                var moduleName;
+                var nodeSetName;
+
+                if (nodeConfig.module) {
+                    moduleName = nodeConfig.module;
+                    nodeSetName = nodeConfig.name.split(":")[1];
+                } else {
+                    moduleName = "node-red";
+                    nodeSetName = nodeConfig.name.replace(/^\d+-/,"").replace(/\.js$/,"");
+                }
+
+                if (!newConfigs[moduleName]) {
+                    newConfigs[moduleName] = {
+                        name: moduleName,
+                        nodes:{}
+                    };
+                }
+                newConfigs[moduleName].nodes[nodeSetName] = {
+                    name: nodeSetName,
+                    types: nodeConfig.types,
+                    enabled: nodeConfig.enabled,
+                    module: moduleName
+                };
+            }
+        }
+        settings.set("nodes",newConfigs);
+        return newConfigs;
+    }
+}
+
+function addNodeSet(id,set,version) {
+    if (!set.err) {
+        set.types.forEach(function(t) {
+            nodeTypeToId[t] = id;
+        });
+    }
+    moduleNodes[set.module] = moduleNodes[set.module]||[];
+    moduleNodes[set.module].push(set.name);
+
+    if (!moduleConfigs[set.module]) {
+        moduleConfigs[set.module] = {
+            name: set.module,
+            nodes: {}
+        };
+    }
+
+    if (version) {
+        moduleConfigs[set.module].version = version;
+    }
+
+    moduleConfigs[set.module].nodes[set.name] = set;
+    nodeList.push(id);
+    nodeConfigCache = null;
+}
+
+function removeNode(id) {
+    var config = moduleConfigs[getModule(id)].nodes[getNode(id)];
+    if (!config) {
+        throw new Error("Unrecognised id: "+id);
+    }
+    delete moduleConfigs[getModule(id)].nodes[getNode(id)];
+    var i = nodeList.indexOf(id);
+    if (i > -1) {
+        nodeList.splice(i,1);
+    }
+    config.types.forEach(function(t) {
+        var typeId = nodeTypeToId[t];
+        if (typeId === id) {
+            delete nodeConstructors[t];
+            delete nodeTypeToId[t];
+        }
+    });
+    config.enabled = false;
+    config.loaded = false;
+    nodeConfigCache = null;
+    return filterNodeInfo(config);
+}
+
+function removeModule(module) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+    var nodes = moduleNodes[module];
+    if (!nodes) {
+        throw new Error("Unrecognised module: "+module);
+    }
+    var infoList = [];
+    for (var i=0;i<nodes.length;i++) {
+        infoList.push(removeNode(module+"/"+nodes[i]));
+    }
+    delete moduleNodes[module];
+    delete moduleConfigs[module];
+    saveNodeList();
+    return infoList;
+}
+
+function getNodeInfo(typeOrId) {
+    var id = typeOrId;
+    if (nodeTypeToId[typeOrId]) {
+        id = nodeTypeToId[typeOrId];
+    }
+    /* istanbul ignore else */
+    if (id) {
+        var module = moduleConfigs[getModule(id)];
+        if (module) {
+            var config = module.nodes[getNode(id)];
+            if (config) {
+                var info = filterNodeInfo(config);
+                if (config.hasOwnProperty("loaded")) {
+                    info.loaded = config.loaded;
+                }
+                info.version = module.version;
+                return info;
+            }
+        }
+    }
+    return null;
+}
+
+function getFullNodeInfo(typeOrId) {
+    // Used by index.enableNodeSet so that .file can be retrieved to pass
+    // to loader.loadNodeSet
+    var id = typeOrId;
+    if (nodeTypeToId[typeOrId]) {
+        id = nodeTypeToId[typeOrId];
+    }
+    /* istanbul ignore else */
+    if (id) {
+        var module = moduleConfigs[getModule(id)];
+        if (module) {
+            return module.nodes[getNode(id)];
+        }
+    }
+    return null;
+}
+
+function getNodeList(filter) {
+    var list = [];
+    for (var module in moduleConfigs) {
+        /* istanbul ignore else */
+        if (moduleConfigs.hasOwnProperty(module)) {
+            var nodes = moduleConfigs[module].nodes;
+            for (var node in nodes) {
+                /* istanbul ignore else */
+                if (nodes.hasOwnProperty(node)) {
+                    var nodeInfo = filterNodeInfo(nodes[node]);
+                    nodeInfo.version = moduleConfigs[module].version;
+                    if (!filter || filter(nodes[node])) {
+                        list.push(nodeInfo);
+                    }
+                }
+            }
+        }
+    }
+    return list;
+}
+
+function getModuleList() {
+    //var list = [];
+    //for (var module in moduleNodes) {
+    //    /* istanbul ignore else */
+    //    if (moduleNodes.hasOwnProperty(module)) {
+    //        list.push(registry.getModuleInfo(module));
+    //    }
+    //}
+    //return list;
+    return moduleConfigs;
+
+}
+
+function getModuleInfo(module) {
+    if (moduleNodes[module]) {
+        var nodes = moduleNodes[module];
+        var m = {
+            name: module,
+            version: moduleConfigs[module].version,
+            nodes: []
+        };
+        for (var i = 0; i < nodes.length; ++i) {
+            var nodeInfo = filterNodeInfo(moduleConfigs[module].nodes[nodes[i]]);
+            nodeInfo.version = m.version;
+            m.nodes.push(nodeInfo);
+        }
+        return m;
+    } else {
+        return null;
+    }
+}
+
+function registerNodeConstructor(type,constructor) {
+    if (nodeConstructors[type]) {
+        throw new Error(type+" already registered");
+    }
+    //TODO: Ensure type is known - but doing so will break some tests
+    //      that don't have a way to register a node template ahead
+    //      of registering the constructor
+    util.inherits(constructor,Node);
+    nodeConstructors[type] = constructor;
+    events.emit("type-registered",type);
+}
+
+function getAllNodeConfigs(lang) {
+    if (!nodeConfigCache) {
+        var result = "";
+        var script = "";
+        for (var i=0;i<nodeList.length;i++) {
+            var id = nodeList[i];
+            var config = moduleConfigs[getModule(id)].nodes[getNode(id)];
+            if (config.enabled && !config.err) {
+                result += config.config;
+                result += loader.getNodeHelp(config,lang||"en-US")||"";
+                //script += config.script;
+            }
+        }
+        //if (script.length > 0) {
+        //    result += '<script type="text/javascript">';
+        //    result += UglifyJS.minify(script, {fromString: true}).code;
+        //    result += '</script>';
+        //}
+        nodeConfigCache = result;
+    }
+    return nodeConfigCache;
+}
+
+function getNodeConfig(id,lang) {
+    var config = moduleConfigs[getModule(id)];
+    if (!config) {
+        return null;
+    }
+    config = config.nodes[getNode(id)];
+    if (config) {
+        var result = config.config;
+        result += loader.getNodeHelp(config,lang||"en-US")
+
+        //if (config.script) {
+        //    result += '<script type="text/javascript">'+config.script+'</script>';
+        //}
+        return result;
+    } else {
+        return null;
+    }
+}
+
+function getNodeConstructor(type) {
+    var id = nodeTypeToId[type];
+
+    var config;
+    if (typeof id === "undefined") {
+        config = undefined;
+    } else {
+        config = moduleConfigs[getModule(id)].nodes[getNode(id)];
+    }
+
+    if (!config || (config.enabled && !config.err)) {
+        return nodeConstructors[type];
+    }
+    return null;
+}
+
+function clear() {
+    nodeConfigCache = null;
+    moduleConfigs = {};
+    nodeList = [];
+    nodeConstructors = {};
+    nodeTypeToId = {};
+}
+
+function getTypeId(type) {
+    return nodeTypeToId[type];
+}
+
+function enableNodeSet(typeOrId) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+
+    var id = typeOrId;
+    if (nodeTypeToId[typeOrId]) {
+        id = nodeTypeToId[typeOrId];
+    }
+    var config;
+    try {
+        config = moduleConfigs[getModule(id)].nodes[getNode(id)];
+        delete config.err;
+        config.enabled = true;
+        nodeConfigCache = null;
+        return saveNodeList().then(function() {
+            return filterNodeInfo(config);
+        });
+    } catch (err) {
+        throw new Error("Unrecognised id: "+typeOrId);
+    }
+}
+
+function disableNodeSet(typeOrId) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+    var id = typeOrId;
+    if (nodeTypeToId[typeOrId]) {
+        id = nodeTypeToId[typeOrId];
+    }
+    var config;
+    try {
+        config = moduleConfigs[getModule(id)].nodes[getNode(id)];
+        // TODO: persist setting
+        config.enabled = false;
+        nodeConfigCache = null;
+        return saveNodeList().then(function() {
+            return filterNodeInfo(config);
+        });
+    } catch (err) {
+        throw new Error("Unrecognised id: "+id);
+    }
+}
+
+function cleanModuleList() {
+    var removed = false;
+    for (var mod in moduleConfigs) {
+        /* istanbul ignore else */
+        if (moduleConfigs.hasOwnProperty(mod)) {
+            var nodes = moduleConfigs[mod].nodes;
+            var node;
+            if (mod == "node-red") {
+                // For core nodes, look for nodes that are enabled, !loaded and !errored
+                for (node in nodes) {
+                    /* istanbul ignore else */
+                    if (nodes.hasOwnProperty(node)) {
+                        var n = nodes[node];
+                        if (n.enabled && !n.err && !n.loaded) {
+                            removeNode(mod+"/"+node);
+                            removed = true;
+                        }
+                    }
+                }
+            } else {
+                if (moduleConfigs[mod] && !moduleNodes[mod]) {
+                    // For node modules, look for missing ones
+                    for (node in nodes) {
+                        /* istanbul ignore else */
+                        if (nodes.hasOwnProperty(node)) {
+                            removeNode(mod+"/"+node);
+                            removed = true;
+                        }
+                    }
+                    delete moduleConfigs[mod];
+                }
+            }
+        }
+    }
+    if (removed) {
+        saveNodeList();
+    }
+}
+
+var registry = module.exports = {
+    init: init,
+    clear: clear,
+
+    registerNodeConstructor: registerNodeConstructor,
+    getNodeConstructor: getNodeConstructor,
+
+    addNodeSet: addNodeSet,
+    enableNodeSet: enableNodeSet,
+    disableNodeSet: disableNodeSet,
+
+    removeModule: removeModule,
+
+    getNodeInfo: getNodeInfo,
+    getFullNodeInfo: getFullNodeInfo,
+    getNodeList: getNodeList,
+    getModuleList: getModuleList,
+    getModuleInfo: getModuleInfo,
+
+    /**
+     * Gets all of the node template configs
+     * @return all of the node templates in a single string
+     */
+    getAllNodeConfigs: getAllNodeConfigs,
+    getNodeConfig: getNodeConfig,
+
+    getTypeId: getTypeId,
+
+    saveNodeList: saveNodeList,
+
+    cleanModuleList: cleanModuleList
+};

red/runtime/settings.js

@@ -0,0 +1,91 @@
+/**
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require("when");
+var clone = require("clone");
+var assert = require("assert");
+var log = require("./log");
+
+var userSettings = null;
+var globalSettings = null;
+var storage = null;
+
+var persistentSettings = {
+    init: function(settings) {
+        userSettings = settings;
+        for (var i in settings) {
+            /* istanbul ignore else */
+            if (settings.hasOwnProperty(i)) {
+                (function() {
+                    var j = i;
+                    persistentSettings.__defineGetter__(j,function() { return userSettings[j]; });
+                    persistentSettings.__defineSetter__(j,function() { throw new Error("Property '"+j+"' is read-only"); });
+                })();
+            }
+        }
+        globalSettings = null;
+    },
+    load: function(_storage) {
+        storage = _storage;
+        return storage.getSettings().then(function(_settings) {
+            globalSettings = _settings;
+        });
+    },
+    get: function(prop) {
+        if (userSettings.hasOwnProperty(prop)) {
+            return clone(userSettings[prop]);
+        }
+        if (globalSettings === null) {
+            throw new Error(log._("settings.not-available"));
+        }
+        return clone(globalSettings[prop]);
+    },
+
+    set: function(prop,value) {
+        if (userSettings.hasOwnProperty(prop)) {
+            throw new Error(log._("settings.property-read-only", {prop:prop}));
+        }
+        if (globalSettings === null) {
+            throw new Error(log._("settings.not-available"));
+        }
+        var current = globalSettings[prop];
+        globalSettings[prop] = value;
+        try {
+            assert.deepEqual(current,value);
+            return when.resolve();
+        } catch(err) {
+            return storage.saveSettings(globalSettings);
+        }
+    },
+
+    available: function() {
+        return (globalSettings !== null);
+    },
+
+    reset: function() {
+        for (var i in userSettings) {
+            /* istanbul ignore else */
+            if (userSettings.hasOwnProperty(i)) {
+                delete persistentSettings[i];
+            }
+        }
+        userSettings = null;
+        globalSettings = null;
+        storage = null;
+    }
+}
+
+module.exports = persistentSettings;

red/runtime/storage/index.js

@@ -0,0 +1,191 @@
+/**
+ * Copyright 2013, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var when = require('when');
+var Path = require('path');
+var log = require("../log");
+
+var storageModule;
+var settingsAvailable;
+var sessionsAvailable;
+
+function moduleSelector(aSettings) {
+    var toReturn;
+    if (aSettings.storageModule) {
+        if (typeof aSettings.storageModule === "string") {
+            // TODO: allow storage modules to be specified by absolute path
+            toReturn = require("./"+aSettings.storageModule);
+        } else {
+            toReturn = aSettings.storageModule;
+        }
+    } else {
+        toReturn = require("./localfilesystem");
+    }
+    return toReturn;
+}
+
+function is_malicious(path) {
+    return path.indexOf('../') != -1 || path.indexOf('..\\') != -1;
+}
+
+var storageModuleInterface = {
+        init: function(settings) {
+            try {
+                storageModule = moduleSelector(settings);
+                settingsAvailable = storageModule.hasOwnProperty("getSettings") && storageModule.hasOwnProperty("saveSettings");
+                sessionsAvailable = storageModule.hasOwnProperty("getSessions") && storageModule.hasOwnProperty("saveSessions");
+            } catch (e) {
+                return when.reject(e);
+            }
+            return storageModule.init(settings);
+        },
+        getFlows: function() {
+            return storageModule.getFlows();
+        },
+        saveFlows: function(flows) {
+            return storageModule.saveFlows(flows);
+        },
+        getCredentials: function() {
+            return storageModule.getCredentials();
+        },
+        saveCredentials: function(credentials) {
+            return storageModule.saveCredentials(credentials);
+        },
+        getSettings: function() {
+            if (settingsAvailable) {
+                return storageModule.getSettings();
+            } else {
+                return when.resolve(null);
+            }
+        },
+        saveSettings: function(settings) {
+            if (settingsAvailable) {
+                return storageModule.saveSettings(settings);
+            } else {
+                return when.resolve();
+            }
+        },
+        getSessions: function() {
+            if (sessionsAvailable) {
+                return storageModule.getSessions();
+            } else {
+                return when.resolve(null);
+            }
+        },
+        saveSessions: function(sessions) {
+            if (sessionsAvailable) {
+                return storageModule.saveSessions(sessions);
+            } else {
+                return when.resolve();
+            }
+        },
+
+        /* Library Functions */
+
+        getLibraryEntry: function(type, path) {
+            if (is_malicious(path)) {
+                var err = new Error();
+                err.code = "forbidden";
+                return when.reject(err);
+            }
+            return storageModule.getLibraryEntry(type, path);
+        },
+        saveLibraryEntry: function(type, path, meta, body) {
+            if (is_malicious(path)) {
+                var err = new Error();
+                err.code = "forbidden";
+                return when.reject(err);
+            }
+            return storageModule.saveLibraryEntry(type, path, meta, body);
+        },
+
+/* Deprecated functions */
+        getAllFlows: function() {
+            if (storageModule.hasOwnProperty("getAllFlows")) {
+                return storageModule.getAllFlows();
+            } else {
+                return listFlows("/");
+            }
+        },
+        getFlow: function(fn) {
+            if (is_malicious(fn)) {
+                var err = new Error();
+                err.code = "forbidden";
+                return when.reject(err);
+            }
+            if (storageModule.hasOwnProperty("getFlow")) {
+                return storageModule.getFlow(fn);
+            } else {
+                return storageModule.getLibraryEntry("flows",fn);
+            }
+
+        },
+        saveFlow: function(fn, data) {
+            if (is_malicious(fn)) {
+                var err = new Error();
+                err.code = "forbidden";
+                return when.reject(err);
+            }
+            if (storageModule.hasOwnProperty("saveFlow")) {
+                return storageModule.saveFlow(fn, data);
+            } else {
+                return storageModule.saveLibraryEntry("flows",fn,{},data);
+            }
+        }
+/* End deprecated functions */
+
+}
+
+
+function listFlows(path) {
+    return storageModule.getLibraryEntry("flows",path).then(function(res) {
+        return when.promise(function(resolve) {
+            var promises = [];
+            res.forEach(function(r) {
+                if (typeof r === "string") {
+                    promises.push(listFlows(Path.join(path,r)));
+                } else {
+                    promises.push(when.resolve(r));
+                }
+            });
+            var i=0;
+            when.settle(promises).then(function(res2) {
+                var result = {};
+                res2.forEach(function(r) {
+                    // TODO: name||fn
+                    if (r.value.fn) {
+                        var name = r.value.name;
+                        if (!name) {
+                            name = r.value.fn.split(".")[0];
+                        }
+                        result.f = result.f || [];
+                        result.f.push(name);
+                    } else {
+                        result.d = result.d || {};
+                        result.d[res[i]] = r.value;
+                        //console.log(">",r.value);
+                    }
+                    i++;
+                });
+                resolve(result);
+            });
+        });
+    });
+}
+
+
+
+module.exports = storageModuleInterface;

red/runtime/storage/localfilesystem.js

@@ -0,0 +1,372 @@
+/**
+ * Copyright 2013, 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var fs = require('fs-extra');
+var when = require('when');
+var nodeFn = require('when/node/function');
+var keys = require('when/keys');
+var fspath = require("path");
+var mkdirp = fs.mkdirs;
+
+var log = require("../log");
+
+var promiseDir = nodeFn.lift(mkdirp);
+
+var settings;
+var flowsFile;
+var flowsFullPath;
+var flowsFileBackup;
+var credentialsFile;
+var credentialsFileBackup;
+var oldCredentialsFile;
+var sessionsFile;
+var libDir;
+var libFlowsDir;
+var globalSettingsFile;
+
+function getFileMeta(root,path) {
+    var fn = fspath.join(root,path);
+    var fd = fs.openSync(fn,"r");
+    var size = fs.fstatSync(fd).size;
+    var meta = {};
+    var read = 0;
+    var length = 10;
+    var remaining = "";
+    var buffer = Buffer(length);
+    while(read < size) {
+        read+=fs.readSync(fd,buffer,0,length);
+        var data = remaining+buffer.toString();
+        var parts = data.split("\n");
+        remaining = parts.splice(-1);
+        for (var i=0;i<parts.length;i+=1) {
+            var match = /^\/\/ (\w+): (.*)/.exec(parts[i]);
+            if (match) {
+                meta[match[1]] = match[2];
+            } else {
+                read = size;
+                break;
+            }
+        }
+    }
+    fs.closeSync(fd);
+    return meta;
+}
+
+function getFileBody(root,path) {
+    var body = "";
+    var fn = fspath.join(root,path);
+    var fd = fs.openSync(fn,"r");
+    var size = fs.fstatSync(fd).size;
+    var scanning = true;
+    var read = 0;
+    var length = 50;
+    var remaining = "";
+    var buffer = Buffer(length);
+    while(read < size) {
+        var thisRead = fs.readSync(fd,buffer,0,length);
+        read += thisRead;
+        if (scanning) {
+            var data = remaining+buffer.slice(0,thisRead).toString();
+            var parts = data.split("\n");
+            remaining = parts.splice(-1)[0];
+            for (var i=0;i<parts.length;i+=1) {
+                if (! /^\/\/ \w+: /.test(parts[i])) {
+                    scanning = false;
+                    body += parts[i]+"\n";
+                }
+            }
+            if (! /^\/\/ \w+: /.test(remaining)) {
+                scanning = false;
+            }
+            if (!scanning) {
+                body += remaining;
+            }
+        } else {
+            body += buffer.slice(0,thisRead).toString();
+        }
+    }
+    fs.closeSync(fd);
+    return body;
+}
+
+/**
+ * Write content to a file using UTF8 encoding.
+ * This forces a fsync before completing to ensure
+ * the write hits disk.
+ */
+function writeFile(path,content) {
+    return when.promise(function(resolve,reject) {
+        var stream = fs.createWriteStream(path);
+        stream.on('open',function(fd) {
+            stream.end(content,'utf8',function() {
+                fs.fsync(fd,resolve);
+            });
+        });
+        stream.on('error',function(err) {
+            reject(err);
+        });
+    });
+}
+
+var localfilesystem = {
+    init: function(_settings) {
+        settings = _settings;
+
+        var promises = [];
+
+        if (!settings.userDir) {
+            if (fs.existsSync(fspath.join(process.env.NODE_RED_HOME,".config.json"))) {
+                settings.userDir = process.env.NODE_RED_HOME;
+            } else {
+                settings.userDir = fspath.join(process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE || process.env.NODE_RED_HOME,".node-red");
+                if (!settings.readOnly) {
+                    promises.push(promiseDir(settings.userDir));
+                }
+            }
+        }
+
+        if (settings.flowFile) {
+            flowsFile = settings.flowFile;
+            // handle Unix and Windows "C:\"
+            if ((flowsFile[0] == "/") || (flowsFile[1] == ":")) {
+                // Absolute path
+                flowsFullPath = flowsFile;
+            } else if (flowsFile.substring(0,2) === "./") {
+                // Relative to cwd
+                flowsFullPath = fspath.join(process.cwd(),flowsFile);
+            } else {
+                if (fs.existsSync(fspath.join(process.cwd(),flowsFile))) {
+                    // Found in cwd
+                    flowsFullPath = fspath.join(process.cwd(),flowsFile);
+                } else {
+                    // Use userDir
+                    flowsFullPath = fspath.join(settings.userDir,flowsFile);
+                }
+            }
+
+        } else {
+            flowsFile = 'flows_'+require('os').hostname()+'.json';
+            flowsFullPath = fspath.join(settings.userDir,flowsFile);
+        }
+        var ffExt = fspath.extname(flowsFullPath);
+        var ffName = fspath.basename(flowsFullPath);
+        var ffBase = fspath.basename(flowsFullPath,ffExt);
+        var ffDir = fspath.dirname(flowsFullPath);
+
+        credentialsFile = fspath.join(settings.userDir,ffBase+"_cred"+ffExt);
+        credentialsFileBackup = fspath.join(settings.userDir,"."+ffBase+"_cred"+ffExt+".backup");
+
+        oldCredentialsFile = fspath.join(settings.userDir,"credentials.json");
+
+        flowsFileBackup = fspath.join(ffDir,"."+ffName+".backup");
+
+        sessionsFile = fspath.join(settings.userDir,".sessions.json");
+
+        libDir = fspath.join(settings.userDir,"lib");
+        libFlowsDir = fspath.join(libDir,"flows");
+
+        globalSettingsFile = fspath.join(settings.userDir,".config.json");
+
+        if (!settings.readOnly) {
+            promises.push(promiseDir(libFlowsDir));
+        }
+
+        return when.all(promises);
+    },
+
+    getFlows: function() {
+        return when.promise(function(resolve) {
+            log.info(log._("storage.localfilesystem.user-dir",{path:settings.userDir}));
+            log.info(log._("storage.localfilesystem.flows-file",{path:flowsFullPath}));
+            fs.exists(flowsFullPath, function(exists) {
+                if (exists) {
+                    resolve(nodeFn.call(fs.readFile,flowsFullPath,'utf8').then(function(data) {
+                        return JSON.parse(data);
+                    }));
+                } else {
+                    log.info(log._("storage.localfilesystem.create"));
+                    resolve([]);
+                }
+            });
+        });
+    },
+
+    saveFlows: function(flows) {
+        if (settings.readOnly) {
+            return when.resolve();
+        }
+
+        if (fs.existsSync(flowsFullPath)) {
+            fs.renameSync(flowsFullPath,flowsFileBackup);
+        }
+
+        var flowData;
+
+        if (settings.flowFilePretty) {
+            flowData = JSON.stringify(flows,null,4);
+        } else {
+            flowData = JSON.stringify(flows);
+        }
+        return writeFile(flowsFullPath, flowData);
+    },
+
+    getCredentials: function() {
+        return when.promise(function(resolve) {
+            fs.exists(credentialsFile, function(exists) {
+                if (exists) {
+                    resolve(nodeFn.call(fs.readFile, credentialsFile, 'utf8').then(function(data) {
+                        return JSON.parse(data)
+                    }));
+                } else {
+                    fs.exists(oldCredentialsFile, function(exists) {
+                        if (exists) {
+                            resolve(nodeFn.call(fs.readFile, oldCredentialsFile, 'utf8').then(function(data) {
+                                return JSON.parse(data)
+                            }));
+                        } else {
+                            resolve({});
+                        }
+                    });
+                }
+            });
+        });
+    },
+
+    saveCredentials: function(credentials) {
+        if (settings.readOnly) {
+            return when.resolve();
+        }
+
+        if (fs.existsSync(credentialsFile)) {
+            fs.renameSync(credentialsFile,credentialsFileBackup);
+        }
+        var credentialData;
+        if (settings.flowFilePretty) {
+            credentialData = JSON.stringify(credentials,null,4);
+        } else {
+            credentialData = JSON.stringify(credentials);
+        }
+        return writeFile(credentialsFile, credentialData);
+    },
+
+    getSettings: function() {
+        if (fs.existsSync(globalSettingsFile)) {
+            return nodeFn.call(fs.readFile,globalSettingsFile,'utf8').then(function(data) {
+                if (data) {
+                    try {
+                        return JSON.parse(data);
+                    } catch(err) {
+                        log.trace("Corrupted config detected - resetting");
+                        return {};
+                    }
+                } else {
+                    return {};
+                }
+            });
+        }
+        return when.resolve({});
+    },
+    saveSettings: function(settings) {
+        if (settings.readOnly) {
+            return when.resolve();
+        }
+        return writeFile(globalSettingsFile,JSON.stringify(settings,null,1));
+    },
+    getSessions: function() {
+        if (fs.existsSync(sessionsFile)) {
+            return nodeFn.call(fs.readFile,sessionsFile,'utf8').then(function(data) {
+                if (data) {
+                    try {
+                        return JSON.parse(data);
+                    } catch(err) {
+                        log.trace("Corrupted sessions file - resetting");
+                        return {};
+                    }
+                } else {
+                    return {};
+                }
+            });
+        }
+        return when.resolve({});
+    },
+    saveSessions: function(sessions) {
+        if (settings.readOnly) {
+            return when.resolve();
+        }
+        return writeFile(sessionsFile,JSON.stringify(sessions));
+    },
+
+    getLibraryEntry: function(type,path) {
+        var root = fspath.join(libDir,type);
+        var rootPath = fspath.join(libDir,type,path);
+        return promiseDir(root).then(function () {
+            return nodeFn.call(fs.lstat, rootPath).then(function(stats) {
+                if (stats.isFile()) {
+                    return getFileBody(root,path);
+                }
+                if (path.substr(-1) == '/') {
+                    path = path.substr(0,path.length-1);
+                }
+                return nodeFn.call(fs.readdir, rootPath).then(function(fns) {
+                    var dirs = [];
+                    var files = [];
+                    fns.sort().filter(function(fn) {
+                        var fullPath = fspath.join(path,fn);
+                        var absoluteFullPath = fspath.join(root,fullPath);
+                        if (fn[0] != ".") {
+                            var stats = fs.lstatSync(absoluteFullPath);
+                            if (stats.isDirectory()) {
+                                dirs.push(fn);
+                            } else {
+                                var meta = getFileMeta(root,fullPath);
+                                meta.fn = fn;
+                                files.push(meta);
+                            }
+                        }
+                    });
+                    return dirs.concat(files);
+                });
+            }).otherwise(function(err) {
+                if (type === "flows" && !/\.json$/.test(path)) {
+                    return localfilesystem.getLibraryEntry(type,path+".json")
+                        .otherwise(function(e) {
+                            throw err;
+                        });
+                } else {
+                    throw err;
+                }
+            });
+        });
+    },
+
+    saveLibraryEntry: function(type,path,meta,body) {
+        if (settings.readOnly) {
+            return when.resolve();
+        }
+        var fn = fspath.join(libDir, type, path);
+        var headers = "";
+        for (var i in meta) {
+            if (meta.hasOwnProperty(i)) {
+                headers += "// "+i+": "+meta[i]+"\n";
+            }
+        }
+        return promiseDir(fspath.dirname(fn)).then(function () {
+            writeFile(fn,headers+body);
+        });
+    }
+};
+
+module.exports = localfilesystem;

red/runtime/util.js

@@ -0,0 +1,112 @@
+/**
+ * Copyright 2014, 2015 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var clone = require("clone");
+
+function generateId() {
+    return (1+Math.random()*4294967295).toString(16);
+}
+
+function ensureString(o) {
+    if (Buffer.isBuffer(o)) {
+        return o.toString();
+    } else if (typeof o === "object") {
+        return JSON.stringify(o);
+    } else if (typeof o === "string") {
+        return o;
+    }
+    return ""+o;
+}
+
+function ensureBuffer(o) {
+    if (Buffer.isBuffer(o)) {
+        return o;
+    } else if (typeof o === "object") {
+        o = JSON.stringify(o);
+    } else if (typeof o !== "string") {
+        o = ""+o;
+    }
+    return new Buffer(o);
+}
+
+function cloneMessage(msg) {
+    // Temporary fix for #97
+    // TODO: remove this http-node-specific fix somehow
+    var req = msg.req;
+    var res = msg.res;
+    delete msg.req;
+    delete msg.res;
+    var m = clone(msg);
+    if (req) {
+        m.req = req;
+        msg.req = req;
+    }
+    if (res) {
+        m.res = res;
+        msg.res = res;
+    }
+    return m;
+}
+
+function compareObjects(obj1,obj2) {
+    if (obj1 === obj2) {
+        return true;
+    }
+    if (obj1 == null || obj2 == null) {
+        return false;
+    }
+    if (!(obj1 instanceof Object) && !(obj2 instanceof Object)) {
+        return false;
+    }
+    var isArray1 = Array.isArray(obj1);
+    var isArray2 = Array.isArray(obj2);
+    if (isArray1 != isArray2) {
+        return false;
+    }
+    if (isArray1 && isArray2) {
+        if (obj1.length != obj2.length) {
+            return false;
+        }
+        for (var i=0;i<obj1.length;i++) {
+            if (!compareObjects(obj1[i],obj2[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+    var keys1 = Object.keys(obj1);
+    var keys2 = Object.keys(obj2);
+    if (keys1.length != keys2.length) {
+        return false;
+    }
+    for (var k in obj1) {
+        /* istanbul ignore else */
+        if (obj1.hasOwnProperty(k)) {
+            if (!compareObjects(obj1[k],obj2[k])) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+module.exports = {
+    ensureString: ensureString,
+    ensureBuffer: ensureBuffer,
+    cloneMessage: cloneMessage,
+    compareObjects: compareObjects,
+    generateId: generateId
+};