Update for 0.19.1

CHANGELOG.md

@@ -1,3 +1,12 @@
+#### 0.19.1: Maintenance Release
+
+ - Pull in latest twitter node
+ - Handle windows paths for context storage
+ - Handle persisting objects with circular refs in context
+ - Ensure js editor can expand to fill available space
+ - Add example localfilesystem contextStorage to settings
+ - Fix template node handling of nested context tags
+
 #### 0.19: Milestone Release
 
 Editor

editor/js/ui/editors/js.js

@@ -16,7 +16,7 @@
 RED.editor.types._js = (function() {
 
 
-    var template = '<script type="text/x-red" data-template-name="_js"><div class="form-row node-text-editor-row" style="width: 700px"><div style="height: 200px;min-height: 150px;" class="node-text-editor" id="node-input-js"></div></div></script>';
+    var template = '<script type="text/x-red" data-template-name="_js"><div class="form-row node-text-editor-row"><div style="height: 200px;min-height: 150px;" class="node-text-editor" id="node-input-js"></div></div></script>';
 
     return {
         init: function() {
@@ -58,7 +58,6 @@ RED.editor.types._js = (function() {
                     for (var i=0;i<rows.size();i++) {
                         height -= $(rows[i]).outerHeight(true);
                     }
-                    height -= (parseInt($("#dialog-form").css("marginTop"))+parseInt($("#dialog-form").css("marginBottom")));
                     $(".node-text-editor").css("height",height+"px");
                     expressionEditor.resize();
                 },

nodes/core/core/80-template.js

@@ -19,6 +19,19 @@ module.exports = function(RED) {
     var mustache = require("mustache");
     var yaml = require("js-yaml");
 
+    function extractTokens(tokens,set) {
+        set = set || new Set();
+        tokens.forEach(function(token) {
+            if (token[0] !== 'text') {
+                set.add(token[1]);
+                if (token.length > 4) {
+                    extractTokens(token[4],set);
+                }
+            }
+        });
+        return set;
+    }
+
     function parseContext(key) {
         var match = /^(flow|global)(\[(\w+)\])?\.(.+)/.exec(key);
         if (match) {
@@ -36,22 +49,16 @@ module.exports = function(RED) {
      * flow and global context
      */
 
-    function NodeContext(msg, nodeContext, parent, escapeStrings, promises, results) {
+    function NodeContext(msg, nodeContext, parent, escapeStrings, cachedContextTokens) {
         this.msgContext = new mustache.Context(msg,parent);
         this.nodeContext = nodeContext;
         this.escapeStrings = escapeStrings;
-        this.promises = promises;
-        this.results = results;
+        this.cachedContextTokens = cachedContextTokens;
     }
 
     NodeContext.prototype = new mustache.Context();
 
     NodeContext.prototype.lookup = function (name) {
-        var results = this.results;
-        if (results) {
-            var val = results.shift();
-            return val;
-        }
         // try message first:
         try {
             var value = this.msgContext.lookup(name);
@@ -64,7 +71,6 @@ module.exports = function(RED) {
                     value = value.replace(/\f/g, "\\f");
                     value = value.replace(/[\b]/g, "\\b");
                 }
-                this.promises.push(Promise.resolve(value));
                 return value;
             }
 
@@ -76,28 +82,10 @@ module.exports = function(RED) {
                 var field = context.field;
                 var target = this.nodeContext[type];
                 if (target) {
-                    var promise = new Promise((resolve, reject) => {
-                        var callback = (err, val) => {
-                            if (err) {
-                                reject(err);
-                            } else {
-                                resolve(val);
-                            }
-                        };
-                        target.get(field, store, callback);
-                    });
-                    this.promises.push(promise);
-                    return '';
-                }
-                else {
-                    this.promises.push(Promise.resolve(''));
-                    return '';
+                    return this.cachedContextTokens[name];
                 }
             }
-            else {
-                this.promises.push(Promise.resolve(''));
-                return '';
-            }
+            return '';
         }
         catch(err) {
             throw err;
@@ -105,7 +93,7 @@ module.exports = function(RED) {
     }
 
     NodeContext.prototype.push = function push (view) {
-        return new NodeContext(view, this.nodeContext, this.msgContext, undefined, this.promises, this.results);
+        return new NodeContext(view, this.nodeContext, this.msgContext, undefined, this.cachedContextTokens);
     };
 
     function TemplateNode(n) {
@@ -118,33 +106,36 @@ module.exports = function(RED) {
         this.outputFormat = n.output || "str";
 
         var node = this;
-        node.on("input", function(msg) {
-            function output(value) {
-                /* istanbul ignore else  */
-                if (node.outputFormat === "json") {
-                    value = JSON.parse(value);
-                }
-                /* istanbul ignore else  */
-                if (node.outputFormat === "yaml") {
-                    value = yaml.load(value);
-                }
 
-                if (node.fieldType === 'msg') {
-                    RED.util.setMessageProperty(msg, node.field, value);
-                    node.send(msg);
-                } else if ((node.fieldType === 'flow') ||
-                           (node.fieldType === 'global')) {
-                    var context = RED.util.parseContextStore(node.field);
-                    var target = node.context()[node.fieldType];
-                    target.set(context.key, value, context.store, function (err) {
-                        if (err) {
-                            node.error(err, msg);
-                        } else {
-                            node.send(msg);
-                        }
-                    });
-                }
+        function output(msg,value) {
+            /* istanbul ignore else  */
+            if (node.outputFormat === "json") {
+                value = JSON.parse(value);
             }
+            /* istanbul ignore else  */
+            if (node.outputFormat === "yaml") {
+                value = yaml.load(value);
+            }
+
+            if (node.fieldType === 'msg') {
+                RED.util.setMessageProperty(msg, node.field, value);
+                node.send(msg);
+            } else if ((node.fieldType === 'flow') ||
+                       (node.fieldType === 'global')) {
+                var context = RED.util.parseContextStore(node.field);
+                var target = node.context()[node.fieldType];
+                target.set(context.key, value, context.store, function (err) {
+                    if (err) {
+                        node.error(err, msg);
+                    } else {
+                        node.send(msg);
+                    }
+                });
+            }
+        }
+
+        node.on("input", function(msg) {
+
             try {
                 /***
                 * Allow template contents to be defined externally
@@ -160,19 +151,44 @@ module.exports = function(RED) {
                 if (node.syntax === "mustache") {
                     var is_json = (node.outputFormat === "json");
                     var promises = [];
-                    mustache.render(template, new NodeContext(msg, node.context(), null, is_json, promises, null));
-                    Promise.all(promises).then(function (values) {
-                        var value = mustache.render(template, new NodeContext(msg, node.context(), null, is_json, null, values));
-                        output(value);
+                    var tokens = extractTokens(mustache.parse(template));
+                    var resolvedTokens = {};
+                    tokens.forEach(function(name) {
+                        var context = parseContext(name);
+                        if (context) {
+                            var type = context.type;
+                            var store = context.store;
+                            var field = context.field;
+                            var target = node.context()[type];
+                            if (target) {
+                                var promise = new Promise((resolve, reject) => {
+                                    target.get(field, store, (err, val) => {
+                                        if (err) {
+                                            reject(err);
+                                        } else {
+                                            resolvedTokens[name] = val;
+                                            resolve();
+                                        }
+                                    });
+                                });
+                                promises.push(promise);
+                                return;
+                            }
+                        }
+                    });
+
+                    Promise.all(promises).then(function() {
+                        var value = mustache.render(template, new NodeContext(msg, node.context(), null, is_json, resolvedTokens));
+                        output(msg, value);
                     }).catch(function (err) {
-                        node.error(err.message);
+                        node.error(err.message,msg);
                     });
                 } else {
-                    output(template);
+                    output(msg, template);
                 }
             }
             catch(err) {
-                node.error(err.message);
+                node.error(err.message, msg);
             }
         });
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-red",
-  "version": "0.19.0",
+  "version": "0.19.1",
   "description": "A visual tool for wiring the Internet of Things",
   "homepage": "http://nodered.org",
   "license": "Apache-2.0",
@@ -62,7 +62,7 @@
     "node-red-node-email": "0.1.*",
     "node-red-node-feedparser": "^0.1.12",
     "node-red-node-rbe": "0.2.*",
-    "node-red-node-twitter": "*",
+    "node-red-node-twitter": "^1.1.0",
     "nopt": "4.0.1",
     "oauth2orize": "1.11.0",
     "on-headers": "1.0.1",

red/runtime/nodes/context/localfilesystem.js

@@ -48,9 +48,12 @@
 var fs = require('fs-extra');
 var path = require("path");
 var util = require("../../util");
+var log = require("../../log");
 
+var safeJSONStringify = require("json-stringify-safe");
 var MemoryStore = require("./memory");
 
+
 function getStoragePath(storageBaseDir, scope) {
     if(scope.indexOf(":") === -1){
         if(scope === "global"){
@@ -118,6 +121,12 @@ function listFiles(storagePath) {
     }).then(dirs => dirs.reduce((acc, val) => acc.concat(val), []));
 }
 
+function stringify(value) {
+    var hasCircular;
+    var result = safeJSONStringify(value,null,4,function(k,v){hasCircular = true})
+    return { json: result, circular: hasCircular };
+}
+
 function LocalFileSystem(config){
     this.config = config;
     this.storageBaseDir = getBasePath(this.config);
@@ -125,6 +134,8 @@ function LocalFileSystem(config){
         this.cache = MemoryStore({});
     }
     this.pendingWrites = {};
+    this.knownCircularRefs = {};
+
     if (config.hasOwnProperty('flushInterval')) {
         this.flushInterval = Math.max(0,config.flushInterval) * 1000;
     } else {
@@ -139,7 +150,7 @@ LocalFileSystem.prototype.open = function(){
         var promises = [];
         return listFiles(self.storageBaseDir).then(function(files) {
             files.forEach(function(file) {
-                var parts = file.split("/");
+                var parts = file.split(path.sep);
                 if (parts[0] === 'global') {
                     scopes.push("global");
                 } else if (parts[1] === 'flow.json') {
@@ -172,7 +183,15 @@ LocalFileSystem.prototype.open = function(){
                 scopes.forEach(function(scope) {
                     var storagePath = getStoragePath(self.storageBaseDir,scope);
                     var context = newContext[scope];
-                    promises.push(fs.outputFile(storagePath + ".json", JSON.stringify(context, undefined, 4), "utf8"));
+                    var stringifiedContext = stringify(context);
+                    if (stringifiedContext.circular && !self.knownCircularRefs[scope]) {
+                        log.warn("Context "+scope+" contains a circular reference that cannot be persisted");
+                        self.knownCircularRefs[scope] = true;
+                    } else {
+                        delete self.knownCircularRefs[scope];
+                    }
+                    log.debug("Flushing localfilesystem context scope "+scope);
+                    promises.push(fs.outputFile(storagePath + ".json", stringifiedContext.json, "utf8"));
                 });
                 delete self._pendingWriteTimeout;
                 return Promise.all(promises);
@@ -221,6 +240,7 @@ LocalFileSystem.prototype.get = function(scope, key, callback) {
 };
 
 LocalFileSystem.prototype.set = function(scope, key, value, callback) {
+    var self = this;
     var storagePath = getStoragePath(this.storageBaseDir ,scope);
     if (this.cache) {
         this.cache.set(scope,key,value,callback);
@@ -229,7 +249,6 @@ LocalFileSystem.prototype.set = function(scope, key, value, callback) {
             // there's a pending write which will handle this
             return;
         } else {
-            var self = this;
             this._pendingWriteTimeout = setTimeout(function() { self._flushPendingWrites.call(self)}, this.flushInterval);
         }
     } else if (callback && typeof callback !== 'function') {
@@ -251,7 +270,14 @@ LocalFileSystem.prototype.set = function(scope, key, value, callback) {
                 }
                 util.setObjectProperty(obj,key[i],v);
             }
-            return fs.outputFile(storagePath + ".json", JSON.stringify(obj, undefined, 4), "utf8");
+            var stringifiedContext = stringify(obj);
+            if (stringifiedContext.circular && !self.knownCircularRefs[scope]) {
+                log.warn("Context "+scope+" contains a circular reference that cannot be persisted");
+                self.knownCircularRefs[scope] = true;
+            } else {
+                delete self.knownCircularRefs[scope];
+            }
+            return fs.outputFile(storagePath + ".json", stringifiedContext.json, "utf8");
         }).then(function(){
             if(typeof callback === "function"){
                 callback(null);
@@ -308,10 +334,11 @@ LocalFileSystem.prototype.clean = function(_activeNodes) {
     } else {
         cachePromise = Promise.resolve();
     }
+    this.knownCircularRefs = {};
     return cachePromise.then(() => listFiles(self.storageBaseDir)).then(function(files) {
         var promises = [];
         files.forEach(function(file) {
-            var parts = file.split("/");
+            var parts = file.split(path.sep);
             var removePromise;
             if (parts[0] === 'global') {
                 // never clean global

settings.js

@@ -213,6 +213,17 @@ module.exports = {
         // j5board:require("johnny-five").Board({repl:false})
     },
 
+    // Context Storage
+    // The following property can be used to enable context storage. The configuration
+    // provided here will enable file-based context that flushes to disk every 30 seconds.
+    // Refer to the documentation for further options: https://nodered.org/docs/api/context/
+    //
+    //contextStorage: {
+    //    default: {
+    //        module:"localfilesystem"
+    //    },
+    //},
+
     // The following property can be used to order the categories in the editor
     // palette. If a node's category is not in the list, the category will get
     // added to the end of the palette.
@@ -245,5 +256,5 @@ module.exports = {
             // To enable the Projects feature, set this value to true
             enabled: false
         }
-    }
+    },
 }

test/nodes/core/core/80-template_spec.js

@@ -177,6 +177,53 @@ describe('template node', function() {
         });
     });
 
+    it('should handle nested context tags - property not set', function(done) {
+        // This comes from the Coursera Node-RED course and is a good example of
+        // multiple conditional tags
+        var template = `{{#flow.time}}time={{flow.time}}{{/flow.time}}{{^flow.time}}!time{{/flow.time}}{{#flow.random}}random={{flow.random}}randomtime={{flow.randomtime}}{{/flow.random}}{{^flow.random}}!random{{/flow.random}}`;
+        var flow = [{id:"n1",z:"t1", type:"template", field:"payload", template:template,wires:[["n2"]]},{id:"n2",z:"t1",type:"helper"}];
+        helper.load(templateNode, flow, function() {
+            initContext(function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('topic', 'bar');
+                        msg.should.have.property('payload', '!time!random');
+                        done();
+                    } catch(err) {
+                        done(err);
+                    }
+                });
+                n1.receive({payload:"foo",topic: "bar"});
+            });
+        });
+    })
+    it('should handle nested context tags - property set', function(done) {
+        // This comes from the Coursera Node-RED course and is a good example of
+        // multiple conditional tags
+        var template = `{{#flow.time}}time={{flow.time}}{{/flow.time}}{{^flow.time}}!time{{/flow.time}}{{#flow.random}}random={{flow.random}}randomtime={{flow.randomtime}}{{/flow.random}}{{^flow.random}}!random{{/flow.random}}`;
+        var flow = [{id:"n1",z:"t1", type:"template", field:"payload", template:template,wires:[["n2"]]},{id:"n2",z:"t1",type:"helper"}];
+        helper.load(templateNode, flow, function() {
+            initContext(function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('topic', 'bar');
+                        msg.should.have.property('payload', 'time=123random=456randomtime=789');
+                        done();
+                    } catch(err) {
+                        done(err);
+                    }
+                });
+                n1.context().flow.set(["time","random","randomtime"],["123","456","789"],function (err) {
+                    n1.receive({payload:"foo",topic: "bar"});
+                });
+            });
+        });
+    })
+
     it('should modify payload from two persistable flow context', function(done) {
         var flow = [{id:"n1",z:"t1", type:"template", field:"payload", template:"payload={{flow[memory1].value}}/{{flow[memory2].value}}",wires:[["n2"]]},{id:"n2",z:"t1",type:"helper"}];
         helper.load(templateNode, flow, function() {
@@ -429,7 +476,7 @@ describe('template node', function() {
             n1.receive({payload:{A:"abc"}});
         });
     });
-    
+
     it('should raise error if passed bad template', function(done) {
         var flow = [{id:"n1", type:"template", field: "payload", template: "payload={{payload",wires:[["n2"]]},{id:"n2",type:"helper"}];
         helper.load(templateNode, flow, function() {

test/red/runtime/nodes/context/localfilesystem_spec.js

@@ -46,9 +46,12 @@ describe('localfilesystem',function() {
 
         it('should store property',function(done) {
             context.get("nodeX","foo",function(err, value){
+                if (err) { return done(err); }
                 should.not.exist(value);
                 context.set("nodeX","foo","test",function(err){
+                    if (err) { return done(err); }
                     context.get("nodeX","foo",function(err, value){
+                        if (err) { return done(err); }
                         value.should.be.equal("test");
                         done();
                     });