clone msg for every single line output

(this may not be desired)
Add tests to ensure properties go trhough

Gruntfile.js

@@ -16,6 +16,7 @@
 
 var path = require("path");
 var fs = require("fs-extra");
+var sass = require("node-sass");
 
 module.exports = function(grunt) {
 
@@ -220,6 +221,7 @@ module.exports = function(grunt) {
         sass: {
             build: {
                 options: {
+                    implementation: sass,
                     outputStyle: 'compressed'
                 },
                 files: [{

package.json

@@ -41,13 +41,13 @@
         "fs-extra": "8.1.0",
         "fs.notify": "0.0.4",
         "hash-sum": "2.0.0",
-        "https-proxy-agent": "2.2.2",
+        "https-proxy-agent": "2.2.3",
         "i18next": "15.1.2",
         "iconv-lite": "0.5.0",
         "is-utf8": "0.2.1",
         "js-yaml": "3.13.1",
         "json-stringify-safe": "5.0.1",
-        "jsonata": "1.6.5",
+        "jsonata": "1.7.0",
         "media-typer": "1.1.0",
         "memorystore": "1.6.1",
         "mime": "2.4.4",
@@ -93,19 +93,20 @@
         "grunt-mocha-istanbul": "5.0.2",
         "grunt-nodemon": "~0.4.2",
         "grunt-npm-command": "~0.1.2",
-        "grunt-sass": "~2.0.0",
+        "grunt-sass": "~3.1.0",
         "grunt-simple-mocha": "~0.4.1",
         "http-proxy": "^1.16.2",
         "istanbul": "0.4.5",
+        "jsdoc-nr-template": "github:node-red/jsdoc-nr-template",
         "minami": "1.2.3",
         "mocha": "^5.2.0",
         "mosca": "^2.8.3",
+        "node-red-node-test-helper": "^0.2.3",
+        "node-sass": "^4.13.0",
         "should": "^8.4.0",
         "sinon": "1.17.7",
         "stoppable": "^1.1.0",
-        "supertest": "3.4.2",
-        "node-red-node-test-helper": "^0.2.3",
-        "jsdoc-nr-template": "node-red/jsdoc-nr-template"
+        "supertest": "3.4.2"
     },
     "engines": {
         "node": ">=8"

packages/node_modules/@node-red/editor-api/lib/editor/locales.js

@@ -15,7 +15,7 @@
  **/
 var fs = require('fs');
 var path = require('path');
-//var apiUtil = require('../util');
+// var apiUtil = require('../util');
 
 var i18n = require("@node-red/util").i18n; // TODO: separate module
 
@@ -41,7 +41,7 @@ module.exports = {
         var namespace = req.params[0];
         var lngs = req.query.lng;
         namespace = namespace.replace(/\.json$/,"");
-        var lang = req.query.lng; //apiUtil.determineLangFromHeaders(req.acceptsLanguages() || []);
+        var lang = req.query.lng || i18n.defaultLang; //apiUtil.determineLangFromHeaders(req.acceptsLanguages() || []);
         var prevLang = i18n.i.language;
         // Trigger a load from disk of the language if it is not the default
         i18n.i.changeLanguage(lang, function(){

packages/node_modules/@node-red/editor-api/lib/index.js

@@ -42,7 +42,7 @@ var editor;
 /**
  * Initialise the module.
  * @param  {Object}     settings   The runtime settings
- * @param  {HTTPServer} server     An instance of HTTP Server
+ * @param  {HTTPServer} _server     An instance of HTTP Server
  * @param  {Storage}    storage    An instance of Node-RED Storage
  * @param  {Runtime}    runtimeAPI An instance of Node-RED Runtime
  * @memberof @node-red/editor-api

packages/node_modules/@node-red/editor-client/locales/en-US/jsonata.json

@@ -1,7 +1,7 @@
 {
     "$string": {
-        "args": "arg",
-        "desc": "Casts the *arg* parameter to a string using the following casting rules:\n\n - Strings are unchanged\n - Functions are converted to an empty string\n - Numeric infinity and NaN throw an error because they cannot be represented as a JSON number\n - All other values are converted to a JSON string using the `JSON.stringify` function"
+        "args": "arg[, prettify]",
+        "desc": "Casts the `arg` parameter to a string using the following casting rules:\n\n - Strings are unchanged\n - Functions are converted to an empty string\n - Numeric infinity and NaN throw an error because they cannot be represented as a JSON number\n - All other values are converted to a JSON string using the `JSON.stringify` function.  If `prettify` is true, then \"prettified\" JSON is produced. i.e One line per field and lines will be indented based on the field depth."
     },
     "$length": {
         "args": "str",
@@ -185,7 +185,7 @@
     },
     "$reduce": {
         "args":"array, function [, init]",
-        "desc":"Returns an aggregated value derived from applying the `function` parameter successively to each value in `array` in combination with the result of the previous application of the function.\n\nThe function must accept two arguments, and behaves like an infix operator between each value within the `array`.\n\nThe optional `init` parameter is used as the initial value in the aggregation."
+        "desc":"Returns an aggregated value derived from applying the `function` parameter successively to each value in `array` in combination with the result of the previous application of the function.\n\nThe function must accept two arguments, and behaves like an infix operator between each value within the `array`. The signature of `function` must be of the form: `myfunc($accumulator, $value[, $index[, $array]])`\n\nThe optional `init` parameter is used as the initial value in the aggregation."
     },
     "$flowContext": {
         "args": "string[, string]",
@@ -230,6 +230,37 @@
     "$parseInteger": {
         "args": "string, picture",
         "desc": "Parses the contents of the `string` parameter to an integer (as a JSON number) using the format specified by the `picture` string. The `picture` string parameter has the same format as `$formatInteger`."
-
+    },
+    "$error": {
+        "args": "[str]",
+        "desc": "Throws an error with a message.  The optional `str` will replace the default message of `$error() function evaluated`"
+    },
+    "$assert": {
+        "args": "arg, str",
+        "desc": "If `arg` is true the function returns undefined.  If `arg` is false an exception is thrown with `str` as the message of the exception."
+    },
+    "$single": {
+        "args": "array, function",
+        "desc": "Returns the one and only value in the `array` parameter that satisfies the `function` predicate (i.e. the `function` returns Boolean `true` when passed the value).  Throws an exception if the number of matching values is not exactly one.\n\nThe function should be supplied in the following signature: `function(value [, index [, array]])` where value is each input of the array, index is the position of that value and the whole array is passed as the third argument"
+    },
+    "$encodeUrl": {
+        "args": "str",
+        "desc": "Encodes a Uniform Resource Locator (URL) component by replacing each instance of certain characters by one, two, three, or four escape sequences representing the UTF-8 encoding of the character.\n\nExample: `$encodeUrlComponent(\"?x=test\")` => `\"%3Fx%3Dtest\"`"
+    },
+    "$encodeUrlComponent": {
+        "args": "str",
+        "desc": "Encodes a Uniform Resource Locator (URL) by replacing each instance of certain characters by one, two, three, or four escape sequences representing the UTF-8 encoding of the character. \n\nExample: `$encodeUrl(\"https://mozilla.org/?x=шеллы\")` => `\"https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B\"`"
+    },
+    "$decodeUrl": {
+        "args": "str",
+        "desc": "Decodes a Uniform Resource Locator (URL) component previously created by encodeUrlComponent. \n\nExample: `$decodeUrlComponent(\"%3Fx%3Dtest\")` => `\"?x=test\"`"
+    },
+    "$decodeUrlComponent": {
+        "args": "str",
+        "desc": "Decodes a Uniform Resource Locator (URL) previously created by encodeUrl. \n\nExample: `$decodeUrl(\"https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B\")` => `\"https://mozilla.org/?x=шеллы\"`"
+    },
+    "$distinct": {
+        "args": "array",
+        "desc": "Returns an array with duplicate values removed from `array`"
     }
 }

packages/node_modules/@node-red/editor-client/src/js/ui/editor.js

@@ -2558,7 +2558,7 @@ RED.editor = (function() {
         /**
          * Register a type editor.
          * @param {string} type - the type name
-         * @param {object} options - the editor definition
+         * @param {object} definition - the editor definition
          * @function
          * @memberof RED.editor
          */

packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js

@@ -788,7 +788,7 @@ RED.palette.editor = (function() {
             initInstallTab();
         })
 
-        packageList = $('<ol>',{style:"position: absolute;top: 78px;bottom: 0;left: 0;right: 0px;"}).appendTo(installTab).editableList({
+        packageList = $('<ol>',{style:"position: absolute;top: 79px;bottom: 0;left: 0;right: 0px;"}).appendTo(installTab).editableList({
             addButton: false,
             scrollOnAdd: false,
             addItem: function(container,i,object) {

packages/node_modules/@node-red/editor-client/src/js/ui/subflow.js

@@ -802,8 +802,8 @@ RED.subflow = (function() {
             }
             $("<option/>", opt).text(item.text).appendTo(locales);
         });
-        currentLocale = RED.i18n.lang();
-        locales.val(currentLocale);
+        var locale = RED.i18n.lang();
+        locales.val(locale);
 
         locales.on("change", function() {
             currentLocale = $(this).val();
@@ -1048,7 +1048,7 @@ RED.subflow = (function() {
              }
              langs.forEach(function(l) {
                  var row = $('<div>').appendTo(content);
-                 $('<span>').css({display:"inline-block",width:"50px"}).text(l+(l===currentLocale?"*":"")).appendTo(row);
+                 $('<span>').css({display:"inline-block",width:"120px"}).text(RED._("languages."+l)+(l===currentLocale?"*":"")).appendTo(row); 
                  $('<span>').text(ui.label[l]||"").appendTo(row);
              });
              return content;
@@ -1371,7 +1371,8 @@ RED.subflow = (function() {
         }
 
         var labels = ui.label || {};
-        var labelText = lookupLabel(labels, labels["en-US"]||tenv.name, currentLocale);
+        var locale = RED.i18n.lang();
+        var labelText = lookupLabel(labels, labels["en-US"]||tenv.name, locale);
         var label = $('<label>').appendTo(row);
         var labelContainer = $('<span></span>').appendTo(label);
         if (ui.icon) {
@@ -1423,7 +1424,7 @@ RED.subflow = (function() {
                 input = $('<select>').css('width','70%').appendTo(row);
                 if (ui.opts.opts) {
                     ui.opts.opts.forEach(function(o) {
-                        $('<option>').val(o.v).text(lookupLabel(o.l, o.l['en-US']||o.v, currentLocale)).appendTo(input);
+                        $('<option>').val(o.v).text(lookupLabel(o.l, o.l['en-US']||o.v, locale)).appendTo(input);
                     })
                 }
                 input.val(val.value);

packages/node_modules/@node-red/editor-client/src/vendor/jsonata/formatter.js

@@ -109,6 +109,7 @@
     {
         '$abs':{ args:[ 'number' ]},
         '$append':{ args:[ 'array1', 'array2' ]},
+        '$assert':{ args: [ 'arg', 'str' ]},
         '$average':{ args:[ 'array' ]},
         '$base64decode':{ args:[  ]},
         '$base64encode':{ args:[  ]},
@@ -116,8 +117,14 @@
         '$ceil':{ args:[ 'number' ]},
         '$contains':{ args:[ 'str', 'pattern' ]},
         '$count':{ args:[ 'array' ]},
+        '$decodeUrl':{ args:[ 'str' ]},
+        '$decodeUrlComponent':{ args:[ 'str' ]},
+        '$distinct':{ args:[ 'array' ]},
         '$each':{ args:[ 'object', 'function' ]},
+        '$encodeUrl':{ args: ['str'] },
+        '$encodeUrlComponent':{ args:[ 'str' ]},
         '$env': { args:[ 'arg' ]},
+        '$error':{ args:[ 'str' ]},
         '$eval': { args: ['expr', 'context']},
         '$exists':{ args:[ 'arg' ]},
         '$filter':{ args:[ 'array', 'function' ]},
@@ -151,12 +158,13 @@
         '$reverse':{ args:[ 'array' ]},
         '$round':{ args:[ 'number', 'precision' ]},
         '$shuffle':{ args:[ 'array' ]},
-        '$sift':{ args:[ 'object', 'function' ]},
+        '$sift':{ args: ['object', 'function'] },
+        '$single':{ args: ['array', 'function'] },
         '$sort':{ args:[ 'array', 'function' ]},
         '$split':{ args:[ 'str', 'separator', 'limit' ]},
         '$spread':{ args:[ 'object' ]},
         '$sqrt':{ args:[ 'number' ]},
-        '$string':{ args:[ 'arg' ]},
+        '$string':{ args:[ 'arg', 'prettify' ]},
         '$substring':{ args:[ 'str', 'start', 'length' ]},
         '$substringAfter':{ args:[ 'str', 'chars' ]},
         '$substringBefore':{ args:[ 'str', 'chars' ]},

packages/node_modules/@node-red/nodes/core/network/21-httprequest.html

@@ -170,6 +170,7 @@
             if (this.authType) {
                 $('#node-input-useAuth').prop('checked', true);
                 $("#node-input-authType-select").val(this.authType);
+                $("#node-input-authType-select").change();
             } else {
                 $('#node-input-useAuth').prop('checked', false);
             }

packages/node_modules/@node-red/nodes/core/network/31-tcpin.js

@@ -371,7 +371,6 @@ module.exports = function(RED) {
             var server = net.createServer(function (socket) {
                 socket.setKeepAlive(true,120000);
                 if (socketTimeout !== null) { socket.setTimeout(socketTimeout); }
-                var remoteDetails = socket.remoteAddress+":"+socket.remotePort;
                 node.log(RED._("tcpin.status.connection-from",{host:socket.remoteAddress, port:socket.remotePort}));
                 connectedSockets.push(socket);
                 node.status({text:RED._("tcpin.status.connections",{count:connectedSockets.length})});

packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js

@@ -230,17 +230,17 @@ module.exports = function(RED) {
                                 node.send(msg); // finally send the array
                             }
                         }
-            			else {
-            			    var len = a.length;
-            			    for (var i = 0; i < len; i++) {
+                        else {
+                            var len = a.length;
+                            for (var i = 0; i < len; i++) {
                                 var newMessage = RED.util.cloneMessage(msg);
                                 newMessage.payload = a[i];
                                 if (!has_parts) {
-                				    newMessage.parts = {
-                    				    id: msg._msgid,
-                    				    index: i,
-                    				    count: len
-                    				};
+                                    newMessage.parts = {
+                                        id: msg._msgid,
+                                        index: i,
+                                        count: len
+                                    };
                                 }
                                 else {
                                     newMessage.parts.index -= node.skip;
@@ -251,8 +251,8 @@ module.exports = function(RED) {
                                     }
                                 }
                                 node.send(newMessage);
-            			    }
-            			}
+                            }
+                        }
                         node.linecount = 0;
                     }
                     catch(e) { node.error(e,msg); }

packages/node_modules/@node-red/nodes/core/storage/10-file.js

@@ -140,8 +140,8 @@ module.exports = function(RED) {
                             try {
                                 var stat = fs.statSync(filename);
                                 node.wstreamIno = stat.ino;
-                            } catch(err) {
                             } 
+                            catch(err) { }
                         });
                         node.wstream.on("error", function(err) {
                             node.error(RED._("file.errors.appendfail",{error:err.toString()}),msg);
@@ -276,7 +276,6 @@ module.exports = function(RED) {
                     ch = "\n";
                     type = "string";
                 }
-                var hwm;
                 var getout = false;
 
                 var rs = fs.createReadStream(filename)
@@ -290,30 +289,24 @@ module.exports = function(RED) {
                                     spare += decode(chunk, node.encoding);
                                     var bits = spare.split("\n");
                                     for (var i=0; i < bits.length - 1; i++) {
-                                        var m = {
-                                            payload:bits[i],
-                                            topic:msg.topic,
-                                            filename:msg.filename,
-                                            parts:{index:count, ch:ch, type:type, id:msg._msgid}
-                                        }
+                                        var sendMessage = RED.util.cloneMessage(msg);
+                                        sendMessage.payload = bits[i];
+                                        sendMessage.parts = {index:count, ch:ch, type:type, id:msg._msgid};
                                         count += 1;
-                                        nodeSend(m);
+                                        nodeSend(sendMessage);
                                     }
                                     spare = bits[i];
                                 }
                                 if (node.format === "stream") {
-                                    var m = {
-                                        payload:chunk,
-                                        topic:msg.topic,
-                                        filename:msg.filename,
-                                        parts:{index:count, ch:ch, type:type, id:msg._msgid}
-                                    }
+                                    var sendMessage = RED.util.cloneMessage(msg);
+                                    sendMessage.payload = chunk;
+                                    sendMessage.parts = {index:count, ch:ch, type:type, id:msg._msgid};
                                     count += 1;
                                     if (chunk.length < hwm) { // last chunk is smaller that high water mark = eof
                                         getout = false;
-                                        m.parts.count = count;
+                                        sendMessage.parts.count = count;
                                     }
-                                    nodeSend(m);
+                                    nodeSend(sendMessage);
                                 }
                             }
                             else {
@@ -332,28 +325,23 @@ module.exports = function(RED) {
                         nodeDone();
                     })
                     .on('end', function() {
+                        var sendMessage = RED.util.cloneMessage(msg);
                         if (node.chunk === false) {
                             if (node.format === "utf8") {
-                                msg.payload = decode(lines, node.encoding);
+                                sendMessage.payload = decode(lines, node.encoding);
                             }
-                            else { msg.payload = lines; }
-                            nodeSend(msg);
+                            else { sendMessage.payload = lines; }
+                            nodeSend(sendMessage);
                         }
                         else if (node.format === "lines") {
-                            var m = { payload: spare,
-                                      parts: {
-                                          index: count,
-                                          count: count+1,
-                                          ch: ch,
-                                          type: type,
-                                          id: msg._msgid
-                                      }
-                                    };
-                            nodeSend(m);
+                            sendMessage.payload = spare;
+                            sendMessage.parts = { index:count, count:count+1, ch:ch, type:type, id:msg._msgid };
+                            nodeSend(sendMessage);
                         }
                         else if (getout) { // last chunk same size as high water mark - have to send empty extra packet.
-                            var m = { parts:{index:count, count:count, ch:ch, type:type, id:msg._msgid} };
-                            nodeSend(m);
+                            delete sendMessage.payload;
+                            sendMessage.parts = { parts:{index:count, count:count, ch:ch, type:type, id:msg._msgid} };
+                            nodeSend(sendMessage);
                         }
                         nodeDone();
                     });

packages/node_modules/@node-red/nodes/locales/en-US/network/21-httprequest.html

@@ -36,7 +36,7 @@
         <dt class="optional">followRedirects</dt>
         <dd>If set to <code>false</code> prevent following Redirect (HTTP 301).<code>true</code> by default</dd>
         <dt class="optional">requestTimeout</dt>
-        <dd>If set to a positive number, will override the globally set <code>httpRequestTimeout</code> parameter.</dd>
+        <dd>If set to a positive number of milliseconds, will override the globally set <code>httpRequestTimeout</code> parameter.</dd>
     </dl>
     <h3>Outputs</h3>
     <dl class="message-properties">

packages/node_modules/@node-red/nodes/locales/ja/network/21-httprequest.html

@@ -33,6 +33,8 @@
         <dd><code>false</code>をセットすると、自己署名証明書を使用するhttpsサイトへのリクエストを許可します。</dd>
         <dt class="optional">followRedirects</dt>
         <dd><code>false</code>をセットすると、リダイレクトを行いません。デフォルトは<code>true</code>です。</dd>
+        <dt class="optional">requestTimeout</dt>
+        <dd>正のミリ秒数をセットすると、 グローバルに設定された<code>httpRequestTimeout</code>パラメータを上書きします。</dd>
     </dl>
     <h3>出力</h3>
     <dl class="message-properties">

packages/node_modules/@node-red/nodes/package.json

@@ -27,7 +27,7 @@
         "fs-extra": "8.1.0",
         "fs.notify": "0.0.4",
         "hash-sum": "2.0.0",
-        "https-proxy-agent": "2.2.2",
+        "https-proxy-agent": "2.2.3",
         "is-utf8": "0.2.1",
         "js-yaml": "3.13.1",
         "media-typer": "1.1.0",

packages/node_modules/@node-red/runtime/lib/nodes/Node.js

@@ -224,7 +224,7 @@ Node.prototype._emitInput = function(arg) {
                     }
                 );
             } catch(err) {
-                node.error(err,msg);
+                node.error(err,arg);
             }
         }
     }

packages/node_modules/@node-red/util/package.json

@@ -18,7 +18,7 @@
         "clone": "2.1.2",
         "i18next": "15.1.2",
         "json-stringify-safe": "5.0.1",
-        "jsonata": "1.6.5",
+        "jsonata": "1.7.0",
         "when": "3.7.8"
     }
 }

test/nodes/core/function/89-delay_spec.js

@@ -251,7 +251,8 @@ describe('delay Node', function() {
             var helperNode1 = helper.getNode("helperNode1");
             var receivedMessagesStack = [];
 
-            var rate = 1000/aLimit;
+            // Add a small grace to the calculated delay
+            var rate = 1000/aLimit + 10;
 
             var receiveTimestamp;
 

test/nodes/core/storage/10-file_spec.js

@@ -1192,6 +1192,43 @@ describe('file Nodes', function() {
             });
         });
 
+        it('should read in a file and output split lines with parts and preserve other properties', function(done) {
+            var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", filename:fileToTest, format:"lines", wires:[["n2"]]},
+                        {id:"n2", type:"helper"}];
+            helper.load(fileNode, flow, function() {
+                var n1 = helper.getNode("fileInNode1");
+                var n2 = helper.getNode("n2");
+                var c = 0;
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('payload');
+                        msg.payload.should.be.a.String();
+                        msg.should.have.property('topic',"dujour");
+                        msg.should.have.property('foo',"bar");
+                        msg.should.have.property('parts');
+                        msg.parts.should.have.property('index',c);
+                        msg.parts.should.have.property('type','string');
+                        msg.parts.should.have.property('ch','\n');
+                        if (c === 0) {
+                            msg.payload.should.equal("File message line 1");
+                        }
+                        if (c === 1) {
+                            msg.payload.should.equal("File message line 2");
+                        }
+                        if (c === 2) {
+                            msg.payload.should.equal("");
+                            done();
+                        }
+                        c++;
+                    }
+                    catch(e) {
+                        done(e);
+                    }
+                });
+                n1.receive({payload:"",topic:"dujour",foo:"bar"});
+            });
+        });
+
         it('should read in a file and output a buffer with parts', function(done) {
             var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", filename:fileToTest, format:"stream", wires:[["n2"]]},
                         {id:"n2", type:"helper"}];
@@ -1212,6 +1249,28 @@ describe('file Nodes', function() {
             });
         });
 
+        it('should read in a file and output a buffer with parts and preserve other properties', function(done) {
+            var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", filename:fileToTest, format:"stream", wires:[["n2"]]},
+                        {id:"n2", type:"helper"}];
+            helper.load(fileNode, flow, function() {
+                var n1 = helper.getNode("fileInNode1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    msg.should.have.property('payload');
+                    msg.should.have.property('topic',"dujour");
+                    msg.should.have.property('foo',"bar");
+                    Buffer.isBuffer(msg.payload).should.be.true();
+                    msg.payload.should.have.length(40);
+                    msg.should.have.property('parts');
+                    msg.parts.should.have.property('count',1);
+                    msg.parts.should.have.property('type','buffer');
+                    msg.parts.should.have.property('ch','');
+                    done();
+                });
+                n1.receive({payload:"",topic:"dujour",foo:"bar"});
+            });
+        });
+
         it('should warn if no filename set', function(done) {
             var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", "format":""}];
             helper.load(fileNode, flow, function() {