From 6aac44db14b72c1121e12d98d23fd2da494663b0 Mon Sep 17 00:00:00 2001 From: Dave Conway-Jones Date: Fri, 25 Jun 2021 14:39:18 +0100 Subject: [PATCH] Add option for file-in node to include all properties (default off) and add test --- .../@node-red/nodes/core/storage/10-file.html | 16 +++++- .../@node-red/nodes/core/storage/10-file.js | 57 ++++++++++++------- .../nodes/locales/en-US/messages.json | 3 +- test/nodes/core/storage/10-file_spec.js | 45 ++++++++++++++- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/storage/10-file.html b/packages/node_modules/@node-red/nodes/core/storage/10-file.html index a7349715f..4de59848e 100755 --- a/packages/node_modules/@node-red/nodes/core/storage/10-file.html +++ b/packages/node_modules/@node-red/nodes/core/storage/10-file.html @@ -48,9 +48,14 @@ +
+ + + +
-
@@ -264,7 +269,8 @@ format: {value:"utf8"}, chunk: {value:false}, sendError: {value: false}, - encoding: {value: "none"} + encoding: {value: "none"}, + allProps: {value: false} }, color:"BurlyWood", inputs:1, @@ -317,6 +323,12 @@ else { $("#encoding-spec").hide(); } + if ((format === "lines") || (format === "stream")) { + $("#file-allprops").show(); + } + else { + $("#file-allprops").hide(); + } }); } }); diff --git a/packages/node_modules/@node-red/nodes/core/storage/10-file.js b/packages/node_modules/@node-red/nodes/core/storage/10-file.js index 711d29904..079b4e82f 100644 --- a/packages/node_modules/@node-red/nodes/core/storage/10-file.js +++ b/packages/node_modules/@node-red/nodes/core/storage/10-file.js @@ -261,6 +261,7 @@ module.exports = function(RED) { this.format = n.format; this.chunk = false; this.encoding = n.encoding || "none"; + this.allProps = n.allProps || false; if (n.sendError === undefined) { this.sendError = true; } else { @@ -299,6 +300,7 @@ module.exports = function(RED) { var rs = fs.createReadStream(fullFilename) .on('readable', function () { var chunk; + var m; var hwm = rs._readableState.highWaterMark; while (null !== (chunk = rs.read())) { if (node.chunk === true) { @@ -307,24 +309,32 @@ module.exports = function(RED) { spare += decode(chunk, node.encoding); var bits = spare.split("\n"); for (var i=0; i < bits.length - 1; i++) { - var m = { - payload:bits[i], - topic:msg.topic, - filename:msg.filename, - parts:{index:count, ch:ch, type:type, id:msg._msgid} + m = {}; + if (node.allProps == true) { + m = RED.util.cloneMessage(msg); } + else { + m.topic = msg.topic; + m.filename = msg.filename; + } + m.payload = bits[i]; + m.parts= {index:count, ch:ch, type:type, id:msg._msgid} count += 1; nodeSend(m); } spare = bits[i]; } if (node.format === "stream") { - var m = { - payload:chunk, - topic:msg.topic, - filename:msg.filename, - parts:{index:count, ch:ch, type:type, id:msg._msgid} + m = {}; + if (node.allProps == true) { + m = RED.util.cloneMessage(msg); } + else { + m.topic = msg.topic; + m.filename = msg.filename; + } + m.payload = chunk; + m.parts = {index:count, ch:ch, type:type, id:msg._msgid} count += 1; if (chunk.length < hwm) { // last chunk is smaller that high water mark = eof getout = false; @@ -357,17 +367,22 @@ module.exports = function(RED) { nodeSend(msg); } else if (node.format === "lines") { - var m = { - payload: spare, - topic:msg.topic, - parts: { - index: count, - count: count+1, - ch: ch, - type: type, - id: msg._msgid - } - }; + var m = {}; + if (node.allProps) { + m = RED.util.cloneMessage(msg); + } + else { + m.topic = msg.topic; + m.filename = msg.filename; + } + m.payload = spare; + m.parts = { + index: count, + count: count + 1, + ch: ch, + type: type, + id: msg._msgid + } nodeSend(m); } else if (getout) { // last chunk same size as high water mark - have to send empty extra packet. diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index e5108710b..a990b3ba5 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -861,7 +861,8 @@ "encoding": "Encoding", "deletelabel": "delete __file__", "utf8String": "UTF8 string", - "binaryBuffer": "binary buffer" + "binaryBuffer": "binary buffer", + "allProps": "include all existing properties in each msg" }, "action": { "append": "append to file", diff --git a/test/nodes/core/storage/10-file_spec.js b/test/nodes/core/storage/10-file_spec.js index 5dd14f2e2..99ed23978 100644 --- a/test/nodes/core/storage/10-file_spec.js +++ b/test/nodes/core/storage/10-file_spec.js @@ -14,7 +14,6 @@ * limitations under the License. **/ -var should = require("should"); var path = require('path'); var fs = require('fs-extra'); var os = require('os'); @@ -1184,6 +1183,9 @@ describe('file Nodes', function() { n2.on("input", function(msg) { try { msg.should.have.property('payload'); + msg.should.have.property('topic'); + msg.should.not.have.property('foo'); + msg.should.not.have.property('bar'); msg.payload.should.be.a.String(); msg.should.have.property('parts'); msg.parts.should.have.property('index',c); @@ -1205,7 +1207,7 @@ describe('file Nodes', function() { done(e); } }); - n1.receive({payload:""}); + n1.receive({payload:"",topic:"A",foo:"bar",bar:"foo"}); }); }); @@ -1245,6 +1247,45 @@ describe('file Nodes', function() { }); }); + it('should read in a file and output split lines with parts and extra props', function(done) { + var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", filename:fileToTest, format:"lines", allProps:true, wires:[["n2"]]}, + {id:"n2", type:"helper"}]; + helper.load(fileNode, flow, function() { + var n1 = helper.getNode("fileInNode1"); + var n2 = helper.getNode("n2"); + var c = 0; + n2.on("input", function(msg) { + // console.log(msg) + try { + msg.should.have.property('payload'); + msg.payload.should.be.a.String(); + msg.should.have.property('topic'); + msg.should.have.property('foo'); + msg.should.have.property('bar'); + msg.should.have.property('parts'); + msg.parts.should.have.property('index',c); + msg.parts.should.have.property('type','string'); + msg.parts.should.have.property('ch','\n'); + if (c === 0) { + msg.payload.should.equal("File message line 1"); + } + if (c === 1) { + msg.payload.should.equal("File message line 2"); + } + if (c === 2) { + msg.payload.should.equal(""); + done(); + } + c++; + } + catch(e) { + done(e); + } + }); + n1.receive({payload:"",topic:"B",foo:"bar",bar:"foo"}); + }); + }); + it('should read in a file and output a buffer with parts', function(done) { var flow = [{id:"fileInNode1", type:"file in", name: "fileInNode", filename:fileToTest, format:"stream", wires:[["n2"]]}, {id:"n2", type:"helper"}];