From ca4960e097609fdf113d83420a667afbce7f9b6b Mon Sep 17 00:00:00 2001 From: Dave Conway-Jones Date: Tue, 10 Nov 2020 14:43:59 +0000 Subject: [PATCH] Fix CSV node repeating array output and add tests to cover it --- .../@node-red/nodes/core/parsers/70-CSV.js | 19 +++-- .../nodes/locales/en-US/parsers/70-CSV.html | 2 +- test/nodes/core/parsers/70-CSV_spec.js | 70 ++++++++++++++++--- 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 15c16da13..c7016a46a 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -63,14 +63,19 @@ module.exports = function(RED) { if (typeof msg.payload == "object") { // convert object to CSV string try { var ou = ""; + if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } if (node.hdrout !== "none" && node.hdrSent === false) { - if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) { - node.template = clean((msg.columns || "").split(",")); + if ((node.template.length === 1) && (node.template[0] === '')) { + if (msg.hasOwnProperty("columns")) { + node.template = clean((msg.columns || "").split(",")); + } + else { + node.template = Object.keys(msg.payload[0]); + } } ou += node.template.join(node.sep) + node.ret; if (node.hdrout === "once") { node.hdrSent = true; } } - if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } for (var s = 0; s < msg.payload.length; s++) { if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) { if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; } @@ -98,10 +103,10 @@ module.exports = function(RED) { } for (var p in msg.payload[0]) { /* istanbul ignore else */ - if (msg.payload[0].hasOwnProperty(p)) { + if (msg.payload[s].hasOwnProperty(p)) { /* istanbul ignore else */ - if (typeof msg.payload[0][p] !== "object") { - var q = "" + msg.payload[0][p]; + if (typeof msg.payload[s][p] !== "object") { + var q = "" + msg.payload[s][p]; if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes q = q.replace(/"/g, '""'); ou += node.quo + q + node.quo + node.sep; @@ -228,7 +233,7 @@ module.exports = function(RED) { // Finished so finalize and send anything left if (f === false) { node.warn(RED._("csv.errors.bad_csv")); } if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } - + if ( node.template[j] && (node.template[j] !== "") ) { if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } else { if (k[j] !== null) k[j].replace(/\r$/,''); } diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html index 6f16f5b72..1f9880a9f 100644 --- a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html +++ b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html @@ -39,7 +39,7 @@ will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.

When converting to CSV, the column template is used to identify which properties to extract from the object and in what order.

If the template is blank then the node can use a simple comma separated list of properties supplied in msg.columns to - determine what to extract. If that is not present then all the object properties are ouput in the order in which they are found.

+ determine what to extract. If that is not present then all the object properties are output in the order in which the properties are found in the first row.

If the input is an array then the columns template is only used to optionally generate a row of column titles.

If 'parse numerical values' option is checked, string numerical values will be returned as numbers, ie. middle value '1,"1.5",2'.

If 'include empty strings' option is checked, empty strings will be returned in result, ie. middle value '"1","",3'.

diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js index 10e167b6d..17cbb9dc1 100644 --- a/test/nodes/core/parsers/70-CSV_spec.js +++ b/test/nodes/core/parsers/70-CSV_spec.js @@ -261,15 +261,15 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - if (c == 0) { + if (c == 0) { c = 1; msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\n" }); check_parts(msg, 0, 1); } - else { + else { msg.should.have.property('payload', { a: "this is", b: "a normal", c: "line" }); check_parts(msg, 0, 1); - done(); + done(); } }); var testString = '"with,a"n,odd","num"ber","of"qu"ot"es"'+String.fromCharCode(10); @@ -287,15 +287,15 @@ describe('CSV node', function() { var c = 0; n2.on("input", function(msg) { //console.log(msg) - if (c == 0) { + if (c == 0) { c = 1; msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\nthis is,a normal,line" }); check_parts(msg, 0, 1); } - else { + else { msg.should.have.property('payload', { a: "this is", b: "another", c: "line" }); check_parts(msg, 0, 1); - done(); + done(); } }); var testString = '"with,a"n,odd","num"ber","of"qu"ot"es"'+String.fromCharCode(10)+'"this is","a normal","line"'+String.fromCharCode(10); @@ -555,14 +555,68 @@ describe('CSV node', function() { }); it('should convert an array of objects to a multi-line csv', function(done) { - var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", wires:[["n2"]] }, + var flow = [ { id:"n1", type:"csv", temp:"a,d,c,b", wires:[["n2"]] }, {id:"n2", type:"helper"} ]; helper.load(csvNode, flow, function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { try { - msg.should.have.property('payload', '4,3,2,1\n1,2,3,4\n'); + msg.should.have.property('payload', '4,1,2,3\n1,4,3,2\n'); + done(); + } + catch(e) { done(e); } + }); + var testJson = [{ d: 1, b: 3, c: 2, a: 4 },{d:4,a:1,c:3,b:2}]; + n1.emit("input", {payload:testJson}); + }); + }); + + it('should convert an array of objects to a multi-line csv and add a header', function(done) { + var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", hdrout:"all", wires:[["n2"]] }, + {id:"n2", type:"helper"} ]; + helper.load(csvNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + try { + msg.should.have.property('payload', 'a,b,c,d\n4,3,2,1\n1,2,3,4\n'); + done(); + } + catch(e) { done(e); } + }); + var testJson = [{ d: 1, b: 3, c: 2, a: 4 },{d:4,a:1,c:3,b:2}]; + n1.emit("input", {payload:testJson}); + }); + }); + + it('should convert an array of objects to a multi-line csv without a template', function(done) { + var flow = [ { id:"n1", type:"csv", temp:"", wires:[["n2"]] }, + {id:"n2", type:"helper"} ]; + helper.load(csvNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + try { + msg.should.have.property('payload', '1,3,2,4\n4,2,3,1\n'); + done(); + } + catch(e) { done(e); } + }); + var testJson = [{ d: 1, b: 3, c: 2, a: 4 },{d:4,a:1,c:3,b:2}]; + n1.emit("input", {payload:testJson}); + }); + }); + + it('should convert an array of objects to a multi-line csv without a template and with a header', function(done) { + var flow = [ { id:"n1", type:"csv", temp:"", hdrout:"all", wires:[["n2"]] }, + {id:"n2", type:"helper"} ]; + helper.load(csvNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + try { + msg.should.have.property('payload', 'd,b,c,a\n1,3,2,4\n4,2,3,1\n'); done(); } catch(e) { done(e); }