frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

redo CSV fix for commas in header template

Browse Source
This commit is contained in:
Dave Conway-Jones 2021-03-11 09:34:30 +00:00
parent 2de43b719e
commit 16b9abbe92
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 852 additions and 814 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -48,7 +48,8 @@ module.exports = function(RED) {
else { node.goodtmpl = true; } else { node.goodtmpl = true; }
return col; return col;
} }
node.template = clean(node.template); var template = clean(node.template);
var notemplate = template.length === 1 && template[0] === '';
node.hdrSent = false; node.hdrSent = false;
this.on("input", function(msg, send, done) { this.on("input", function(msg, send, done) {
@ -58,19 +59,21 @@ module.exports = function(RED) {
if (msg.hasOwnProperty("payload")) { if (msg.hasOwnProperty("payload")) {
if (typeof msg.payload == "object") { // convert object to CSV string if (typeof msg.payload == "object") { // convert object to CSV string
try { try {
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template);
}
var ou = ""; var ou = "";
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
if (node.hdrout !== "none" && node.hdrSent === false) { if (node.hdrout !== "none" && node.hdrSent === false) {
if ((node.template.length === 1) && (node.template[0] === '')) { if ((template.length === 1) && (template[0] === '')) {
if (msg.hasOwnProperty("columns")) { if (msg.hasOwnProperty("columns")) {
node.template = clean(msg.columns || ""); template = clean(msg.columns || "");
} }
else { else {
node.template = Object.keys(msg.payload[0]); template = Object.keys(msg.payload[0]);
} }
} }
// ou += node.template.join(node.sep) + node.ret; ou += template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
ou += node.template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
if (node.hdrout === "once") { node.hdrSent = true; } if (node.hdrout === "once") { node.hdrSent = true; }
} }
for (var s = 0; s < msg.payload.length; s++) { for (var s = 0; s < msg.payload.length; s++) {
@ -89,10 +92,10 @@ module.exports = function(RED) {
ou += msg.payload[s].join(node.sep) + node.ret; ou += msg.payload[s].join(node.sep) + node.ret;
} }
else { else {
if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) { if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
node.template = clean(msg.columns || "")//.split(",")); template = clean(msg.columns || "");
} }
if ((node.template.length === 1) && (node.template[0] === '')) { if ((template.length === 1) && (template[0] === '')) {
/* istanbul ignore else */ /* istanbul ignore else */
if (tmpwarn === true) { // just warn about missing template once if (tmpwarn === true) { // just warn about missing template once
node.warn(RED._("csv.errors.obj_csv")); node.warn(RED._("csv.errors.obj_csv"));
@ -118,12 +121,12 @@ module.exports = function(RED) {
ou = ou.slice(0,-1) + node.ret; ou = ou.slice(0,-1) + node.ret;
} }
else { else {
for (var t=0; t < node.template.length; t++) { for (var t=0; t < template.length; t++) {
if (node.template[t] === '') { if (template[t] === '') {
ou += node.sep; ou += node.sep;
} }
else { else {
var p = RED.util.ensureString(RED.util.getMessageProperty(msg,"payload["+s+"]['"+node.template[t]+"']")); var p = RED.util.ensureString(RED.util.getMessageProperty(msg,"payload["+s+"]['"+template[t]+"']"));
/* istanbul ignore else */ /* istanbul ignore else */
if (p === "undefined") { p = ""; } if (p === "undefined") { p = ""; }
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
@ -141,7 +144,7 @@ module.exports = function(RED) {
} }
} }
msg.payload = ou; msg.payload = ou;
msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); } if (msg.payload !== '') { send(msg); }
done(); done();
} }
@ -176,7 +179,7 @@ module.exports = function(RED) {
if ((node.hdrin === true) && first) { // if the template is in the first line if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; } if (line.length - i === 1) { tmp += line[i]; }
node.template = clean(tmp); template = clean(tmp);
first = false; first = false;
} }
else { tmp += line[i]; } else { tmp += line[i]; }
@ -190,14 +193,14 @@ module.exports = function(RED) {
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; } //if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
} }
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") ) { if ( template[j] && (template[j] !== "") ) {
// if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null; if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
} }
j += 1; j += 1;
// if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",' // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
@ -205,15 +208,15 @@ module.exports = function(RED) {
} }
else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
//console.log(j,k,o,k[j]); //console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") ) { if ( template[j] && (template[j] !== "") ) {
// if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3' // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
if (line[i-1] === node.sep) k[j] = null; if (line[i-1] === node.sep) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
else { if (k[j] !== null) k[j].replace(/\r$/,''); } else { if (k[j] !== null) k[j].replace(/\r$/,''); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array a.push(o); // add to the array
@ -230,14 +233,14 @@ module.exports = function(RED) {
} }
// Finished so finalize and send anything left // Finished so finalize and send anything left
if (f === false) { node.warn(RED._("csv.errors.bad_csv")); } if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") ) { if ( template[j] && (template[j] !== "") ) {
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
else { if (k[j] !== null) k[j].replace(/\r$/,''); } else { if (k[j] !== null) k[j].replace(/\r$/,''); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array a.push(o); // add to the array
@ -252,14 +255,14 @@ module.exports = function(RED) {
} }
if (msg.parts.index + 1 === msg.parts.count) { if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store; msg.payload = node.store;
msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
delete msg.parts; delete msg.parts;
send(msg); send(msg);
node.store = []; node.store = [];
} }
} }
else { else {
msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
send(msg); // finally send the array send(msg); // finally send the array
} }
} }
@ -267,7 +270,7 @@ module.exports = function(RED) {
var len = a.length; var len = a.length;
for (var i = 0; i < len; i++) { for (var i = 0; i < len; i++) {
var newMessage = RED.util.cloneMessage(msg); var newMessage = RED.util.cloneMessage(msg);
newMessage.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
newMessage.payload = a[i]; newMessage.payload = a[i];
if (!has_parts) { if (!has_parts) {
newMessage.parts = { newMessage.parts = {
@ -296,7 +299,7 @@ module.exports = function(RED) {
} }
else { else {
if (!msg.hasOwnProperty("reset")) { if (!msg.hasOwnProperty("reset")) {
send(msg); // If no payload and not reset - just pass it on. node.send(msg); // If no payload and not reset - just pass it on.
} }
done(); done();
} }

49
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -18,7 +18,6 @@
var should = require("should"); var should = require("should");
var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js"); var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js");
var helper = require("node-red-node-test-helper"); var helper = require("node-red-node-test-helper");
const { c } = require("tar");
describe('CSV node', function() { describe('CSV node', function() {
@ -39,7 +38,7 @@ describe('CSV node', function() {
helper.load(csvNode, flow, function() { helper.load(csvNode, flow, function() {
var n1 = helper.getNode("csvNode1"); var n1 = helper.getNode("csvNode1");
n1.should.have.property('name', 'csvNode'); n1.should.have.property('name', 'csvNode');
n1.should.have.property('template', [ '' ]); n1.should.have.property('template','');
n1.should.have.property('sep', ','); n1.should.have.property('sep', ',');
n1.should.have.property('quo', '"'); n1.should.have.property('quo', '"');
n1.should.have.property('ret', '\n'); n1.should.have.property('ret', '\n');
@ -154,7 +153,7 @@ describe('CSV node', function() {
}); });
}); });
it('should allow passing in a tempalte as first line of CSV', function(done) { it('should allow passing in a template as first line of CSV', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, wires:[["n2"]] },
{id:"n2", type:"helper"} ]; {id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() { helper.load(csvNode, flow, function() {
@ -170,7 +169,6 @@ describe('CSV node', function() {
n1.emit("input", {payload:testString}); n1.emit("input", {payload:testString});
}); });
}); });
it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) { it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] },
{id:"n2", type:"helper"} ]; {id:"n2", type:"helper"} ];
@ -715,6 +713,25 @@ describe('CSV node', function() {
}); });
}); });
it('should be able to include column names as first row, and missing properties', function(done) {
var flow = [ { id:"n1", type:"csv", hdrout:true, ret:"\r\n", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
try {
msg.should.have.property('payload', 'col1,col2,col3,col4\r\nH1,H2,H3,H4\r\nA,B,,\r\nA,,C,\r\nA,,,D\r\n');
done();
}
catch(e) { done(e); }
});
var testJson = [{"col1":"H1","col2":"H2","col3":"H3","col4":"H4"},{"col1":"A","col2":"B"},{"col1":"A","col3":"C"},{"col1":"A","col4":"D"}];
n1.emit("input", {payload:testJson});
});
});
it('should be able to pass in column names', function(done) { it('should be able to pass in column names', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrout:"once", ret:"\r\n", wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:"", hdrout:"once", ret:"\r\n", wires:[["n2"]] },
{id:"n2", type:"helper"} ]; {id:"n2", type:"helper"} ];
@ -736,9 +753,27 @@ describe('CSV node', function() {
catch(e) { done(e); } catch(e) { done(e); }
}); });
var testJson = [{ d: 1, b: 3, c: 2, a: 4 }]; var testJson = [{ d: 1, b: 3, c: 2, a: 4 }];
n1.emit("input", {payload:testJson, columns:"a,,b,a"}); n1.emit("input", {payload:testJson, columns:"a,,b,a", parts:{index:0}});
n1.emit("input", {payload:testJson}); n1.emit("input", {payload:testJson, parts:{index:1}});
n1.emit("input", {payload:testJson}); n1.emit("input", {payload:testJson, parts:{index:2}});
});
});
it('should be able to pass in column names - with payload as an array', function(done) {
var flow = [ { id:"n1", type:"csv", hdrout:"once", ret:"\r\n", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
try {
msg.should.have.property('payload', 'a,,b,a\r\n4,,3,4\r\n4,,3,4\r\n4,,3,4\r\n');
done()
}
catch(e) { done(e); }
});
var testJson = { d: 1, b: 3, c: 2, a: 4 };
n1.emit("input", {payload:[testJson,testJson,testJson], columns:"a,,b,a"});
}); });
}); });