frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Fix CSV node to handle headers with quotes and spaces

Browse Source 
This is a breaking change so needs thinking about
Includes updated tests
This commit is contained in:
Dave Conway-Jones 2022-10-19 09:19:54 +01:00
parent 5b27bcd781
commit 501c78666d
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 58 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -38,16 +38,36 @@ module.exports = function(RED) {
if (this.hdrout === true) { this.hdrout = "all"; } if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true; var tmpwarn = true;
var node = this; var node = this;
var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); // var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimmed, de-quoted and retrimmed // pass in an array of column names to be trimmed, de-quoted and retrimmed
var clean = function(col,sep) { var clean = function(col,sep) {
if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); } var ff = true; // flag to indicate if inside or outside a pair of quotes true = outside.
col = col.trim().split(re) || [""]; var jj = 0; // pointer into array of template items
col = col.map(x => x.replace(/"/g,'').trim()); var kk = [""]; // array of data for each of the template items
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } for (var ii = 0; ii < col.length; ii++) {
if (col[ii] === node.quo) { // if it's a quote toggle inside or outside
if (ii === 0 || col[ii-1] === sep) { ff = !ff; }
else if (col[ii-1] === node.quo) { } // do nothing, "" = " in CSV world
else if (!ff && kk[jj][0] !== node.quo) { ff = !ff; }
else { kk[jj] += col[ii]; }
}
else if ((col[ii] === sep) && ff) { // if it is the end of the group then finish
jj += 1;
ff = true;
kk[jj] = col.length - 1 === ii ? null : "";
}
else if (col[ii] === " " && ff && (ii == 0 | col[ii-1] == sep | col[ii+1] == sep)) {
// skip
}
else {
kk[jj] += col[ii];
}
}
if ((kk.length === 1) && (kk[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; } else { node.goodtmpl = true; }
return col; return kk;
} }
var template = clean(node.template,','); var template = clean(node.template,',');
var notemplate = template.length === 1 && template[0] === ''; var notemplate = template.length === 1 && template[0] === '';
@ -61,7 +81,7 @@ module.exports = function(RED) {
if (typeof msg.payload == "object") { // convert object to CSV string if (typeof msg.payload == "object") { // convert object to CSV string
try { try {
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) { if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template); template = clean(node.template,",");
} }
var ou = ""; var ou = "";
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
@ -136,7 +156,10 @@ module.exports = function(RED) {
} }
else { else {
var tt = template[t]; var tt = template[t];
if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; } if (template[t].indexOf('"') >=0 ) {
tt = tt.replaceAll("'","\\'");
tt = "'"+tt+"'";
}
else { tt = '"'+tt+'"'; } else { tt = '"'+tt+'"'; }
var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']'); var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
/* istanbul ignore else */ /* istanbul ignore else */
@ -199,7 +222,7 @@ module.exports = function(RED) {
if ((node.hdrin === true) && first) { // if the template is in the first line if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; } if (line.length - i === 1) { tmp += line[i]; }
template = clean(tmp,node.sep); template = clean(tmp.trimEnd(),node.sep);
first = false; first = false;
} }
else { tmp += line[i]; } else { tmp += line[i]; }

53
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -138,21 +138,22 @@ describe('CSV node', function() {
}); });
}); });
it('should remove quotes and whitespace from template', function(done) { // it('should remove quotes and whitespace from template', function(done) {
var flow = [ { id:"n1", type:"csv", temp:'"a", "b" , " c "," d " ', wires:[["n2"]] }, // var flow = [ { id:"n1", type:"csv", temp:'"a", "b" , " c "," d " ', wires:[["n2"]] },
{id:"n2", type:"helper"} ]; // {id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() { // helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1"); // var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); // var n2 = helper.getNode("n2");
n2.on("input", function(msg) { // n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); // console.log("GOT",msg.payload)
check_parts(msg, 0, 1); // msg.should.have.property('payload', { a: 1, b: 2, " c ": 3, " d ": 4 });
done(); // check_parts(msg, 0, 1);
}); // done();
var testString = "1,2,3,4"+String.fromCharCode(10); // });
n1.emit("input", {payload:testString}); // var testString = "1,2,3,4"+String.fromCharCode(10);
}); // n1.emit("input", {payload:testString});
}); // });
// });
it('should create column names if no template provided', function(done) { it('should create column names if no template provided', function(done) {
var flow = [ { id:"n1", type:"csv", temp:'', wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:'', wires:[["n2"]] },
@ -195,8 +196,8 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, " d, d ": 4 });
msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); msg.should.have.property('columns', 'a,b b,"c,c"," d, d "');
check_parts(msg, 0, 1); check_parts(msg, 0, 1);
done(); done();
}); });
@ -212,8 +213,8 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, " d, d ": 4 });
msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); msg.should.have.property('columns', 'a,b b,"c,c"," d, d "');
check_parts(msg, 0, 1); check_parts(msg, 0, 1);
done(); done();
}); });
@ -229,8 +230,8 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, "d, d": 4 }); msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, " d, d ": 4 });
msg.should.have.property('columns', 'a,b b,c;c,"d, d"'); msg.should.have.property('columns', 'a,b b,c;c," d, d "');
check_parts(msg, 0, 1); check_parts(msg, 0, 1);
done(); done();
}); });
@ -246,8 +247,8 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c/c":3, "d, d": 4 }); msg.should.have.property('payload', { a: 1, "b b":2, "c/c":3, " d, d ": 4 });
msg.should.have.property('columns', 'a,b b,c/c,"d, d"'); msg.should.have.property('columns', 'a,b b,c/c," d, d "');
check_parts(msg, 0, 1); check_parts(msg, 0, 1);
done(); done();
}); });
@ -263,8 +264,8 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c\\c":3, "d, d": 4 }); msg.should.have.property('payload', { a: 1, "b b":2, "c\\c":3, " d, d ": 4 });
msg.should.have.property('columns', 'a,b b,c\\c,"d, d"'); msg.should.have.property('columns', 'a,b b,c\\c," d, d "');
check_parts(msg, 0, 1); check_parts(msg, 0, 1);
done(); done();
}); });
@ -699,7 +700,6 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
// console.log("GOT",msg)
try { try {
msg.should.have.property('payload', '4,foo,true,,0,"Hello\nWorld",,,undefined,null,null\n'); msg.should.have.property('payload', '4,foo,true,,0,"Hello\nWorld",,,undefined,null,null\n');
done(); done();
@ -718,7 +718,6 @@ describe('CSV node', function() {
var n1 = helper.getNode("n1"); var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2"); var n2 = helper.getNode("n2");
n2.on("input", function(msg) { n2.on("input", function(msg) {
// console.log("GOT",msg)
try { try {
msg.should.have.property('payload', '1,foo,"ba""r","di,ng",,undefined,null\n'); msg.should.have.property('payload', '1,foo,"ba""r","di,ng",,undefined,null\n');
done(); done();