frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

CSV node - handle commas in msg.columns if quoted.

Browse Source 
and add more tests
To close #2860
This commit is contained in:
Dave Conway-Jones 2021-02-12 16:55:41 +00:00
parent f5da2eb633
commit 302c5cfe09
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 48 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -18,7 +18,7 @@ module.exports = function(RED) {
"use strict"; "use strict";
function CSVNode(n) { function CSVNode(n) {
RED.nodes.createNode(this,n); RED.nodes.createNode(this,n);
this.template = (n.temp || "").split(","); this.template = (n.temp || "");
this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r"); this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
this.quo = '"'; this.quo = '"';
this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r"); this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
@ -38,16 +38,12 @@ module.exports = function(RED) {
if (this.hdrout === true) { this.hdrout = "all"; } if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true; var tmpwarn = true;
var node = this; var node = this;
var re = new RegExp(',(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimed, de-quoted and retrimed // pass in an array of column names to be trimed, de-quoted and retrimed
var clean = function(col) { var clean = function(col) {
for (var t = 0; t < col.length; t++) { col = col.trim().split(re) || [""];
col[t] = col[t].trim(); // remove leading and trailing whitespace col = col.map(x => x.replace(/"/g,'').trim());
if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
// remove leading and trailing quotes (if they exist) - and remove whitepace again.
col[t] = col[t].substr(1,col[t].length -2).trim();
}
}
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; } else { node.goodtmpl = true; }
return col; return col;
@ -67,13 +63,14 @@ module.exports = function(RED) {
if (node.hdrout !== "none" && node.hdrSent === false) { if (node.hdrout !== "none" && node.hdrSent === false) {
if ((node.template.length === 1) && (node.template[0] === '')) { if ((node.template.length === 1) && (node.template[0] === '')) {
if (msg.hasOwnProperty("columns")) { if (msg.hasOwnProperty("columns")) {
node.template = clean((msg.columns || "").split(",")); node.template = clean(msg.columns || "");
} }
else { else {
node.template = Object.keys(msg.payload[0]); node.template = Object.keys(msg.payload[0]);
} }
} }
ou += node.template.join(node.sep) + node.ret; // ou += node.template.join(node.sep) + node.ret;
ou += node.template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
if (node.hdrout === "once") { node.hdrSent = true; } if (node.hdrout === "once") { node.hdrSent = true; }
} }
for (var s = 0; s < msg.payload.length; s++) { for (var s = 0; s < msg.payload.length; s++) {
@ -93,7 +90,7 @@ module.exports = function(RED) {
} }
else { else {
if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) { if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) {
node.template = clean((msg.columns || "").split(",")); node.template = clean(msg.columns || "")//.split(","));
} }
if ((node.template.length === 1) && (node.template[0] === '')) { if ((node.template.length === 1) && (node.template[0] === '')) {
/* istanbul ignore else */ /* istanbul ignore else */
@ -144,7 +141,7 @@ module.exports = function(RED) {
} }
} }
msg.payload = ou; msg.payload = ou;
msg.columns = node.template.join(','); msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); } if (msg.payload !== '') { send(msg); }
done(); done();
} }
@ -179,7 +176,7 @@ module.exports = function(RED) {
if ((node.hdrin === true) && first) { // if the template is in the first line if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; } if (line.length - i === 1) { tmp += line[i]; }
node.template = clean(tmp.split(node.sep)); node.template = clean(tmp);
first = false; first = false;
} }
else { tmp += line[i]; } else { tmp += line[i]; }
@ -255,14 +252,14 @@ module.exports = function(RED) {
} }
if (msg.parts.index + 1 === msg.parts.count) { if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store; msg.payload = node.store;
msg.columns = node.template.filter(val => val).join(','); msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
delete msg.parts; delete msg.parts;
send(msg); send(msg);
node.store = []; node.store = [];
} }
} }
else { else {
msg.columns = node.template.filter(val => val).join(','); msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
send(msg); // finally send the array send(msg); // finally send the array
} }
} }
@ -270,7 +267,7 @@ module.exports = function(RED) {
var len = a.length; var len = a.length;
for (var i = 0; i < len; i++) { for (var i = 0; i < len; i++) {
var newMessage = RED.util.cloneMessage(msg); var newMessage = RED.util.cloneMessage(msg);
newMessage.columns = node.template.filter(val => val).join(','); newMessage.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
newMessage.payload = a[i]; newMessage.payload = a[i];
if (!has_parts) { if (!has_parts) {
newMessage.parts = { newMessage.parts = {

35
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -18,6 +18,7 @@
var should = require("should"); var should = require("should");
var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js"); var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js");
var helper = require("node-red-node-test-helper"); var helper = require("node-red-node-test-helper");
const { c } = require("tar");
describe('CSV node', function() { describe('CSV node', function() {
@ -136,6 +137,40 @@ describe('CSV node', function() {
}); });
}); });
it('should allow commas and spaces in the template', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b b,\"c,c\",\" d, d \"", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 });
msg.should.have.property('columns', 'a,b b,"c,c","d, d"');
check_parts(msg, 0, 1);
done();
});
var testString = "1,2,3,4"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should allow passing in a tempalte as first line of CSV', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 });
msg.should.have.property('columns', 'a,b b,"c,c","d, d"');
check_parts(msg, 0, 1);
done();
});
var testString = 'a,b b,"c,c"," d, d "'+"\n"+"1,2,3,4"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) { it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] },
{id:"n2", type:"helper"} ]; {id:"n2", type:"helper"} ];