frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

fix CSV parsing with other than , separator 

(and joining as well...
and add tests
to close #2925
This commit is contained in:
Dave Conway-Jones 2021-04-10 22:17:31 +01:00
parent 6087002562
commit 858b3d640a
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 42 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -38,10 +38,11 @@ module.exports = function(RED) {
if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true;
var node = this;
var re = new RegExp(',(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
var re = new RegExp(node.sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimed, de-quoted and retrimed
var clean = function(col) {
// pass in an array of column names to be trimmed, de-quoted and retrimmed
var clean = function(col,sep) {
if (sep) { re = new RegExp(sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
col = col.trim().split(re) || [""];
col = col.map(x => x.replace(/"/g,'').trim());
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
@ -67,7 +68,7 @@ module.exports = function(RED) {
if (node.hdrout !== "none" && node.hdrSent === false) {
if ((template.length === 1) && (template[0] === '')) {
if (msg.hasOwnProperty("columns")) {
template = clean(msg.columns || "");
template = clean(msg.columns || "",",");
}
else {
template = Object.keys(msg.payload[0]);
@ -93,7 +94,7 @@ module.exports = function(RED) {
}
else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
template = clean(msg.columns || "");
template = clean(msg.columns || "",",");
}
if ((template.length === 1) && (template[0] === '')) {
/* istanbul ignore else */

36
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -170,6 +170,24 @@ describe('CSV node', function() {
n1.emit("input", {payload:testString});
});
});
it('should allow passing in a template as first line of CSV (not comma)', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, sep:";", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, "d, d": 4 });
msg.should.have.property('columns', 'a,b b,c;c,"d, d"');
check_parts(msg, 0, 1);
done();
});
var testString = 'a;b b;"c;c";" d, d "'+"\n"+"1;2;3;4"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
@ -609,6 +627,24 @@ describe('CSV node', function() {
});
});
it('should convert a simple object back to a tsv using a tab as a separator', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", sep:"\t", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
try {
msg.should.have.property('payload', '1\tfoo\t"ba""r"\tdi,ng\n');
done();
}
catch(e) { done(e); }
});
var testJson = { d:1, b:"foo", c:"ba\"r", a:"di,ng" };
n1.emit("input", {payload:testJson});
});
});
it('should handle a template with spaces in the property names', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b o,c p,,e", wires:[["n2"]] },
{id:"n2", type:"helper"} ];