From 858b3d640adad3390ca49777197b8e3380b1fb8d Mon Sep 17 00:00:00 2001
From: Dave Conway-Jones <conway@uk.ibm.com>
Date: Sat, 10 Apr 2021 22:17:31 +0100
Subject: [PATCH] fix CSV parsing with other than , separator

(and joining as well...
and add tests
to close #2925
---
 .../@node-red/nodes/core/parsers/70-CSV.js    | 11 +++---
 test/nodes/core/parsers/70-CSV_spec.js        | 36 +++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js
index c5be65020..95a4eb2b2 100644
--- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js
+++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js
@@ -38,10 +38,11 @@ module.exports = function(RED) {
         if (this.hdrout === true) { this.hdrout = "all"; }
         var tmpwarn = true;
         var node = this;
-        var re = new RegExp(',(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
+        var re = new RegExp(node.sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
 
-        // pass in an array of column names to be trimed, de-quoted and retrimed
-        var clean = function(col) {
+        // pass in an array of column names to be trimmed, de-quoted and retrimmed
+        var clean = function(col,sep) {
+            if (sep) { re = new RegExp(sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
             col = col.trim().split(re) || [""];
             col = col.map(x => x.replace(/"/g,'').trim());
             if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
@@ -67,7 +68,7 @@ module.exports = function(RED) {
                         if (node.hdrout !== "none" && node.hdrSent === false) {
                             if ((template.length === 1) && (template[0] === '')) {
                                 if (msg.hasOwnProperty("columns")) {
-                                    template = clean(msg.columns || "");
+                                    template = clean(msg.columns || "",",");
                                 }
                                 else {
                                     template = Object.keys(msg.payload[0]);
@@ -93,7 +94,7 @@ module.exports = function(RED) {
                             }
                             else {
                                 if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
-                                    template = clean(msg.columns || "");
+                                    template = clean(msg.columns || "",",");
                                 }
                                 if ((template.length === 1) && (template[0] === '')) {
                                     /* istanbul ignore else */
diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js
index fa73380ae..cccf7bf1c 100644
--- a/test/nodes/core/parsers/70-CSV_spec.js
+++ b/test/nodes/core/parsers/70-CSV_spec.js
@@ -170,6 +170,24 @@ describe('CSV node', function() {
                 n1.emit("input", {payload:testString});
             });
         });
+
+        it('should allow passing in a template as first line of CSV (not comma)', function(done) {
+            var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, sep:";", wires:[["n2"]] },
+                {id:"n2", type:"helper"} ];
+            helper.load(csvNode, flow, function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, "d, d": 4 });
+                    msg.should.have.property('columns', 'a,b b,c;c,"d, d"');
+                    check_parts(msg, 0, 1);
+                    done();
+                });
+                var testString = 'a;b b;"c;c";" d, d "'+"\n"+"1;2;3;4"+String.fromCharCode(10);
+                n1.emit("input", {payload:testString});
+            });
+        });
+
         it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) {
             var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] },
                 {id:"n2", type:"helper"} ];
@@ -609,6 +627,24 @@ describe('CSV node', function() {
             });
         });
 
+        it('should convert a simple object back to a tsv using a tab as a separator', function(done) {
+            var flow = [ { id:"n1", type:"csv", temp:"", sep:"\t", wires:[["n2"]] },
+                {id:"n2", type:"helper"} ];
+            helper.load(csvNode, flow, function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('payload', '1\tfoo\t"ba""r"\tdi,ng\n');
+                        done();
+                    }
+                    catch(e) { done(e); }
+                });
+                var testJson = { d:1, b:"foo", c:"ba\"r", a:"di,ng" };
+                n1.emit("input", {payload:testJson});
+            });
+        });
+
         it('should handle a template with spaces in the property names', function(done) {
             var flow = [ { id:"n1", type:"csv", temp:"a,b o,c p,,e", wires:[["n2"]] },
                 {id:"n2", type:"helper"} ];