frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Fix CSV handling of special chars as separators

Browse Source 
(ie escape regex special chars)
and add tests
to close #2950
This commit is contained in:
Dave Conway-Jones 2021-04-23 10:47:23 +01:00
parent e57ebdb583
commit 4cebbf8d22
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 38 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -38,18 +38,18 @@ module.exports = function(RED) {
if (this.hdrout === true) { this.hdrout = "all"; } if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true; var tmpwarn = true;
var node = this; var node = this;
var re = new RegExp(node.sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimmed, de-quoted and retrimmed // pass in an array of column names to be trimmed, de-quoted and retrimmed
var clean = function(col,sep) { var clean = function(col,sep) {
if (sep) { re = new RegExp(sep+'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); } if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
col = col.trim().split(re) || [""]; col = col.trim().split(re) || [""];
col = col.map(x => x.replace(/"/g,'').trim()); col = col.map(x => x.replace(/"/g,'').trim());
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; } else { node.goodtmpl = true; }
return col; return col;
} }
var template = clean(node.template); var template = clean(node.template,',');
var notemplate = template.length === 1 && template[0] === ''; var notemplate = template.length === 1 && template[0] === '';
node.hdrSent = false; node.hdrSent = false;
@ -185,7 +185,7 @@ module.exports = function(RED) {
if ((node.hdrin === true) && first) { // if the template is in the first line if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; } if (line.length - i === 1) { tmp += line[i]; }
template = clean(tmp); template = clean(tmp,node.sep);
first = false; first = false;
} }
else { tmp += line[i]; } else { tmp += line[i]; }

34
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -87,6 +87,40 @@ describe('CSV node', function() {
}); });
}); });
it('should convert a simple string to a javascript object with | separator (no template)', function(done) {
var flow = [ { id:"n1", type:"csv", sep:"|", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 });
msg.should.have.property('columns', "col1,col2,col3,col4");
check_parts(msg, 0, 1);
done();
});
var testString = "1|2|3|4"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should convert a simple string to a javascript object with tab separator (with template)', function(done) {
var flow = [ { id:"n1", type:"csv", sep:"\t", temp:"A,B,,D", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { A: 1, B: 2, D: 4 });
msg.should.have.property('columns', "A,B,D");
check_parts(msg, 0, 1);
done();
});
var testString = "1\t2\t3\t4"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should remove quotes and whitespace from template', function(done) { it('should remove quotes and whitespace from template', function(done) {
var flow = [ { id:"n1", type:"csv", temp:'"a", "b" , " c "," d " ', wires:[["n2"]] }, var flow = [ { id:"n1", type:"csv", temp:'"a", "b" , " c "," d " ', wires:[["n2"]] },
{id:"n2", type:"helper"} ]; {id:"n2", type:"helper"} ];