diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html index 46f416f9e..ddf306ba8 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html @@ -17,7 +17,20 @@ - +
+ +
+ +
+
+
+
+
+
@@ -60,10 +73,10 @@
- + -
@@ -75,6 +88,7 @@ color:"#DEBD5C", defaults: { name: {value:""}, + spec: {value:"rfc"}, sep: { value:',', required:true, label:RED._("node-red:csv.label.separator"), @@ -83,7 +97,7 @@ hdrin: {value:""}, hdrout: {value:"none"}, multi: {value:"one",required:true}, - ret: {value:'\\n'}, + ret: {value:'\\r\\n'}, // default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)") temp: {value:""}, skip: {value:"0"}, strings: {value:true}, @@ -123,6 +137,27 @@ $("#node-input-sep").hide(); } }); + + $("#csv-option-spec").on("change", function() { + if ($("#csv-option-spec").val() == "rfc") { + $(".form-tips.csv-lecacy-warning").hide(); + } else { + $(".form-tips.csv-lecacy-warning").show(); + } + }); + // new nodes will have `spec` set to "rfc" (default), but existing nodes will either not have + // a spec value or it will be empty - we need to maintain the legacy behaviour for existing + // flows but default to rfc for new nodes + let spec = !this.spec ? "" : "rfc" + $("#csv-option-spec").val(spec).trigger("change") + }, + oneditsave: function() { + const specFormVal = $("#csv-option-spec").val() || '' // empty === legacy + const spectNodeVal = this.spec || '' // empty === legacy, null/undefined means in-place node upgrade (keep as is) + if (specFormVal !== spectNodeVal) { + // only update the flow value if changed (avoid marking the node dirty unnecessarily) + this.spec = specFormVal + } } }); diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 17bf56322..c6a91d61a 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -15,322 +15,674 @@ **/ module.exports = function(RED) { + const csv = require('./lib/csv') + "use strict"; function CSVNode(n) { - RED.nodes.createNode(this,n); - this.template = (n.temp || ""); - this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r"); - this.quo = '"'; - this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r"); - this.winflag = (this.ret === "\r\n"); - this.lineend = "\n"; - this.multi = n.multi || "one"; - this.hdrin = n.hdrin || false; - this.hdrout = n.hdrout || "none"; - this.goodtmpl = true; - this.skip = parseInt(n.skip || 0); - this.store = []; - this.parsestrings = n.strings; - this.include_empty_strings = n.include_empty_strings || false; - this.include_null_values = n.include_null_values || false; - if (this.parsestrings === undefined) { this.parsestrings = true; } - if (this.hdrout === false) { this.hdrout = "none"; } - if (this.hdrout === true) { this.hdrout = "all"; } - var tmpwarn = true; - var node = this; - var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); + RED.nodes.createNode(this,n) + const node = this + const RFC4180Mode = n.spec === 'rfc' + const legacyMode = !RFC4180Mode - // pass in an array of column names to be trimmed, de-quoted and retrimmed - var clean = function(col,sep) { - if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); } - col = col.trim().split(re) || [""]; - col = col.map(x => x.replace(/"/g,'').trim()); - if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } - else { node.goodtmpl = true; } - return col; - } - var template = clean(node.template,','); - var notemplate = template.length === 1 && template[0] === ''; - node.hdrSent = false; + node.status({}) // clear status - this.on("input", function(msg, send, done) { - if (msg.hasOwnProperty("reset")) { - node.hdrSent = false; + if (legacyMode) { + this.template = (n.temp || ""); + this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r"); + this.quo = '"'; + this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r"); + this.winflag = (this.ret === "\r\n"); + this.lineend = "\n"; + this.multi = n.multi || "one"; + this.hdrin = n.hdrin || false; + this.hdrout = n.hdrout || "none"; + this.goodtmpl = true; + this.skip = parseInt(n.skip || 0); + this.store = []; + this.parsestrings = n.strings; + this.include_empty_strings = n.include_empty_strings || false; + this.include_null_values = n.include_null_values || false; + if (this.parsestrings === undefined) { this.parsestrings = true; } + if (this.hdrout === false) { this.hdrout = "none"; } + if (this.hdrout === true) { this.hdrout = "all"; } + var tmpwarn = true; + // var node = this; + var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); + + // pass in an array of column names to be trimmed, de-quoted and retrimmed + var clean = function(col,sep) { + if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); } + col = col.trim().split(re) || [""]; + col = col.map(x => x.replace(/"/g,'').trim()); + if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } + else { node.goodtmpl = true; } + return col; } - if (msg.hasOwnProperty("payload")) { - if (typeof msg.payload == "object") { // convert object to CSV string - try { - if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) { - template = clean(node.template); - } - const ou = []; - if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } - if (node.hdrout !== "none" && node.hdrSent === false) { - if ((template.length === 1) && (template[0] === '')) { - if (msg.hasOwnProperty("columns")) { - template = clean(msg.columns || "",","); - } - else { - template = Object.keys(msg.payload[0]); - } + var template = clean(node.template,','); + var notemplate = template.length === 1 && template[0] === ''; + node.hdrSent = false; + + this.on("input", function(msg, send, done) { + if (msg.hasOwnProperty("reset")) { + node.hdrSent = false; + } + if (msg.hasOwnProperty("payload")) { + if (typeof msg.payload == "object") { // convert object to CSV string + try { + if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) { + template = clean(node.template); } - ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep)); - if (node.hdrout === "once") { node.hdrSent = true; } - } - for (var s = 0; s < msg.payload.length; s++) { - if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) { - if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; } - for (var t = 0; t < msg.payload[s].length; t++) { - if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; } - if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes - msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""'); - msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; - } - else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas" - msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; - } - else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n" - msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; - } - } - ou.push(msg.payload[s].join(node.sep)); - } - else { - if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) { - template = clean(msg.columns || "",","); - } + const ou = []; + if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } + if (node.hdrout !== "none" && node.hdrSent === false) { if ((template.length === 1) && (template[0] === '')) { - /* istanbul ignore else */ - if (tmpwarn === true) { // just warn about missing template once - node.warn(RED._("csv.errors.obj_csv")); - tmpwarn = false; + if (msg.hasOwnProperty("columns")) { + template = clean(msg.columns || "",","); } - const row = []; - for (var p in msg.payload[0]) { + else { + template = Object.keys(msg.payload[0]); + } + } + ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep)); + if (node.hdrout === "once") { node.hdrSent = true; } + } + for (var s = 0; s < msg.payload.length; s++) { + if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) { + if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; } + for (var t = 0; t < msg.payload[s].length; t++) { + if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; } + if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes + msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""'); + msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; + } + else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas" + msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; + } + else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n" + msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; + } + } + ou.push(msg.payload[s].join(node.sep)); + } + else { + if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) { + template = clean(msg.columns || "",","); + } + if ((template.length === 1) && (template[0] === '')) { /* istanbul ignore else */ - if (msg.payload[s].hasOwnProperty(p)) { + if (tmpwarn === true) { // just warn about missing template once + node.warn(RED._("csv.errors.obj_csv")); + tmpwarn = false; + } + const row = []; + for (var p in msg.payload[0]) { /* istanbul ignore else */ - if (typeof msg.payload[s][p] !== "object") { - // Fix to honour include null values flag - //if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) { - var q = ""; - if (msg.payload[s][p] !== undefined) { - q += msg.payload[s][p]; + if (msg.payload[s].hasOwnProperty(p)) { + /* istanbul ignore else */ + if (typeof msg.payload[s][p] !== "object") { + // Fix to honour include null values flag + //if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) { + var q = ""; + if (msg.payload[s][p] !== undefined) { + q += msg.payload[s][p]; + } + if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes + q = q.replace(/"/g, '""'); + row.push(node.quo + q + node.quo); + } + else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" + row.push(node.quo + q + node.quo); + } + else { row.push(q); } // otherwise just add } - if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes - q = q.replace(/"/g, '""'); - row.push(node.quo + q + node.quo); - } - else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" - row.push(node.quo + q + node.quo); - } - else { row.push(q); } // otherwise just add } } + ou.push(row.join(node.sep)); // add separator } - ou.push(row.join(node.sep)); // add separator + else { + const row = []; + for (var t=0; t < template.length; t++) { + if (template[t] === '') { + row.push(''); + } + else { + var tt = template[t]; + if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; } + else { tt = '"'+tt+'"'; } + var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']'); + /* istanbul ignore else */ + if (p === undefined) { p = ""; } + // fix to honour include null values flag + //if (p === null && node.include_null_values !== true) { p = "";} + p = RED.util.ensureString(p); + if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes + p = p.replace(/"/g, '""'); + row.push(node.quo + p + node.quo); + } + else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" + row.push(node.quo + p + node.quo); + } + else { row.push(p); } // otherwise just add + } + } + ou.push(row.join(node.sep)); // add separator + } + } + } + // join lines, don't forget to add the last new line + msg.payload = ou.join(node.ret) + node.ret; + msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); + if (msg.payload !== '') { + send(msg); + } + done(); + } + catch(e) { done(e); } + } + else if (typeof msg.payload == "string") { // convert CSV string to object + try { + var f = true; // flag to indicate if inside or outside a pair of quotes true = outside. + var j = 0; // pointer into array of template items + var k = [""]; // array of data for each of the template items + var o = {}; // output object to build up + var a = []; // output array is needed for multiline option + var first = true; // is this the first line + var last = false; + var line = msg.payload; + var linecount = 0; + var tmp = ""; + var has_parts = msg.hasOwnProperty("parts"); + var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i; + if (msg.hasOwnProperty("parts")) { + linecount = msg.parts.index; + if (msg.parts.index > node.skip) { first = false; } + if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; } + } + + // For now we are just going to assume that any \r or \n means an end of line... + // got to be a weird csv that has singleton \r \n in it for another reason... + + // Now process the whole file/line + var nocr = (line.match(/[\r\n]/g)||[]).length; + if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; } + for (var i = 0; i < line.length; i++) { + if (first && (linecount < node.skip)) { + if (line[i] === "\n") { linecount += 1; } + continue; + } + if ((node.hdrin === true) && first) { // if the template is in the first line + if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break + if (line.length - i === 1) { tmp += line[i]; } + template = clean(tmp,node.sep); + first = false; + } + else { tmp += line[i]; } } else { - const row = []; - for (var t=0; t < template.length; t++) { - if (template[t] === '') { - row.push(''); + if (line[i] === node.quo) { // if it's a quote toggle inside or outside + f = !f; + if (line[i-1] === node.quo) { + if (f === false) { k[j] += '\"'; } + } // if it's a quotequote then it's actually a quote + //if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; } + } + else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish + if (!node.goodtmpl) { template[j] = "col"+(j+1); } + if ( template[j] && (template[j] !== "") ) { + // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null + if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null; + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } + if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; } - else { - var tt = template[t]; - if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; } - else { tt = '"'+tt+'"'; } - var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']'); - /* istanbul ignore else */ - if (p === undefined) { p = ""; } - // fix to honour include null values flag - //if (p === null && node.include_null_values !== true) { p = "";} - p = RED.util.ensureString(p); - if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes - p = p.replace(/"/g, '""'); - row.push(node.quo + p + node.quo); - } - else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" - row.push(node.quo + p + node.quo); - } - else { row.push(p); } // otherwise just add + j += 1; + // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",' + k[j] = line.length - 1 === i ? null : ""; + } + else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines + //console.log(j,k,o,k[j]); + if (!node.goodtmpl) { template[j] = "col"+(j+1); } + if ( template[j] && (template[j] !== "") ) { + // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3' + if (line[i-1] === node.sep) k[j] = null; + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } + else { if (k[j] !== null) k[j].replace(/\r$/,''); } + if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; } + if (JSON.stringify(o) !== "{}") { // don't send empty objects + a.push(o); // add to the array + } + j = 0; + k = [""]; + o = {}; + f = true; // reset in/out flag ready for next line. + } + else { // just add to the part of the message + k[j] += line[i]; } - ou.push(row.join(node.sep)); // add separator } } - } - // join lines, don't forget to add the last new line - msg.payload = ou.join(node.ret) + node.ret; - msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); - if (msg.payload !== '') { send(msg); } - done(); - } - catch(e) { done(e); } - } - else if (typeof msg.payload == "string") { // convert CSV string to object - try { - var f = true; // flag to indicate if inside or outside a pair of quotes true = outside. - var j = 0; // pointer into array of template items - var k = [""]; // array of data for each of the template items - var o = {}; // output object to build up - var a = []; // output array is needed for multiline option - var first = true; // is this the first line - var last = false; - var line = msg.payload; - var linecount = 0; - var tmp = ""; - var has_parts = msg.hasOwnProperty("parts"); - var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i; - if (msg.hasOwnProperty("parts")) { - linecount = msg.parts.index; - if (msg.parts.index > node.skip) { first = false; } - if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; } - } + // Finished so finalize and send anything left + if (f === false) { node.warn(RED._("csv.errors.bad_csv")); } + if (!node.goodtmpl) { template[j] = "col"+(j+1); } - // For now we are just going to assume that any \r or \n means an end of line... - // got to be a weird csv that has singleton \r \n in it for another reason... - - // Now process the whole file/line - var nocr = (line.match(/[\r\n]/g)||[]).length; - if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; } - for (var i = 0; i < line.length; i++) { - if (first && (linecount < node.skip)) { - if (line[i] === "\n") { linecount += 1; } - continue; + if ( template[j] && (template[j] !== "") ) { + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } + else { if (k[j] !== null) k[j].replace(/\r$/,''); } + if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; } - if ((node.hdrin === true) && first) { // if the template is in the first line - if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break - if (line.length - i === 1) { tmp += line[i]; } - template = clean(tmp,node.sep); - first = false; - } - else { tmp += line[i]; } + + if (JSON.stringify(o) !== "{}") { // don't send empty objects + a.push(o); // add to the array } - else { - if (line[i] === node.quo) { // if it's a quote toggle inside or outside - f = !f; - if (line[i-1] === node.quo) { - if (f === false) { k[j] += '\"'; } - } // if it's a quotequote then it's actually a quote - //if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; } - } - else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish - if (!node.goodtmpl) { template[j] = "col"+(j+1); } - if ( template[j] && (template[j] !== "") ) { - // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null - if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null; - if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } - if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; - if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; - if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; - } - j += 1; - // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",' - k[j] = line.length - 1 === i ? null : ""; - } - else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines - //console.log(j,k,o,k[j]); - if (!node.goodtmpl) { template[j] = "col"+(j+1); } - if ( template[j] && (template[j] !== "") ) { - // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3' - if (line[i-1] === node.sep) k[j] = null; - if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } - else { if (k[j] !== null) k[j].replace(/\r$/,''); } - if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; - if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; - if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; - } - if (JSON.stringify(o) !== "{}") { // don't send empty objects - a.push(o); // add to the array - } - j = 0; - k = [""]; - o = {}; - f = true; // reset in/out flag ready for next line. - } - else { // just add to the part of the message - k[j] += line[i]; - } - } - } - // Finished so finalize and send anything left - if (f === false) { node.warn(RED._("csv.errors.bad_csv")); } - if (!node.goodtmpl) { template[j] = "col"+(j+1); } - if ( template[j] && (template[j] !== "") ) { - if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } - else { if (k[j] !== null) k[j].replace(/\r$/,''); } - if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; - if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; - if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; - } - - if (JSON.stringify(o) !== "{}") { // don't send empty objects - a.push(o); // add to the array - } - - if (node.multi !== "one") { - msg.payload = a; - if (has_parts && nocr <= 1) { - if (JSON.stringify(o) !== "{}") { - node.store.push(o); + if (node.multi !== "one") { + msg.payload = a; + if (has_parts && nocr <= 1) { + if (JSON.stringify(o) !== "{}") { + node.store.push(o); + } + if (msg.parts.index + 1 === msg.parts.count) { + msg.payload = node.store; + msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); + delete msg.parts; + send(msg); + node.store = []; + } } - if (msg.parts.index + 1 === msg.parts.count) { - msg.payload = node.store; + else { msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); - delete msg.parts; - send(msg); - node.store = []; + send(msg); // finally send the array } } else { - msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); - send(msg); // finally send the array - } - } - else { - var len = a.length; - for (var i = 0; i < len; i++) { - var newMessage = RED.util.cloneMessage(msg); - newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); - newMessage.payload = a[i]; - if (!has_parts) { - newMessage.parts = { - id: msg._msgid, - index: i, - count: len - }; + var len = a.length; + for (var i = 0; i < len; i++) { + var newMessage = RED.util.cloneMessage(msg); + newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); + newMessage.payload = a[i]; + if (!has_parts) { + newMessage.parts = { + id: msg._msgid, + index: i, + count: len + }; + } + else { + newMessage.parts.index -= node.skip; + newMessage.parts.count -= node.skip; + if (node.hdrin) { // if we removed the header line then shift the counts by 1 + newMessage.parts.index -= 1; + newMessage.parts.count -= 1; + } + } + if (last) { newMessage.complete = true; } + send(newMessage); } - else { - newMessage.parts.index -= node.skip; - newMessage.parts.count -= node.skip; - if (node.hdrin) { // if we removed the header line then shift the counts by 1 - newMessage.parts.index -= 1; - newMessage.parts.count -= 1; + if (has_parts && last && len === 0) { + send({complete:true}); + } + } + node.linecount = 0; + done(); + } + catch(e) { done(e); } + } + else { node.warn(RED._("csv.errors.csv_js")); done(); } + } + else { + if (!msg.hasOwnProperty("reset")) { + node.send(msg); // If no payload and not reset - just pass it on. + } + done(); + } + }); + } + + if(RFC4180Mode) { + node.template = (n.temp || "") + node.sep = (n.sep || ',').replace(/\\t/g, "\t").replace(/\\n/g, "\n").replace(/\\r/g, "\r") + node.quo = '"' + // default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)") + node.ret = (n.ret || "\r\n").replace(/\\n/g, "\n").replace(/\\r/g, "\r") + node.multi = n.multi || "one" + node.hdrin = n.hdrin || false + node.hdrout = n.hdrout || "none" + node.goodtmpl = true + node.skip = parseInt(n.skip || 0) + node.store = [] + node.parsestrings = n.strings + node.include_empty_strings = n.include_empty_strings || false + node.include_null_values = n.include_null_values || false + if (node.parsestrings === undefined) { node.parsestrings = true } + if (node.hdrout === false) { node.hdrout = "none" } + if (node.hdrout === true) { node.hdrout = "all" } + const dontSendHeaders = node.hdrout === "none" + const sendHeadersOnce = node.hdrout === "once" + const sendHeadersAlways = node.hdrout === "all" + const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways) + const quoteables = [node.sep, node.quo, "\n", "\r"] + const templateQuoteables = [',', '"', "\n", "\r"] + let badTemplateWarnOnce = true + + const columnStringToTemplateArray = function (col, sep) { + // NOTE: enforce strict column template parsing in RFC4180 mode + const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true }) + if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false } + return parsed.headers.length ? parsed.headers : null + } + const templateArrayToColumnString = function (template, keepEmptyColumns) { + // NOTE: enforce strict column template parsing in RFC4180 mode + const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true }) + return keepEmptyColumns + ? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables})) + : parsed.header // exclues empty columns + // TODO: resolve inconsistency between CSV->JSON and JSON->CSV + // CSV->JSON: empty columns are excluded + // JSON->CSV: empty columns are kept in some cases + } + function addQuotes(cell, options) { + options = options || {} + return csv.quoteCell(cell, { + quote: options.quote || node.quo || '"', + separator: options.separator || node.sep || ',', + quoteables: options.quoteables || quoteables + }) + } + const hasTemplate = (t) => t?.length > 0 && !(t.length === 1 && t[0] === '') + let template + try { + template = columnStringToTemplateArray(node.template, ',') || [''] + } catch (e) { + node.warn(RED._("csv.errors.bad_template")) // is warning really necessary now we have status? + node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") }) + return // dont hook up the node + } + const noTemplate = hasTemplate(template) === false + node.hdrSent = false + + node.on("input", function (msg, send, done) { + node.status({}) // clear status + if (msg.hasOwnProperty("reset")) { + node.hdrSent = false + } + if (msg.hasOwnProperty("payload")) { + let inputData = msg.payload + if (typeof inputData == "object") { // convert object to CSV string + try { + // first determine the payload kind. Array or objects? Array of primitives? Array of arrays? Just an object? + // then, if necessary, convert to an array of objects/arrays + let isObject = !Array.isArray(inputData) && typeof inputData === 'object' + let isArrayOfObjects = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] === 'object' + let isArrayOfArrays = Array.isArray(inputData) && inputData.length > 0 && Array.isArray(inputData[0]) + let isArrayOfPrimitives = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] !== 'object' + + if (isObject) { + inputData = [inputData] + isArrayOfObjects = true + isObject = false + } else if (isArrayOfPrimitives) { + inputData = [inputData] + isArrayOfArrays = true + isArrayOfPrimitives = false + } + + const stringBuilder = [] + if (!(noTemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) { + template = columnStringToTemplateArray(node.template) || [''] + } + + // build header line + if (sendHeaders && node.hdrSent === false) { + if (hasTemplate(template) === false) { + if (msg.hasOwnProperty("columns")) { + template = columnStringToTemplateArray(msg.columns || "", ",") || [''] + } + else { + template = Object.keys(inputData[0]) || [''] } } - if (last) { newMessage.complete = true; } - send(newMessage); + stringBuilder.push(templateArrayToColumnString(template, true)) + if (sendHeadersOnce) { node.hdrSent = true } } - if (has_parts && last && len === 0) { - send({complete:true}); + + // build csv lines + for (let s = 0; s < inputData.length; s++) { + let row = inputData[s] + if (isArrayOfArrays) { + /*** row is an array of arrays ***/ + const _hasTemplate = hasTemplate(template) + const len = _hasTemplate ? template.length : row.length + const result = [] + for (let t = 0; t < len; t++) { + let cell = row[t] + if (cell === undefined) { cell = "" } + if(_hasTemplate) { + const header = template[t] + if (header) { + result[t] = addQuotes(RED.util.ensureString(cell)) + } + } else { + result[t] = addQuotes(RED.util.ensureString(cell)) + } + } + stringBuilder.push(result.join(node.sep)) + } else { + /*** row is an object ***/ + if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) { + template = columnStringToTemplateArray(msg.columns || "", ",") + } + if (hasTemplate(template) === false) { + /*** row is an object but we still don't have a template ***/ + if (badTemplateWarnOnce === true) { + node.warn(RED._("csv.errors.obj_csv")) + badTemplateWarnOnce = false + } + const rowData = [] + for (let header in inputData[0]) { + if (row.hasOwnProperty(header)) { + const cell = row[header] + if (typeof cell !== "object") { + let cellValue = "" + if (cell !== undefined) { + cellValue += cell + } + rowData.push(addQuotes(cellValue)) + } + } + } + stringBuilder.push(rowData.join(node.sep)) + } else { + /*** row is an object and we have a template ***/ + const rowData = [] + for (let t = 0; t < template.length; t++) { + if (!template[t]) { + rowData.push('') + } + else { + let cellValue = inputData[s][template[t]] + if (cellValue === undefined) { cellValue = "" } + cellValue = RED.util.ensureString(cellValue) + rowData.push(addQuotes(cellValue)) + } + } + stringBuilder.push(rowData.join(node.sep)); // add separator + } + } } + + // join lines, don't forget to add the last new line + msg.payload = stringBuilder.join(node.ret) + node.ret + msg.columns = templateArrayToColumnString(template) + if (msg.payload !== '') { send(msg) } + done() + } + catch (e) { + done(e) } - node.linecount = 0; - done(); } - catch(e) { done(e); } + else if (typeof inputData == "string") { // convert CSV string to object + try { + let firstLine = true; // is this the first line + let last = false + let linecount = 0 + const has_parts = msg.hasOwnProperty("parts") + + // determine if this is a multi part message and if so what part we are processing + if (msg.hasOwnProperty("parts")) { + linecount = msg.parts.index + if (msg.parts.index > node.skip) { firstLine = false } + if (msg.parts.hasOwnProperty("count") && (msg.parts.index + 1 >= msg.parts.count)) { last = true } + } + + // If skip is set, compute the cursor position to start parsing from + let _cursor = 0 + if (node.skip > 0 && linecount < node.skip) { + for (; _cursor < inputData.length; _cursor++) { + if (firstLine && (linecount < node.skip)) { + if (inputData[_cursor] === "\r" || inputData[_cursor] === "\n") { + linecount += 1 + } + continue + } + break + } + if (_cursor >= inputData.length) { + return // skip this line + } + } + + // count the number of line breaks in the string + const noofCR = ((_cursor ? inputData.slice(_cursor) : inputData).match(/[\r\n]/g) || []).length + + // if we have `parts` and we are outputting multiple objects and we have more than one line + // then we need to set firstLine to true so that we process the header line + if (has_parts && node.multi === "mult" && noofCR > 1) { + firstLine = true + } + + // if we are processing the first line and the node has been set to extract the header line + // update the template with the header line + if (firstLine && node.hdrin === true) { + /** @type {import('./lib/csv/index.js').CSVParseOptions} */ + const csvOptionsForHeaderRow = { + cursor: _cursor, + separator: node.sep, + quote: node.quo, + dataHasHeaderRow: true, + headersOnly: true, + outputStyle: 'array', + strict: true // enforce strict parsing of the header row + } + try { + const csvHeader = csv.parse(inputData, csvOptionsForHeaderRow) + template = csvHeader.headers + _cursor = csvHeader.cursor + } catch (e) { + // node.warn(RED._("csv.errors.bad_template")) // add warning? + node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") }) + throw e + } + } + + // now we process the data lines + /** @type {import('./lib/csv/index.js').CSVParseOptions} */ + const csvOptions = { + cursor: _cursor, + separator: node.sep, + quote: node.quo, + dataHasHeaderRow: false, + headers: hasTemplate(template) ? template : null, + outputStyle: 'object', + includeNullValues: node.include_null_values, + includeEmptyStrings: node.include_empty_strings, + parseNumeric: node.parsestrings, + strict: false // relax the strictness of the parser for data rows + } + const csvParseResult = csv.parse(inputData, csvOptions) + const data = csvParseResult.data + + // output results + if (node.multi !== "one") { + if (has_parts && noofCR <= 1) { + if (data.length > 0) { + node.store.push(...data) + } + if (msg.parts.index + 1 === msg.parts.count) { + msg.payload = node.store + msg.columns = csvParseResult.header + // msg._mode = 'RFC4180 mode' + delete msg.parts + send(msg) + node.store = [] + } + } + else { + msg.columns = csvParseResult.header + // msg._mode = 'RFC4180 mode' + msg.payload = data + send(msg); // finally send the array + } + } + else { + const len = data.length + for (let row = 0; row < len; row++) { + const newMessage = RED.util.cloneMessage(msg) + newMessage.columns = csvParseResult.header + newMessage.payload = data[row] + if (!has_parts) { + newMessage.parts = { + id: msg._msgid, + index: row, + count: len + } + } + else { + newMessage.parts.index -= node.skip + newMessage.parts.count -= node.skip + if (node.hdrin) { // if we removed the header line then shift the counts by 1 + newMessage.parts.index -= 1 + newMessage.parts.count -= 1 + } + } + if (last) { newMessage.complete = true } + // newMessage._mode = 'RFC4180 mode' + send(newMessage) + } + if (has_parts && last && len === 0) { + // send({complete:true, _mode: 'RFC4180 mode'}) + send({ complete: true }) + } + } + + node.linecount = 0 + done() + } + catch (e) { + done(e) + } + } + else { + // RFC-vs-legacy mode difference: In RFC mode, we throw catchable errors and provide a status message + const err = new Error(RED._("csv.errors.csv_js")) + node.status({ fill: "red", shape: "dot", text: err.message }) + done(err) + } } - else { node.warn(RED._("csv.errors.csv_js")); done(); } - } - else { - if (!msg.hasOwnProperty("reset")) { - node.send(msg); // If no payload and not reset - just pass it on. + else { + if (!msg.hasOwnProperty("reset")) { + node.send(msg); // If no payload and not reset - just pass it on. + } + done() } - done(); - } - }); + }) + } } - RED.nodes.registerType("csv",CSVNode); + + RED.nodes.registerType("csv",CSVNode) } diff --git a/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js new file mode 100644 index 000000000..73cf4b292 --- /dev/null +++ b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js @@ -0,0 +1,324 @@ + +/** + * @typedef {Object} CSVParseOptions + * @property {number} [cursor=0] - an index into the CSV to start parsing from + * @property {string} [separator=','] - the separator character + * @property {string} [quote='"'] - the quote character + * @property {boolean} [headersOnly=false] - only parse the headers and return them + * @property {string[]} [headers=[]] - an array of headers to use instead of the first row of the CSV data + * @property {boolean} [dataHasHeaderRow=true] - whether the CSV data to parse has a header row + * @property {boolean} [outputHeader=true] - whether the output data should include a header row (only applies to array output) + * @property {boolean} [parseNumeric=false] - parse numeric values into numbers + * @property {boolean} [includeNullValues=false] - include null values in the output + * @property {boolean} [includeEmptyStrings=true] - include empty strings in the output + * @property {string} [outputStyle='object'] - output an array of arrays or an array of objects + * @property {boolean} [strict=false] - throw an error if the CSV is malformed + */ + +/** + * Parses a CSV string into an array of arrays or an array of objects. + * + * NOTES: + * * Deviations from the RFC4180 spec (for the sake of user fiendliness, system implementations and flexibility), this parser will: + * * accept any separator character, not just `,` + * * accept any quote character, not just `"` + * * parse `\r`, `\n` or `\r\n` as line endings (RRFC4180 2.1 states lines are separated by CRLF) + * * Only single character `quote` is supported + * * `quote` is `"` by default + * * Any cell that contains a `quote` or `separator` will be quoted + * * Any `quote` characters inside a cell will be escaped as per RFC 4180 2.6 + * * Only single character `separator` is supported + * * Only `array` and `object` output styles are supported + * * `array` output style is an array of arrays [[],[],[]] + * * `object` output style is an array of objects [{},{},{}] + * * Only `headers` or `dataHasHeaderRow` are supported, not both + * @param {string} csvIn - the CSV string to parse + * @param {CSVParseOptions} parseOptions - options + * @throws {Error} + */ +function parse(csvIn, parseOptions) { + /* Normalise options */ + parseOptions = parseOptions || {}; + const separator = parseOptions.separator ?? ','; + const quote = parseOptions.quote ?? '"'; + const headersOnly = parseOptions.headersOnly ?? false; + const headers = Array.isArray(parseOptions.headers) ? parseOptions.headers : [] + const dataHasHeaderRow = parseOptions.dataHasHeaderRow ?? true; + const outputHeader = parseOptions.outputHeader ?? true; + const parseNumeric = parseOptions.parseNumeric ?? false; + const includeNullValues = parseOptions.includeNullValues ?? false; + const includeEmptyStrings = parseOptions.includeEmptyStrings ?? true; + const outputStyle = ['array', 'object'].includes(parseOptions.outputStyle) ? parseOptions.outputStyle : 'object'; // 'array [[],[],[]]' or 'object [{},{},{}] + const strict = parseOptions.strict ?? false + + /* Local variables */ + const cursorMax = csvIn.length; + const ouputArrays = outputStyle === 'array'; + const headersSupplied = headers.length > 0 + // The original regex was an "is-a-number" positive logic test. /^ *[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+ *$/i; + // Below, is less strict and inverted logic but coupled with +cast it is 13%+ faster than original regex+parsefloat + // and has the benefit of understanding hexadecimals, binary and octal numbers. + const skipNumberConversion = /^ *(\+|-0\d|0\d)/ + const cellBuilder = [] + let rowBuilder = [] + let cursor = typeof parseOptions.cursor === 'number' ? parseOptions.cursor : 0; + let newCell = true, inQuote = false, closed = false, output = []; + + /* inline helper functions */ + const finaliseCell = () => { + let cell = cellBuilder.join('') + cellBuilder.length = 0 + // push the cell: + // NOTE: if cell is empty but newCell==true, then this cell had zero chars - push `null` + // otherwise push empty string + return rowBuilder.push(cell || (newCell ? null : '')) + } + const finaliseRow = () => { + if (cellBuilder.length) { + finaliseCell() + } + if (rowBuilder.length) { + output.push(rowBuilder) + rowBuilder = [] + } + } + + /* Main parsing loop */ + while (cursor < cursorMax) { + const char = csvIn[cursor] + if (inQuote) { + if (char === quote && csvIn[cursor + 1] === quote) { + cellBuilder.push(quote) + cursor += 2; + newCell = false; + closed = false; + } else if (char === quote) { + inQuote = false; + cursor += 1; + newCell = false; + closed = true; + } else { + cellBuilder.push(char) + newCell = false; + closed = false; + cursor++; + } + } else { + if (char === separator) { + finaliseCell() + cursor += 1; + newCell = true; + closed = false; + } else if (char === quote) { + if (newCell) { + inQuote = true; + cursor += 1; + newCell = false; + closed = false; + } + else if (strict) { + throw new UnquotedQuoteError(cursor) + } else { + // not strict, keep 1 quote if the next char is not a cell/record separator + cursor++ + if (csvIn[cursor] && csvIn[cursor] !== '\n' && csvIn[cursor] !== '\r' && csvIn[cursor] !== separator) { + cellBuilder.push(char) + if (csvIn[cursor] === quote) { + cursor++ // skip the next quote + } + } + } + } else { + if (char === '\n' || char === '\r') { + finaliseRow() + if (csvIn[cursor + 1] === '\n') { + cursor += 2; + } else { + cursor++ + } + newCell = true; + closed = false; + if (headersOnly) { + break + } + } else { + if (closed) { + if (strict) { + throw new DataAfterCloseError(cursor) + } else { + cursor--; // move back to grab the previously discarded char + closed = false + } + } else { + cellBuilder.push(char) + newCell = false; + cursor++; + } + } + } + } + } + if (strict && inQuote) { + throw new ParseError(`Missing quote, unclosed cell`, cursor) + } + // finalise the last cell/row + finaliseRow() + let firstRowIsHeader = false + // if no headers supplied, generate them + if (output.length >= 1) { + if (headersSupplied) { + // headers already supplied + } else if (dataHasHeaderRow) { + // take the first row as the headers + headers.push(...output[0]) + firstRowIsHeader = true + } else { + // generate headers col1, col2, col3, etc + for (let i = 0; i < output[0].length; i++) { + headers.push("col" + (i + 1)) + } + } + } + + const finalResult = { + /** @type {String[]} headers as an array of string */ + headers: headers, + /** @type {String} headers as a comma-separated string */ + header: null, + /** @type {Any[]} Result Data (may include header row: check `firstRowIsHeader` flag) */ + data: [], + /** @type {Boolean|undefined} flag to indicate if the first row is a header row (only applies when `outputStyle` is 'array') */ + firstRowIsHeader: undefined, + /** @type {'array'|'object'} flag to indicate the output style */ + outputStyle: outputStyle, + /** @type {Number} The current cursor position */ + cursor: cursor, + } + + const quotedHeaders = [] + for (let i = 0; i < headers.length; i++) { + if (!headers[i]) { + continue + } + quotedHeaders.push(quoteCell(headers[i], { quote, separator: ',' })) + } + finalResult.header = quotedHeaders.join(',') // always quote headers and join with comma + + // output is an array of arrays [[],[],[]] + if (ouputArrays || headersOnly) { + if (!firstRowIsHeader && !headersOnly && outputHeader && headers.length > 0) { + if (output.length > 0) { + output.unshift(headers) + } else { + output = [headers] + } + firstRowIsHeader = true + } + if (headersOnly) { + delete finalResult.firstRowIsHeader + return finalResult + } + finalResult.firstRowIsHeader = firstRowIsHeader + finalResult.data = (firstRowIsHeader && !outputHeader) ? output.slice(1) : output + return finalResult + } + + // output is an array of objects [{},{},{}] + const outputObjects = [] + let i = firstRowIsHeader ? 1 : 0 + for (; i < output.length; i++) { + const rowObject = {} + let isEmpty = true + for (let j = 0; j < headers.length; j++) { + if (!headers[j]) { + continue + } + let v = output[i][j] === undefined ? null : output[i][j] + if (v === null && !includeNullValues) { + continue + } else if (v === "" && !includeEmptyStrings) { + continue + } else if (parseNumeric === true && v && !skipNumberConversion.test(v)) { + const vTemp = +v + const isNumber = !isNaN(vTemp) + if(isNumber) { + v = vTemp + } + } + rowObject[headers[j]] = v + isEmpty = false + } + // determine if this row is empty + if (!isEmpty) { + outputObjects.push(rowObject) + } + } + finalResult.data = outputObjects + delete finalResult.firstRowIsHeader + return finalResult +} + +/** + * Quotes a cell in a CSV string if necessary. Addiionally, any double quotes inside the cell will be escaped as per RFC 4180 2.6 (https://datatracker.ietf.org/doc/html/rfc4180#section-2). + * @param {string} cell - the string to quote + * @param {*} options - options + * @param {string} [options.quote='"'] - the quote character + * @param {string} [options.separator=','] - the separator character + * @param {string[]} [options.quoteables] - an array of characters that, when encountered, will trigger the application of outer quotes + * @returns + */ +function quoteCell(cell, { quote = '"', separator = ",", quoteables } = { + quote: '"', + separator: ",", + quoteables: [quote, separator, '\r', '\n'] + }) { + quoteables = quoteables || [quote, separator, '\r', '\n']; + + let doubleUp = false; + if (cell.indexOf(quote) !== -1) { // add double quotes if any quotes + doubleUp = true; + } + const quoteChar = quoteables.some(q => cell.includes(q)) ? quote : ''; + return quoteChar + (doubleUp ? cell.replace(/"/g, '""') : cell) + quoteChar; +} + +// #region Custom Error Classes +class ParseError extends Error { + /** + * @param {string} message - the error message + * @param {number} cursor - the cursor index where the error occurred + */ + constructor(message, cursor) { + super(message) + this.name = 'ParseError' + this.cursor = cursor + } +} + +class UnquotedQuoteError extends ParseError { + /** + * @param {number} cursor - the cursor index where the error occurred + */ + constructor(cursor) { + super('Quote found in the middle of an unquoted field', cursor) + this.name = 'UnquotedQuoteError' + } +} + +class DataAfterCloseError extends ParseError { + /** + * @param {number} cursor - the cursor index where the error occurred + */ + constructor(cursor) { + super('Data found after closing quote', cursor) + this.name = 'DataAfterCloseError' + } +} + +// #endregion + +exports.parse = parse +exports.quoteCell = quoteCell +exports.ParseError = ParseError +exports.UnquotedQuoteError = UnquotedQuoteError +exports.DataAfterCloseError = DataAfterCloseError diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index a7b583878..323ef06a5 100644 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -849,7 +849,13 @@ "newline": "Newline", "usestrings": "parse numerical values", "include_empty_strings": "include empty strings", - "include_null_values": "include null values" + "include_null_values": "include null values", + "spec": "Parser" + }, + "spec": { + "rfc": "RFC4180", + "legacy": "Legacy", + "legacy_warning": "Legacy mode will be removed in a future release." }, "placeholder": { "columns": "comma-separated column names" @@ -878,6 +884,7 @@ "once": "send headers once, until msg.reset" }, "errors": { + "bad_template": "Malformed columns template.", "csv_js": "This node only handles CSV strings or js objects.", "obj_csv": "No columns template specified for object -> CSV.", "bad_csv": "Malformed CSV data - output probably corrupt." diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html index baa3b036b..56b6d7cca 100644 --- a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html +++ b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html @@ -36,7 +36,9 @@

Details

The column template can contain an ordered list of column names. When converting CSV to an object, the column names - will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.

+ will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV. +

When the RFC parser is selected, the column template must be compliant with RFC4180.

+

When converting to CSV, the columns template is used to identify which properties to extract from the object and in what order.

If the columns template is blank then you can use a simple comma separated list of properties supplied in msg.columns to determine what to extract and in what order. If neither are present then all the object properties are output in the order @@ -49,4 +51,5 @@

If outputting multiple messages they will have their parts property set and form a complete message sequence.

If the node is set to only send column headers once, then setting msg.reset to any value will cause the node to resend the headers.

Note: the column template must be comma separated - even if a different separator is chosen for the data.

+

Note: in RFC mode, catchable errors will be thrown for malformed CSV headers and invalid input payload data

diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js index 681711b3b..f362ecdbf 100644 --- a/test/nodes/core/parsers/70-CSV_spec.js +++ b/test/nodes/core/parsers/70-CSV_spec.js @@ -15,12 +15,15 @@ * limitations under the License. **/ -// var should = require("should"); -var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js"); -var helper = require("node-red-node-test-helper"); +// const should = require("should"); +const csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js"); +const statusNode = require("nr-test-utils").require("@node-red/nodes/core/common/25-status.js"); +const functionNode = require("nr-test-utils").require("@node-red/nodes/core/function/10-function.js"); +const delayNode = require("nr-test-utils").require("@node-red/nodes/core/function/89-delay.js"); +const helper = require("node-red-node-test-helper"); // const { neq } = require("semver"); -describe('CSV node', function() { +describe('CSV node (Legacy Mode)', function() { before(function(done) { helper.startServer(done); @@ -38,16 +41,20 @@ describe('CSV node', function() { var flow = [{id:"csvNode1", type:"csv", name: "csvNode" }]; helper.load(csvNode, flow, function() { var n1 = helper.getNode("csvNode1"); - n1.should.have.property('name', 'csvNode'); - n1.should.have.property('template',''); - n1.should.have.property('sep', ','); - n1.should.have.property('quo', '"'); - n1.should.have.property('ret', '\n'); - n1.should.have.property('winflag', false); - n1.should.have.property('lineend', '\n'); - n1.should.have.property('multi', 'one'); - n1.should.have.property('hdrin', false); - done(); + try { + n1.should.have.property('name', 'csvNode'); + n1.should.have.property('template',''); + n1.should.have.property('sep', ','); + n1.should.have.property('quo', '"'); + n1.should.have.property('ret', '\n'); + // n1.should.have.property('winflag', false); + // n1.should.have.property('lineend', '\n'); + n1.should.have.property('multi', 'one'); + n1.should.have.property('hdrin', false); + done(); + } catch (error) { + done(error); + } }); }); @@ -77,10 +84,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); - msg.should.have.property('columns', "a,b,c,d"); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); + msg.should.have.property('columns', "a,b,c,d"); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -94,10 +104,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); - msg.should.have.property('columns', "col1,col2,col3,col4"); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); + msg.should.have.property('columns', "col1,col2,col3,col4"); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1|2|3|4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -111,10 +124,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { A: 1, B: 2, D: 4 }); - msg.should.have.property('columns', "A,B,D"); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { A: 1, B: 2, D: 4 }); + msg.should.have.property('columns', "A,B,D"); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1\t2\t3\t4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -128,10 +144,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { A: 1, B: 2, D: 4 }); - msg.should.have.property('columns', "A,B,D"); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { A: 1, B: 2, D: 4 }); + msg.should.have.property('columns', "A,B,D"); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = "1 2 3 4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -145,9 +163,11 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -161,10 +181,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); - msg.should.have.property('columns', "col1,col2,col3,col4"); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); + msg.should.have.property('columns', "col1,col2,col3,col4"); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -178,10 +201,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, d: 4 }); - msg.should.have.property('columns', 'a,d'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, d: 4 }); + msg.should.have.property('columns', 'a,d'); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -195,10 +221,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); - msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -212,10 +240,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); - msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = 'a,b b,"c,c"," d, d "'+"\n"+"1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -229,10 +259,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, "d, d": 4 }); - msg.should.have.property('columns', 'a,b b,c;c,"d, d"'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, "b b":2, "c;c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,c;c,"d, d"'); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = 'a;b b;"c;c";" d, d "'+"\n"+"1;2;3;4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -246,10 +278,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, "b b":2, "c/c":3, "d, d": 4 }); - msg.should.have.property('columns', 'a,b b,c/c,"d, d"'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, "b b":2, "c/c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,c/c,"d, d"'); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = 'a/b b/"c/c"/" d, d "'+"\n"+"1/2/3/4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -263,10 +297,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, "b b":2, "c\\c":3, "d, d": 4 }); - msg.should.have.property('columns', 'a,b b,c\\c,"d, d"'); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1, "b b":2, "c\\c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,c\\c,"d, d"'); + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } }); var testString = 'a\\b b\\"c\\c"\\" d, d "'+"\n"+"1\\2\\3\\4"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -280,9 +316,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 123, b: "0123", c: '+123', d: 'e123', e: 'E123', f: -123 }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 123, b: "0123", c: '+123', d: 'e123', e: 'E123', f: -123 }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '123,0123,+123,e123,E123,-123'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -296,9 +335,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: "1.23", b: "0123", c: "+123", d: "e123", e: "0", f: "-123", g: "1e3" }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: "1.23", b: "0123", c: "+123", d: "e123", e: "0", f: "-123", g: "1e3" }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '1.23,0123,+123,e123,0,-123,1e3'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -312,9 +354,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1.23, b: -123, c: 1000, d: 0 }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 1.23, b: -123, c: 1000, d: 0 }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = ' 1.23 , -123,1e3 , 0 '+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -328,9 +373,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 12000, b: 0.012, c: -12000, d: -0.012 }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 12000, b: 0.012, c: -12000, d: -0.012 }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '12E3,12e-3,-12e3,-12E-3'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -345,29 +393,36 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - //console.log(msg); - msg.should.have.property('payload', { a:1, b:-2, c:'+3', d:'04', f:'-05', g:'ab"cd', h:'with,a,comma' }); - check_parts(msg, 0, 1); - done(); + try { + //console.log(msg); + msg.should.have.property('payload', { a:1, b:-2, c:'+3', d:'04', f:'-05', g:'ab"cd', h:'with,a,comma' }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '"1","-2","+3","04","","-05","ab""cd","with,a,comma"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); }); }); - it('should allow blank strings in the input if selected', function(done) { - var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", include_empty_strings:true, wires:[["n2"]] }, + it('should allow blank strings in the input if selected', async function() { + const flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", include_empty_strings:true, wires:[["n2"]] }, {id:"n2", type:"helper"} ]; - helper.load(csvNode, flow, function() { - var n1 = helper.getNode("n1"); - var n2 = helper.getNode("n2"); - n2.on("input", function(msg) { - //console.log(msg); - msg.should.have.property('payload', { a: 1, b: '', c: '', d: '', e: '-05', f: 'ab"cd', g: 'with,a,comma' }); - //check_parts(msg, 0, 1); - done(); + await helper.load(csvNode, flow) + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + await new Promise((resolve, reject) => { + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, b: '', c: '', d: '', e: '-05', f: 'ab"cd', g: 'with,a,comma' }); + //check_parts(msg, 0, 1); + resolve() + } catch (err) { + reject(err); + } }); - var testString = '"1","","","","-05","ab""cd","with,a,comma"'+String.fromCharCode(10); + const testString = '"1","","","","-05","ab""cd","with,a,comma"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); }); }); @@ -379,10 +434,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - //console.log(msg); - msg.should.have.property('payload', { a: 1, b: null, c: '+3', d: null, e: '-05', f: 'ab"cd', g: 'with,a,comma' }); - //check_parts(msg, 0, 1); - done(); + try { + //console.log(msg); + msg.should.have.property('payload', { a: 1, b: null, c: '+3', d: null, e: '-05', f: 'ab"cd', g: 'with,a,comma' }); + // check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '"1",,"+3",,"-05","ab""cd","with,a,comma"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -396,10 +454,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - //console.log(msg); - msg.should.have.property('payload', { a: "with a\nnew line", b: "and a\rcarriage return", c: "and why\r\nnot both"}); - check_parts(msg, 0, 1); - done(); + try { + //console.log(msg); + msg.should.have.property('payload', { a: "with a\nnew line", b: "and a\rcarriage return", c: "and why\r\nnot both"}); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = '"with a'+String.fromCharCode(10)+'new line","and a'+String.fromCharCode(13)+'carriage return","and why\r\nnot both"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -414,16 +475,19 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - if (c == 0) { - c = 1; - msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\n" }); - check_parts(msg, 0, 1); - } - else { - msg.should.have.property('payload', { a: "this is", b: "a normal", c: "line" }); - check_parts(msg, 0, 1); - done(); + try { + if (c == 0) { + c = 1; + msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\n" }); + check_parts(msg, 0, 1); + } + else { + msg.should.have.property('payload', { a: "this is", b: "a normal", c: "line" }); + check_parts(msg, 0, 1); + done(); + } } + catch(e) { done(e); } }); var testString = '"with,a"n,odd","num"ber","of"qu"ot"es"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -439,17 +503,20 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - //console.log(msg) - if (c == 0) { - c = 1; - msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\nthis is,a normal,line" }); - check_parts(msg, 0, 1); - } - else { - msg.should.have.property('payload', { a: "this is", b: "another", c: "line" }); - check_parts(msg, 0, 1); - done(); + try { + //console.log(msg) + if (c == 0) { + c = 1; + msg.should.have.property('payload', { a: "with,an", b: "odd,number", c: "ofquotes\nthis is,a normal,line\n" }); + check_parts(msg, 0, 1); + } + else { + msg.should.have.property('payload', { a: "this is", b: "another", c: "line" }); + check_parts(msg, 0, 1); + done(); + } } + catch(e) { done(e); } }); var testString = '"with,a"n,odd","num"ber","of"qu"ot"es"'+String.fromCharCode(10)+'"this is","a normal","line"'+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -465,17 +532,20 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - //console.log(msg); - if (c === 0) { - msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); - check_parts(msg, 0, 2); - c += 1; - } - else { - msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); - check_parts(msg, 1, 2); - done(); + try { + //console.log(msg); + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + check_parts(msg, 1, 2); + done(); + } } + catch(e) { done(e); } }); var testString = "w,x,y,z\n1,2,3,4\n\n5,6,7,8"; n1.emit("input", {payload:testString}); @@ -489,10 +559,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', [ { a: 1, b: 2, c: 3, d: 4 },{ a: 5, b: -6, c: '07', d: '+8' },{ a: 9, b: 0, c: 'a', d: 'b' },{ a: 'c', b: 'd', c: 'e', d: 'f' } ]); - msg.should.have.property('columns','a,b,c,d'); - msg.should.not.have.property('parts'); - done(); + try { + msg.should.have.property('payload', [ { a: 1, b: 2, c: 3, d: 4 },{ a: 5, b: -6, c: '07', d: '+8' },{ a: 9, b: 0, c: 'a', d: 'b' },{ a: 'c', b: 'd', c: 'e', d: 'f' } ]); + msg.should.have.property('columns','a,b,c,d'); + msg.should.not.have.property('parts'); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4\n5,-6,07,+8\n9,0,a,b\nc,d,e,f"; n1.emit("input", {payload:testString}); @@ -506,10 +579,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', [{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6},{"a":7,"b":8,"c":9}]); - msg.should.have.property('columns','a,b,c'); - msg.should.not.have.property('parts'); - done(); + try { + msg.should.have.property('payload', [{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6},{"a":7,"b":8,"c":9}]); + msg.should.have.property('columns','a,b,c'); + msg.should.not.have.property('parts'); + done(); + } + catch(e) { done(e); } }); n1.emit("input", {"payload":"a,b,c","parts":{"index":0,"ch":"\n","type":"string","id":"1"}}); @@ -526,10 +602,13 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', [{"Col1":"V1","Col2":"V2"},{"Col1":"V3","Col2":"V4"},{"Col1":"V5","Col2":"V6"}]); - msg.should.have.property('columns','Col1,Col2'); - msg.should.have.property('parts'); - done(); + try { + msg.should.have.property('payload', [{"Col1":"V1","Col2":"V2"},{"Col1":"V3","Col2":"V4"},{"Col1":"V5","Col2":"V6"}]); + msg.should.have.property('columns','Col1,Col2'); + msg.should.have.property('parts'); + done(); + } + catch(e) { done(e); } }); //var testString = "1,2,3,4\n5,-6,07,+8\n9,0,a,b\nc,d,e,f"; // n1.emit("input", {payload:testString}); @@ -545,9 +624,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: "a", b: "127.0.0.1", c: 56.7, d: -32.8, e: "+76.22C" }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: "a", b: "127.0.0.1", c: 56.7, d: -32.8, e: "+76.22C" }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "a,127.0.0.1,56.7,-32.8,+76.22C"; n1.emit("input", {payload:testString}); @@ -561,9 +643,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); - check_parts(msg, 3, 4); - done(); + try { + msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); + check_parts(msg, 3, 4); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10); n1.emit("input", {payload:testString, parts: {id:"X", index:3, count:4} }); @@ -578,16 +663,19 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - if (c === 0) { - msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); - check_parts(msg, 0, 2); - c += 1; - } - else { - msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); - check_parts(msg, 1, 2); - done(); + try { + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + check_parts(msg, 1, 2); + done(); + } } + catch(e) { done(e); } }); var testString1 = "w,x,y,z\n"; var testString2 = "1,2,3,4\n"; @@ -605,9 +693,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -621,9 +712,12 @@ describe('CSV node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - msg.should.have.property('payload', { "9": "C", "0": "D", "A": "E", "B": "F" }); - check_parts(msg, 0, 1); - done(); + try { + msg.should.have.property('payload', { "9": "C", "0": "D", "A": "E", "B": "F" }); + check_parts(msg, 0, 1); + done(); + } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -638,16 +732,19 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - if (c===0) { - msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); - check_parts(msg, 0, 2); - c = c+1; - } - else { - msg.should.have.property('payload', { a: "C", b: "D", c: "E", d: "F" }); - check_parts(msg, 1, 2); - done(); + try { + if (c===0) { + msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); + check_parts(msg, 0, 2); + c = c+1; + } + else { + msg.should.have.property('payload', { a: "C", b: "D", c: "E", d: "F" }); + check_parts(msg, 1, 2); + done(); + } } + catch(e) { done(e); } }); var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10); n1.emit("input", {payload:testString}); @@ -662,18 +759,21 @@ describe('CSV node', function() { var n2 = helper.getNode("n2"); var c = 0; n2.on("input", function(msg) { - if (c === 0) { - msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); - msg.should.have.property('columns', 'w,x,y,z'); - check_parts(msg, 0, 2); - c += 1; - } - else { - msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); - msg.should.have.property('columns', 'w,x,y,z'); - check_parts(msg, 1, 2); - done(); + try { + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + msg.should.have.property('columns', 'w,x,y,z'); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + msg.should.have.property('columns', 'w,x,y,z'); + check_parts(msg, 1, 2); + done(); + } } + catch(e) { done(e); } }); var testStringA = "foo\n"; var testStringB = "bar\n"; @@ -1089,3 +1189,1257 @@ describe('CSV node', function() { }); }); }); + +describe('CSV node (RFC Mode)', function () { + + before(function (done) { + helper.startServer(done); + }); + + after(function (done) { + helper.stopServer(done); + }); + + afterEach(function () { + helper.unload(); + }); + + it('should be loaded with defaults', function (done) { + // RFC-Legacy difference + // In RFC mode, the default line separator is \r\n (RFC4180 Section 2.1) + // In Legacy mode, the line separator is set to \n + const flow = [{ id: "n1", type: "csv", spec: "rfc", name: "csvNode" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + try { + n1.should.have.property('name', 'csvNode'); + n1.should.have.property('template', ''); + n1.should.have.property('sep', ','); + n1.should.have.property('quo', '"'); + n1.should.have.property('ret', '\r\n'); // RFC-Legacy difference + n1.should.have.property('multi', 'one'); + n1.should.have.property('hdrin', false); + done(); + } catch (error) { + done(error); + } + }); + }); + + describe('csv to json', function () { + let parts_id = undefined; + + afterEach(function () { + parts_id = undefined; + }); + + function check_parts(msg, index, count) { + msg.should.have.property('parts'); + if (parts_id === undefined) { + parts_id = msg.parts.id; + } + else { + msg.parts.should.have.property('id', parts_id); + } + msg.parts.should.have.property('index', index); + msg.parts.should.have.property('count', count); + } + + it('should convert a simple csv string to a javascript object', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); + msg.should.have.property('columns', "a,b,c,d"); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should convert a simple string to a javascript object with | separator (no template)', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", sep: "|", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); + msg.should.have.property('columns', "col1,col2,col3,col4"); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1|2|3|4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should convert a simple string to a javascript object with tab separator (with template)', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", sep: "\t", temp: "A,B,,D", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { A: 1, B: 2, D: 4 }); + msg.should.have.property('columns', "A,B,D"); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1\t2\t3\t4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should convert a simple string to a javascript object with space separator (with spaced template)', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + // In legacy mode, the template "A, B, , D" is trimmed to "A,B,D" + const flow = [{ id: "n1", type: "csv", spec: "rfc", sep: " ", temp: "A, B, , D", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + // 'payload', { A: 1, B: 2, D: 4 } // Legacy + msg.should.have.property('payload', { A: 1, ' B': 2, ' ': 3, ' D': 4 }); // RFC + // 'columns', "A,B,D" // Legacy + msg.should.have.property('columns', "A, B, , D"); // RFC + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = "1 2 3 4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should not remove quotes and whitespace from template - should set status and send warning', async function () { + // RFC-vs-Legacy difference + // In RFC mode, the column template is parsed in strict mode + // meaning this template is invalid because: + // * it encounters a quote in an unquoted field (the "b" in the 2nd column) + // * The 2nd column starts with a space but then a single quote is encountered. RFC4180 Section 2.6 + // says Fields containing line breaks (CRLF), double quotes, and commas should be enclosed in double-quotes + // * it contains a data between the closing quote and separator (4th column starts ok but then has a space after the closing quote) + // * Since we adhere to RFC4180 Section 2.4, "Spaces are considered part of a field and should not be ignored" + // we must treat the space as part of the data meaning field is not correctly close with a quote + const flow = [ + { id: "n1", type: "csv", spec: "rfc", temp: '', ret: '\n', wires: [["n2"]] }, + { id: "n2", type: "helper" } + ]; + await helper.load(csvNode, flow) + + const n1 = helper.getNode("n1") + const n2 = helper.getNode("n2") + + //modify the flow and deploy change that sets the csv node to a bad template - thus updating the nodes status and logging a warning + const newConfig = flow.map(n => ({ ...n })); + newConfig[0].temp = '"a", "b" , " c "," d " '; + await helper.setFlows(newConfig, "nodes") // deploy "nodes" only + + // small sleep for msg flow and logging to complete + await new Promise(resolve => setTimeout(resolve, 50)); + + // check that status message was sent + n1.status.called.should.be.true(); + n1.status.lastCall.args[0].should.deepEqual({ fill: 'red', shape: 'dot', text: 'csv.errors.bad_template' }); + + // warn message should be logged + n1.warn.called.should.be.true(); + n1.warn.lastCall.args[0].should.equal('csv.errors.bad_template'); + const logs = helper.log().args.filter(function (evt) { + return evt[0].type == "csv"; + }); + logs.should.have.length(1); + logs[0][0].should.have.a.property('msg'); + logs[0][0].msg.should.equal('csv.errors.bad_template'); + }); + + it('should create column names if no template provided', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: '', wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 }); + msg.should.have.property('columns', "col1,col2,col3,col4"); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow dropping of fields from the template', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,,,d", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, d: 4 }); + msg.should.have.property('columns', 'a,d'); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow commas and spaces in the template', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + // In legacy mode, the template `a,b b,\"c,c\",\" d, d \"` is trimmed to `a,b b,"c,c","d, d"` + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b b,\"c,c\",\" d, d \"", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + // 'payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 } // Legacy + msg.should.have.property('payload', { a: 1, "b b": 2, "c,c": 3, " d, d ": 4 }); // RFC-vs-Legacy difference + // 'columns', 'a,b b,"c,c","d, d"' // Legacy + msg.should.have.property('columns', 'a,b b,"c,c"," d, d "'); // RFC-vs-Legacy difference + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow passing in a template as first line of CSV', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, "b b": 2, "c,c": 3, " d, d ": 4 }); // RFC-vs-Legacy difference + msg.should.have.property('columns', 'a,b b,"c,c"," d, d "'); // RFC-vs-Legacy difference + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = 'a,b b,"c,c"," d, d "' + "\n" + "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow passing in a template as first line of CSV (not comma)', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, sep: ";", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, "b b": 2, "c;c": 3, " d, d ": 4 }); // RFC-vs-Legacy difference + msg.should.have.property('columns', 'a,b b,c;c," d, d "'); // RFC-vs-Legacy difference + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = 'a;b b;"c;c";" d, d "' + "\n" + "1;2;3;4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow passing in a template as first line of CSV (special char /)', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, sep: "/", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, "b b": 2, "c/c": 3, " d, d ": 4 }); // RFC-vs-Legacy difference + msg.should.have.property('columns', 'a,b b,c/c," d, d "'); // RFC-vs-Legacy difference + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = 'a/b b/"c/c"/" d, d "' + "\n" + "1/2/3/4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow passing in a template as first line of CSV (special char \\)', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, spaces are respected in the template (RFC4180 Section 2.4) + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, sep: "\\", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, "b b": 2, "c\\c": 3, " d, d ": 4 }); // RFC-vs-Legacy difference + msg.should.have.property('columns', 'a,b b,c\\c," d, d "'); // RFC-vs-Legacy difference + check_parts(msg, 0, 1); + done(); + } catch (e) { done(e); } + }); + const testString = 'a\\b b\\"c\\c"\\" d, d "' + "\n" + "1\\2\\3\\4" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should leave numbers starting with 0, e and + as strings (except 0.)', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 123, b: "0123", c: '+123', d: 'e123', e: 'E123', f: -123 }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '123,0123,+123,e123,E123,-123' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should not parse numbers when told not to do so', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", strings: false, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: "1.23", b: "0123", c: "+123", d: "e123", e: "0", f: "-123", g: "1e3" }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '1.23,0123,+123,e123,0,-123,1e3' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should parse numbers when told to do so', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1.23, b: -123, c: 1000, d: 0 }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = ' 1.23 , -123,1e3 , 0 ' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should leave handle strings with scientific notation as numbers', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 12000, b: 0.012, c: -12000, d: -0.012 }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '12E3,12e-3,-12e3,-12E-3' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + + it('should allow quotes in the input (but drop blank strings)', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g,h", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + //console.log(msg); + msg.should.have.property('payload', { a: 1, b: -2, c: '+3', d: '04', f: '-05', g: 'ab"cd', h: 'with,a,comma' }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '"1","-2","+3","04","","-05","ab""cd","with,a,comma"' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow blank strings in the input if selected', async function () { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", include_empty_strings: true, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + await helper.load(csvNode, flow) + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + await new Promise((resolve, reject) => { + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, b: '', c: '', d: '', e: '-05', f: 'ab"cd', g: 'with,a,comma' }); + resolve() + } catch (err) { + reject(err); + } + }); + const testString = '"1","","","","-05","ab""cd","with,a,comma"' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should allow missing columns (nulls) in the input if selected', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", include_null_values: true, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + //console.log(msg); + msg.should.have.property('payload', { a: 1, b: null, c: '+3', d: null, e: '-05', f: 'ab"cd', g: 'with,a,comma' }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '"1",,"+3",,"-05","ab""cd","with,a,comma"' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should handle cr and lf in the input', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + //console.log(msg); + msg.should.have.property('payload', { a: "with a\nnew line", b: "and a\rcarriage return", c: "and why\r\nnot both" }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = '"with a' + String.fromCharCode(10) + 'new line","and a' + String.fromCharCode(13) + 'carriage return","and why\r\nnot both"' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should recover from an odd number of quotes in the input', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, the column template is parsed in strict mode + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let c = 0; + n2.on("input", function (msg) { + try { + if (c == 0) { + c = 1; + // 'payload', { a: "with,an", b: "odd,number", c: "ofquotes\n" } // Legacy + msg.should.have.property('payload', { a: 'with,a"n', b: 'odd', c: 'num"ber', d: 'of"qu"ot"es' }); // RFC-vs-Legacy difference + // the result in this bad input data case as defined in RFC4180 would be to fail the parse (strict mode does this, but we don't enforce it in the data) + // However, to better parse _acceptably_ bad data like a,b"b,c'c,"d,d" (which the newer RFC mode parser correctly parses as a,b"b,c'c,"d,d") we would + // need specific code to handle this case and that code would be on the hot path for all data. + // This feels like an acceptable trade-off especially as the legacy mode parser is to be kept for backwards compatibility (until we can remove it) + check_parts(msg, 0, 1); + } + else { + msg.should.have.property('payload', { a: "this is", b: "a normal", c: "line" }); + check_parts(msg, 0, 1); + done(); + } + } + catch (e) { done(e); } + }); + const testString = '"with,a"n,odd","num"ber","of"qu"ot"es"' + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + n1.emit("input", { payload: '"this is","a normal","line"' }); + }); + }); + + it('should handle newlines in the input data', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e,f,g", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + + n2.on("input", function (msg) { + try { + //console.log(msg) + // input => 'ay,be,"c has 2\nnew\nlines",dee,eee,eff,gee' + msg.should.have.property('payload', { a: 'ay', b: 'be', c: 'c has 2\nnew\nlines', d: 'dee', e: 'eee', f: 'eff', g: 'gee' }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + + n1.emit("input", { payload: 'ay,be,"c has 2\nnew\nlines",dee,eee,eff,gee' }); + }); + }); + + it('should be able to use the first line as a template', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", hdrin: true, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let c = 0; + n2.on("input", function (msg) { + try { + //console.log(msg); + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + check_parts(msg, 1, 2); + done(); + } + } + catch (e) { done(e); } + }); + const testString = "w,x,y,z\n1,2,3,4\n\n5,6,7,8"; + n1.emit("input", { payload: testString }); + }); + }); + + it('should be able to output multiple lines as one array', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", multi: "yes", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', [{ a: 1, b: 2, c: 3, d: 4 }, { a: 5, b: -6, c: '07', d: '+8' }, { a: 9, b: 0, c: 'a', d: 'b' }, { a: 'c', b: 'd', c: 'e', d: 'f' }]); + msg.should.have.property('columns', 'a,b,c,d'); + msg.should.not.have.property('parts'); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4\n5,-6,07,+8\n9,0,a,b\nc,d,e,f"; + n1.emit("input", { payload: testString }); + }); + }); + + it('should be able to create an array from multiple parts', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, multi: "mult", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', [{ "a": 1, "b": 2, "c": 3 }, { "a": 4, "b": 5, "c": 6 }, { "a": 7, "b": 8, "c": 9 }]); + msg.should.have.property('columns', 'a,b,c'); + msg.should.not.have.property('parts'); + done(); + } + catch (e) { done(e); } + }); + + n1.emit("input", { "payload": "a,b,c", "parts": { "index": 0, "ch": "\n", "type": "string", "id": "1" } }); + n1.emit("input", { "payload": "1,2,3", "parts": { "index": 1, "ch": "\n", "type": "string", "id": "1" } }); + n1.emit("input", { "payload": "4,5,6", "parts": { "index": 2, "ch": "\n", "type": "string", "id": "1" } }); + n1.emit("input", { "payload": "7,8,9", "parts": { "index": 3, count: 4, "ch": "\n", "type": "string", "id": "1" } }); + }); + }); + + it('should be able to output multiple objects as an array from an input of parts', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, multi: "yes", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', [{ "Col1": "V1", "Col2": "V2" }, { "Col1": "V3", "Col2": "V4" }, { "Col1": "V5", "Col2": "V6" }]); + msg.should.have.property('columns', 'Col1,Col2'); + msg.should.have.property('parts'); + done(); + } + catch (e) { done(e); } + }); + //var testString = "1,2,3,4\n5,-6,07,+8\n9,0,a,b\nc,d,e,f"; + // n1.emit("input", {payload:testString}); + n1.emit("input", { "payload": "Col1,Col2\nV1,V2\nV3,V4\nV5,V6", "topic": "", "parts": { "id": "3af07e18.865652", "type": "array", "count": 2, "len": 1, "index": 0 } }); + //n1.emit("input", {"payload":"Var1,Var2\nW1,W2\nW3,W4\nW5,W6","topic":"","parts":{"id":"3af07e18.865652","type":"array","count":2,"len":1,"index":1}}); + }); + }); + + it('should handle numbers in strings but not IP addresses', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d,e", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: "a", b: "127.0.0.1", c: 56.7, d: -32.8, e: "+76.22C" }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "a,127.0.0.1,56.7,-32.8,+76.22C"; + n1.emit("input", { payload: testString }); + }); + }); + + it('should preserve parts property', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 }); + check_parts(msg, 3, 4); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10); + n1.emit("input", { payload: testString, parts: { id: "X", index: 3, count: 4 } }); + }); + }); + + it('should be able to use the first of multiple parts as a template if parts are present', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let c = 0; + n2.on("input", function (msg) { + try { + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + check_parts(msg, 1, 2); + done(); + } + } + catch (e) { done(e); } + }); + const testString1 = "w,x,y,z\n"; + const testString2 = "1,2,3,4\n"; + const testString3 = "5,6,7,8\n"; + n1.emit("input", { payload: testString1, parts: { id: "X", index: 0, count: 3 } }); + n1.emit("input", { payload: testString2, parts: { id: "X", index: 1, count: 3 } }); + n1.emit("input", { payload: testString3, parts: { id: "X", index: 2, count: 3 } }); + }); + }); + + it('should skip several lines from start if requested', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", skip: 2, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10) + "5,6,7,8" + String.fromCharCode(10) + "9,0,A,B" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should skip several lines from start then use next line as a template', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", hdrin: true, skip: 2, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', { "9": "C", "0": "D", "A": "E", "B": "F" }); + check_parts(msg, 0, 1); + done(); + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10) + "5,6,7,8" + String.fromCharCode(10) + "9,0,A,B" + String.fromCharCode(10) + "C,D,E,F" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should skip several lines from start and correct parts', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", skip: 2, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let c = 0; + n2.on("input", function (msg) { + try { + if (c === 0) { + msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" }); + check_parts(msg, 0, 2); + c = c + 1; + } + else { + msg.should.have.property('payload', { a: "C", b: "D", c: "E", d: "F" }); + check_parts(msg, 1, 2); + done(); + } + } + catch (e) { done(e); } + }); + const testString = "1,2,3,4" + String.fromCharCode(10) + "5,6,7,8" + String.fromCharCode(10) + "9,0,A,B" + String.fromCharCode(10) + "C,D,E,F" + String.fromCharCode(10); + n1.emit("input", { payload: testString }); + }); + }); + + it('should be able to skip and then use the first of multiple parts as a template if parts are present', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrin: true, skip: 2, wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let c = 0; + n2.on("input", function (msg) { + try { + if (c === 0) { + msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 }); + msg.should.have.property('columns', 'w,x,y,z'); + check_parts(msg, 0, 2); + c += 1; + } + else { + msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 }); + msg.should.have.property('columns', 'w,x,y,z'); + check_parts(msg, 1, 2); + done(); + } + } + catch (e) { done(e); } + }); + const testStringA = "foo\n"; + const testStringB = "bar\n"; + const testString1 = "w,x,y,z\n"; + const testString2 = "1,2,3,4\n"; + const testString3 = "5,6,7,8\n"; + n1.emit("input", { payload: testStringA, parts: { id: "X", index: 0, count: 5 } }); + n1.emit("input", { payload: testStringB, parts: { id: "X", index: 1, count: 5 } }); + n1.emit("input", { payload: testString1, parts: { id: "X", index: 2, count: 5 } }); + n1.emit("input", { payload: testString2, parts: { id: "X", index: 3, count: 5 } }); + n1.emit("input", { payload: testString3, parts: { id: "X", index: 4, count: 5 } }); + }); + }); + + }); + + describe('json object to csv', function () { + + it('should convert a simple object back to a csv', function (done) { + // RFC-vs-Legacy difference + // By default, the legacy mode parser will use \n as the line separator + // The RFC mode parser will use \r\n as the line separator as per RFC4180 Section 2.1 + // This is configurable in both modes meaning that the legacy mode parser can be made to use \r\n and the RFC mode parser can be made to use \n + // Any existing flows will still use the line ending that they were created with as it will be stored in the flow file + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,,e,f,g,h,i,j,k", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + // console.log("GOT",msg) + try { + // 'payload', '4,foo,true,,0,"Hello\nWorld",,,undefined,null,null\n'); // Legacy + msg.should.have.property('payload', '4,foo,true,,0,"Hello\nWorld",,,undefined,null,null\r\n'); // RFC-vs-Legacy difference + done(); + } catch (e) { + done(e); + } + }); + const testJson = { e: 0, d: 1, b: "foo", c: true, a: 4, f: "Hello\nWorld", h: undefined, i: "undefined", j: null, k: "null" }; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert a simple object back to a csv with no template', function (done) { + // RFC-vs-Legacy differences + // line separator handling: + // By default, the legacy mode parser will use \n as the line separator + // The RFC mode parser will use \r\n as the line separator as per RFC4180 Section 2.1 + // spaces: + // By default, the legacy mode parser will trim spaces from the start and end template columns + // The RFC mode parser will not trim spaces from the start and end template columns (RFC4180 Section 2.4) + // This means the original test flow, when ran in RFC mode, was expecting to find a prop name of " " in the input data object + // To make this test work, we need to change the column template to not be a space! + + // flow = [ { id:"n1", type:"csv", spec: "rfc", temp:" ", wires:[["n2"]] }, // Legacy + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", wires: [["n2"]] }, // RFC-vs-Legacy difference - legacy mode trims spaces, RFC mode respects them. + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + // console.log("GOT",msg) + try { + msg.should.have.property('payload', '1,foo,"ba""r","di,ng",,undefined,null\r\n'); + done(); + } catch (e) { + done(e); + } + }); + const testJson = { d: 1, b: "foo", c: "ba\"r", a: "di,ng", e: undefined, f: "undefined", g: null, h: "null" }; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert a simple object back to a tsv using a tab as a separator', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", sep: "\t", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', '1\tfoo\t"ba""r"\tdi,ng\n'); + done(); + } catch (e) { + done(e); + } + }); + const testJson = { d: 1, b: "foo", c: "ba\"r", a: "di,ng" }; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should handle a template with spaces in the property names', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b o,c p,,e", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', '4,foo,true,,0\n'); + done(); + } catch (e) { + done(e); + } + }); + const testJson = { e: 0, d: 1, "b o": "foo", "c p": true, a: 4 }; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should handle a template with quotes in the property names', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrout: "all", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + // 'payload', 'a"a,b\'b\nA1,B1\nA2,B2\n'); // Legacy + msg.should.have.property('payload', '"a""a",b\'b\nA1,B1\nA2,B2\n'); // RFC-vs-Legacy difference - RFC4180 Section 2.6, 2.7 quote handling + done(); + } catch (e) { + done(e); + } + }); + const testJson = [ + { + "a\"a": "A1", + "b'b": "B1" + }, + { + "a\"a": "A2", + "b'b": "B2" + } + ] + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert an array of objects to a multi-line csv', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,d,c,b", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', '4,1,2,3\n1,4,3,2\n'); + done(); + } catch (e) { + done(e); + } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }, { d: 4, a: 1, c: 3, b: 2 }]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert an array of objects to a multi-line csv and add a header', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", hdrout: "all", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'a,b,c,d\n4,3,2,1\n1,2,3,"a\nb"\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }, { d: "a\nb", a: 1, c: 3, b: 2 }]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert an array of objects to a multi-line csv without a template', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', '1,3,2,4\n4,2,3,1\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }, { d: 4, a: 1, c: 3, b: 2 }]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert an array of objects to a multi-line csv without a template and with a header', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrout: "all", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'd,b,c,a\n1,3,2,4\n4,"f\ng",3,1\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }, { d: 4, a: 1, c: 3, b: "f\ng" }]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert a simple array back to a csv', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + // 'payload', ',0,1,foo,"ba""r","di,ng","fa\nba"\n'); + msg.should.have.property('payload', ',0,1,foo\n'); // RFC-vs-Legacy difference - respect that user has specified a template with 4 columns + done(); + } + catch (e) { done(e); } + }); + const testJson = ["", 0, 1, "foo", 'ba"r', 'di,ng', "fa\nba"]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should convert an array of arrays back to a multi-line csv', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + // 'payload', '0,1,2,3,4\n4,3,2,1,0\n'); // Legacy + msg.should.have.property('payload', '0,1,2,3\n4,3,2,1\n'); // RFC-vs-Legacy difference - respect that user has specified a template with 4 columns + done(); + } + catch (e) { done(e); } + }); + const testJson = [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0]]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should be able to include column names as first row', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", hdrout: true, ret: "\r\n", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'a,b,c,d\r\n4,3,2,1\r\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }]; + n1.emit("input", { payload: testJson }); + }); + }); + + it('should be able to include column names as first row, and missing properties', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", hdrout: true, ret: "\r\n", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'col1,col2,col3,col4\r\nH1,H2,H3,H4\r\nA,B,,\r\nA,,C,\r\nA,,,"D\nE"\r\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = [{ "col1": "H1", "col2": "H2", "col3": "H3", "col4": "H4" }, { "col1": "A", "col2": "B" }, { "col1": "A", "col3": "C" }, { "col1": "A", "col4": "D\nE" }]; + n1.emit("input", { payload: testJson }); + }); + }); + + + it('should be able to pass in column names', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", hdrout: "once", ret: "\r\n", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + let count = 0; + n2.on("input", function (msg) { + count += 1; + try { + if (count === 1) { + msg.should.have.property('payload', 'a,,b,a\r\n4,,3,4\r\n'); + } + if (count === 3) { + msg.should.have.property('payload', '4,,3,4\r\n'); + done() + } + } + catch (e) { done(e); } + }); + const testJson = [{ d: 1, b: 3, c: 2, a: 4 }]; + n1.emit("input", { payload: testJson, columns: "a,,b,a", parts: { index: 0 } }); + n1.emit("input", { payload: testJson, parts: { index: 1 } }); + n1.emit("input", { payload: testJson, parts: { index: 2 } }); + }); + }); + + it('should be able to pass in column names - with payload as an array', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", hdrout: "once", ret: "\r\n", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'a,,b,a\r\n4,,3,4\r\n4,,3,4\r\n4,,3,4\r\n'); + done() + } + catch (e) { done(e); } + }); + const testJson = { d: 1, b: 3, c: 2, a: 4 }; + n1.emit("input", { payload: [testJson, testJson, testJson], columns: "a,,b,a" }); + }); + }); + + it('should handle quotes and sub-properties', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', '{},"text,with,commas","This ""is"" a banana","{""sub"":""object""}"\n'); + done(); + } + catch (e) { done(e); } + }); + const testJson = { d: { sub: "object" }, b: "text,with,commas", c: 'This "is" a banana', a: { sub2: undefined } }; + n1.emit("input", { payload: testJson }); + }); + }); + + }); + + it('should just pass through if no payload provided', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('topic', { a: 4, b: 3, c: 2, d: 1 }); + msg.should.not.have.property('payload'); + + done(); + } + catch (e) { done(e); } + }); + const testJson = { d: 1, b: 3, c: 2, a: 4 }; + n1.emit("input", { topic: testJson }); + }); + }); + + it('should warn if provided a number or boolean', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [["n2"]] }, + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + setTimeout(function () { + try { + const logEvents = helper.log().args.filter(function (evt) { + return evt[0].type == "csv"; + }); + logEvents.should.have.length(2); + logEvents[0][0].should.have.a.property('msg'); + logEvents[0][0].msg.toString().should.endWith('csv.errors.csv_js'); + logEvents[1][0].should.have.a.property('msg'); + logEvents[1][0].msg.toString().should.endWith('csv.errors.csv_js'); + done(); + } catch (err) { + done(err); + } + }, 150); + n1.emit("input", { payload: 1 }); + n1.emit("input", { payload: true }); + }); + }); + + it('should call done when message processing is completed', function (done) { + const completeNode = require("nr-test-utils").require("@node-red/nodes/core/common/24-complete.js"); + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [[]] }, + { id: "c1", type: "complete", scope: ["n1"], uncaught: false, wires: [["h1"]] }, + { id: "h1", type: "helper", wires: [[]] }]; + helper.load([csvNode, completeNode], flow, function () { + const n1 = helper.getNode("n1"); + const h1 = helper.getNode("h1"); + h1.on("input", function (msg) { + try { + msg.should.have.a.property('payload', "1,2,3,4"); + done(); + } catch (e) { + done(e); + } + }); + n1.receive({ payload: "1,2,3,4" }); + }); + }); + + it('should not call done or pass the bad msg through when input causes an error - should throw error and set status', function (done) { + // RFC-vs-Legacy difference + // In RFC mode, instead of passing the bad data through, the node will throw an error and set the status to indicate the error + const completeNode = require("nr-test-utils").require("@node-red/nodes/core/common/24-complete.js"); + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b,c,d", wires: [[]] }, + { id: "c1", type: "complete", scope: ["n1"], uncaught: false, wires: [["h1"]] }, + { id: "h1", type: "helper", wires: [[]] }]; + helper.load([csvNode, completeNode], flow, function () { + const n1 = helper.getNode("n1"); + const h1 = helper.getNode("h1"); + const c1 = helper.getNode("c1"); + n1.receive({ payload: 1 }); // neither object nor string + setTimeout(function () { + try { + c1.send.should.not.be.called(); + const logEvents = helper.log().args.filter(function (evt) { + return evt[0].type == "csv"; + }); + logEvents.should.have.length(1); + logEvents[0][0].should.have.a.property('msg'); + logEvents[0][0].msg.should.be.an.Error() + logEvents[0][0].msg.toString().should.endWith('csv.errors.csv_js'); + // check status was set + n1.status.called.should.be.true(); + n1.status.lastCall.args[0].should.have.property('fill', 'red'); + n1.status.lastCall.args[0].should.have.property('shape', 'dot'); + n1.status.lastCall.args[0].should.have.property('text', 'csv.errors.csv_js'); + // check error was thrown + n1.error.called.should.be.true(); + n1.error.lastCall.args[0].should.have.property('message', 'csv.errors.csv_js'); + done(); + } catch (err) { + done(err); + } + }, 150); + + }); + }); +});