frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2025-03-01 10:36:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4540 from Steve-Mcl/3934-csv-rfc4180

Browse Source 
Add RFC4180 compliant mode to CSV node
This commit is contained in:
Nick O'Leary 2024-03-07 15:39:25 +00:00 committed by GitHub
commit ce133c1c04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 2537 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html vendored
View File

@ -17,7 +17,20 @@
</select> </select>
<input style="width:40px;" type="text" id="node-input-sep" pattern="."> <input style="width:40px;" type="text" id="node-input-sep" pattern=".">
</div> </div>
<div class="form-row">
<label><i class="fa fa-code"></i> <span data-i18n="csv.label.spec"></span></label>
<div style="display: inline-grid;width: 70%;">
<select style="width:100%" id="csv-option-spec">
<option value="rfc" data-i18n="csv.spec.rfc"></option>
<option value="" data-i18n="csv.spec.legacy"></option>
</select>
<div>
<div class="form-tips csv-lecacy-warning" data-i18n="node-red:csv.spec.legacy_warning"
style="width: calc(100% - 18px); margin-top: 4px; max-width: unset;">
</div>
</div>
</div>
</div>
<div class="form-row"> <div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> <span data-i18n="common.label.name"></span></label> <label for="node-input-name"><i class="fa fa-tag"></i> <span data-i18n="common.label.name"></span></label>
<input type="text" id="node-input-name" data-i18n="[placeholder]common.label.name"> <input type="text" id="node-input-name" data-i18n="[placeholder]common.label.name">
@ -60,10 +73,10 @@
<div class="form-row" style="padding-left:20px;"> <div class="form-row" style="padding-left:20px;">
<label></label> <label></label>
<label style="width:auto; margin-right:10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label> <label style="width:auto; margin-right:10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label>
<select style="width:150px;" id="node-input-ret"> <select style="width:calc(70% - 108px);" id="node-input-ret">
<option value='\r\n' data-i18n="csv.newline.windows"></option>
<option value='\n' data-i18n="csv.newline.linux"></option> <option value='\n' data-i18n="csv.newline.linux"></option>
<option value='\r' data-i18n="csv.newline.mac"></option> <option value='\r' data-i18n="csv.newline.mac"></option>
<option value='\r\n' data-i18n="csv.newline.windows"></option>
</select> </select>
</div> </div>
</script> </script>
@ -75,6 +88,7 @@
color:"#DEBD5C", color:"#DEBD5C",
defaults: { defaults: {
name: {value:""}, name: {value:""},
spec: {value:"rfc"},
sep: { sep: {
value:',', required:true, value:',', required:true,
label:RED._("node-red:csv.label.separator"), label:RED._("node-red:csv.label.separator"),
@ -83,7 +97,7 @@
hdrin: {value:""}, hdrin: {value:""},
hdrout: {value:"none"}, hdrout: {value:"none"},
multi: {value:"one",required:true}, multi: {value:"one",required:true},
ret: {value:'\\n'}, ret: {value:'\\r\\n'}, // default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)")
temp: {value:""}, temp: {value:""},
skip: {value:"0"}, skip: {value:"0"},
strings: {value:true}, strings: {value:true},
@ -123,6 +137,27 @@
$("#node-input-sep").hide(); $("#node-input-sep").hide();
} }
}); });
$("#csv-option-spec").on("change", function() {
if ($("#csv-option-spec").val() == "rfc") {
$(".form-tips.csv-lecacy-warning").hide();
} else {
$(".form-tips.csv-lecacy-warning").show();
}
});
// new nodes will have `spec` set to "rfc" (default), but existing nodes will either not have
// a spec value or it will be empty - we need to maintain the legacy behaviour for existing
// flows but default to rfc for new nodes
let spec = !this.spec ? "" : "rfc"
$("#csv-option-spec").val(spec).trigger("change")
},
oneditsave: function() {
const specFormVal = $("#csv-option-spec").val() || '' // empty === legacy
const spectNodeVal = this.spec || '' // empty === legacy, null/undefined means in-place node upgrade (keep as is)
if (specFormVal !== spectNodeVal) {
// only update the flow value if changed (avoid marking the node dirty unnecessarily)
this.spec = specFormVal
}
} }
}); });
</script> </script>

908
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -15,322 +15,674 @@
**/ **/
module.exports = function(RED) { module.exports = function(RED) {
const csv = require('./lib/csv')
"use strict"; "use strict";
function CSVNode(n) { function CSVNode(n) {
RED.nodes.createNode(this,n); RED.nodes.createNode(this,n)
this.template = (n.temp || ""); const node = this
this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r"); const RFC4180Mode = n.spec === 'rfc'
this.quo = '"'; const legacyMode = !RFC4180Mode
this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
this.winflag = (this.ret === "\r\n");
this.lineend = "\n";
this.multi = n.multi || "one";
this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || "none";
this.goodtmpl = true;
this.skip = parseInt(n.skip || 0);
this.store = [];
this.parsestrings = n.strings;
this.include_empty_strings = n.include_empty_strings || false;
this.include_null_values = n.include_null_values || false;
if (this.parsestrings === undefined) { this.parsestrings = true; }
if (this.hdrout === false) { this.hdrout = "none"; }
if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true;
var node = this;
var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimmed, de-quoted and retrimmed node.status({}) // clear status
var clean = function(col,sep) {
if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
col = col.trim().split(re) || [""];
col = col.map(x => x.replace(/"/g,'').trim());
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; }
return col;
}
var template = clean(node.template,',');
var notemplate = template.length === 1 && template[0] === '';
node.hdrSent = false;
this.on("input", function(msg, send, done) { if (legacyMode) {
if (msg.hasOwnProperty("reset")) { this.template = (n.temp || "");
node.hdrSent = false; this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
this.quo = '"';
this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
this.winflag = (this.ret === "\r\n");
this.lineend = "\n";
this.multi = n.multi || "one";
this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || "none";
this.goodtmpl = true;
this.skip = parseInt(n.skip || 0);
this.store = [];
this.parsestrings = n.strings;
this.include_empty_strings = n.include_empty_strings || false;
this.include_null_values = n.include_null_values || false;
if (this.parsestrings === undefined) { this.parsestrings = true; }
if (this.hdrout === false) { this.hdrout = "none"; }
if (this.hdrout === true) { this.hdrout = "all"; }
var tmpwarn = true;
// var node = this;
var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
// pass in an array of column names to be trimmed, de-quoted and retrimmed
var clean = function(col,sep) {
if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
col = col.trim().split(re) || [""];
col = col.map(x => x.replace(/"/g,'').trim());
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; }
return col;
} }
if (msg.hasOwnProperty("payload")) { var template = clean(node.template,',');
if (typeof msg.payload == "object") { // convert object to CSV string var notemplate = template.length === 1 && template[0] === '';
try { node.hdrSent = false;
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template); this.on("input", function(msg, send, done) {
} if (msg.hasOwnProperty("reset")) {
const ou = []; node.hdrSent = false;
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } }
if (node.hdrout !== "none" && node.hdrSent === false) { if (msg.hasOwnProperty("payload")) {
if ((template.length === 1) && (template[0] === '')) { if (typeof msg.payload == "object") { // convert object to CSV string
if (msg.hasOwnProperty("columns")) { try {
template = clean(msg.columns || "",","); if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
} template = clean(node.template);
else {
template = Object.keys(msg.payload[0]);
}
} }
ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep)); const ou = [];
if (node.hdrout === "once") { node.hdrSent = true; } if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
} if (node.hdrout !== "none" && node.hdrSent === false) {
for (var s = 0; s < msg.payload.length; s++) {
if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
for (var t = 0; t < msg.payload[s].length; t++) {
if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
}
ou.push(msg.payload[s].join(node.sep));
}
else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
template = clean(msg.columns || "",",");
}
if ((template.length === 1) && (template[0] === '')) { if ((template.length === 1) && (template[0] === '')) {
/* istanbul ignore else */ if (msg.hasOwnProperty("columns")) {
if (tmpwarn === true) { // just warn about missing template once template = clean(msg.columns || "",",");
node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false;
} }
const row = []; else {
for (var p in msg.payload[0]) { template = Object.keys(msg.payload[0]);
}
}
ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
if (node.hdrout === "once") { node.hdrSent = true; }
}
for (var s = 0; s < msg.payload.length; s++) {
if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
for (var t = 0; t < msg.payload[s].length; t++) {
if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
}
ou.push(msg.payload[s].join(node.sep));
}
else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
template = clean(msg.columns || "",",");
}
if ((template.length === 1) && (template[0] === '')) {
/* istanbul ignore else */ /* istanbul ignore else */
if (msg.payload[s].hasOwnProperty(p)) { if (tmpwarn === true) { // just warn about missing template once
node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false;
}
const row = [];
for (var p in msg.payload[0]) {
/* istanbul ignore else */ /* istanbul ignore else */
if (typeof msg.payload[s][p] !== "object") { if (msg.payload[s].hasOwnProperty(p)) {
// Fix to honour include null values flag /* istanbul ignore else */
//if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) { if (typeof msg.payload[s][p] !== "object") {
var q = ""; // Fix to honour include null values flag
if (msg.payload[s][p] !== undefined) { //if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) {
q += msg.payload[s][p]; var q = "";
if (msg.payload[s][p] !== undefined) {
q += msg.payload[s][p];
}
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""');
row.push(node.quo + q + node.quo);
}
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
row.push(node.quo + q + node.quo);
}
else { row.push(q); } // otherwise just add
} }
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""');
row.push(node.quo + q + node.quo);
}
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
row.push(node.quo + q + node.quo);
}
else { row.push(q); } // otherwise just add
} }
} }
ou.push(row.join(node.sep)); // add separator
} }
ou.push(row.join(node.sep)); // add separator else {
const row = [];
for (var t=0; t < template.length; t++) {
if (template[t] === '') {
row.push('');
}
else {
var tt = template[t];
if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; }
else { tt = '"'+tt+'"'; }
var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
/* istanbul ignore else */
if (p === undefined) { p = ""; }
// fix to honour include null values flag
//if (p === null && node.include_null_values !== true) { p = "";}
p = RED.util.ensureString(p);
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""');
row.push(node.quo + p + node.quo);
}
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
row.push(node.quo + p + node.quo);
}
else { row.push(p); } // otherwise just add
}
}
ou.push(row.join(node.sep)); // add separator
}
}
}
// join lines, don't forget to add the last new line
msg.payload = ou.join(node.ret) + node.ret;
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') {
send(msg);
}
done();
}
catch(e) { done(e); }
}
else if (typeof msg.payload == "string") { // convert CSV string to object
try {
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
var j = 0; // pointer into array of template items
var k = [""]; // array of data for each of the template items
var o = {}; // output object to build up
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var last = false;
var line = msg.payload;
var linecount = 0;
var tmp = "";
var has_parts = msg.hasOwnProperty("parts");
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
}
// For now we are just going to assume that any \r or \n means an end of line...
// got to be a weird csv that has singleton \r \n in it for another reason...
// Now process the whole file/line
var nocr = (line.match(/[\r\n]/g)||[]).length;
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
continue;
}
if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; }
template = clean(tmp,node.sep);
first = false;
}
else { tmp += line[i]; }
} }
else { else {
const row = []; if (line[i] === node.quo) { // if it's a quote toggle inside or outside
for (var t=0; t < template.length; t++) { f = !f;
if (template[t] === '') { if (line[i-1] === node.quo) {
row.push(''); if (f === false) { k[j] += '\"'; }
} // if it's a quotequote then it's actually a quote
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
}
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( template[j] && (template[j] !== "") ) {
// if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
} }
else { j += 1;
var tt = template[t]; // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; } k[j] = line.length - 1 === i ? null : "";
else { tt = '"'+tt+'"'; } }
var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']'); else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
/* istanbul ignore else */ //console.log(j,k,o,k[j]);
if (p === undefined) { p = ""; } if (!node.goodtmpl) { template[j] = "col"+(j+1); }
// fix to honour include null values flag if ( template[j] && (template[j] !== "") ) {
//if (p === null && node.include_null_values !== true) { p = "";} // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
p = RED.util.ensureString(p); if (line[i-1] === node.sep) k[j] = null;
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
p = p.replace(/"/g, '""'); else { if (k[j] !== null) k[j].replace(/\r$/,''); }
row.push(node.quo + p + node.quo); if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
} if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
row.push(node.quo + p + node.quo);
}
else { row.push(p); } // otherwise just add
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array
}
j = 0;
k = [""];
o = {};
f = true; // reset in/out flag ready for next line.
}
else { // just add to the part of the message
k[j] += line[i];
} }
ou.push(row.join(node.sep)); // add separator
} }
} }
} // Finished so finalize and send anything left
// join lines, don't forget to add the last new line if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
msg.payload = ou.join(node.ret) + node.ret; if (!node.goodtmpl) { template[j] = "col"+(j+1); }
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); }
done();
}
catch(e) { done(e); }
}
else if (typeof msg.payload == "string") { // convert CSV string to object
try {
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
var j = 0; // pointer into array of template items
var k = [""]; // array of data for each of the template items
var o = {}; // output object to build up
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var last = false;
var line = msg.payload;
var linecount = 0;
var tmp = "";
var has_parts = msg.hasOwnProperty("parts");
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
}
// For now we are just going to assume that any \r or \n means an end of line... if ( template[j] && (template[j] !== "") ) {
// got to be a weird csv that has singleton \r \n in it for another reason... if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
// Now process the whole file/line if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
var nocr = (line.match(/[\r\n]/g)||[]).length; if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; } if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
continue;
} }
if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if (JSON.stringify(o) !== "{}") { // don't send empty objects
if (line.length - i === 1) { tmp += line[i]; } a.push(o); // add to the array
template = clean(tmp,node.sep);
first = false;
}
else { tmp += line[i]; }
} }
else {
if (line[i] === node.quo) { // if it's a quote toggle inside or outside
f = !f;
if (line[i-1] === node.quo) {
if (f === false) { k[j] += '\"'; }
} // if it's a quotequote then it's actually a quote
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
}
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( template[j] && (template[j] !== "") ) {
// if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
}
j += 1;
// if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
k[j] = line.length - 1 === i ? null : "";
}
else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
//console.log(j,k,o,k[j]);
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( template[j] && (template[j] !== "") ) {
// if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
if (line[i-1] === node.sep) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array
}
j = 0;
k = [""];
o = {};
f = true; // reset in/out flag ready for next line.
}
else { // just add to the part of the message
k[j] += line[i];
}
}
}
// Finished so finalize and send anything left
if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
if ( template[j] && (template[j] !== "") ) { if (node.multi !== "one") {
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); } msg.payload = a;
else { if (k[j] !== null) k[j].replace(/\r$/,''); } if (has_parts && nocr <= 1) {
if (node.include_null_values && k[j] === null) o[template[j]] = k[j]; if (JSON.stringify(o) !== "{}") {
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; node.store.push(o);
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; }
} if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store;
if (JSON.stringify(o) !== "{}") { // don't send empty objects msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
a.push(o); // add to the array delete msg.parts;
} send(msg);
node.store = [];
if (node.multi !== "one") { }
msg.payload = a;
if (has_parts && nocr <= 1) {
if (JSON.stringify(o) !== "{}") {
node.store.push(o);
} }
if (msg.parts.index + 1 === msg.parts.count) { else {
msg.payload = node.store;
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
delete msg.parts; send(msg); // finally send the array
send(msg);
node.store = [];
} }
} }
else { else {
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); var len = a.length;
send(msg); // finally send the array for (var i = 0; i < len; i++) {
} var newMessage = RED.util.cloneMessage(msg);
} newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
else { newMessage.payload = a[i];
var len = a.length; if (!has_parts) {
for (var i = 0; i < len; i++) { newMessage.parts = {
var newMessage = RED.util.cloneMessage(msg); id: msg._msgid,
newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); index: i,
newMessage.payload = a[i]; count: len
if (!has_parts) { };
newMessage.parts = { }
id: msg._msgid, else {
index: i, newMessage.parts.index -= node.skip;
count: len newMessage.parts.count -= node.skip;
}; if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1;
newMessage.parts.count -= 1;
}
}
if (last) { newMessage.complete = true; }
send(newMessage);
} }
else { if (has_parts && last && len === 0) {
newMessage.parts.index -= node.skip; send({complete:true});
newMessage.parts.count -= node.skip; }
if (node.hdrin) { // if we removed the header line then shift the counts by 1 }
newMessage.parts.index -= 1; node.linecount = 0;
newMessage.parts.count -= 1; done();
}
catch(e) { done(e); }
}
else { node.warn(RED._("csv.errors.csv_js")); done(); }
}
else {
if (!msg.hasOwnProperty("reset")) {
node.send(msg); // If no payload and not reset - just pass it on.
}
done();
}
});
}
if(RFC4180Mode) {
node.template = (n.temp || "")
node.sep = (n.sep || ',').replace(/\\t/g, "\t").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
node.quo = '"'
// default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)")
node.ret = (n.ret || "\r\n").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
node.multi = n.multi || "one"
node.hdrin = n.hdrin || false
node.hdrout = n.hdrout || "none"
node.goodtmpl = true
node.skip = parseInt(n.skip || 0)
node.store = []
node.parsestrings = n.strings
node.include_empty_strings = n.include_empty_strings || false
node.include_null_values = n.include_null_values || false
if (node.parsestrings === undefined) { node.parsestrings = true }
if (node.hdrout === false) { node.hdrout = "none" }
if (node.hdrout === true) { node.hdrout = "all" }
const dontSendHeaders = node.hdrout === "none"
const sendHeadersOnce = node.hdrout === "once"
const sendHeadersAlways = node.hdrout === "all"
const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
const quoteables = [node.sep, node.quo, "\n", "\r"]
const templateQuoteables = [',', '"', "\n", "\r"]
let badTemplateWarnOnce = true
const columnStringToTemplateArray = function (col, sep) {
// NOTE: enforce strict column template parsing in RFC4180 mode
const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false }
return parsed.headers.length ? parsed.headers : null
}
const templateArrayToColumnString = function (template, keepEmptyColumns) {
// NOTE: enforce strict column template parsing in RFC4180 mode
const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true })
return keepEmptyColumns
? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables}))
: parsed.header // exclues empty columns
// TODO: resolve inconsistency between CSV->JSON and JSON->CSV
// CSV->JSON: empty columns are excluded
// JSON->CSV: empty columns are kept in some cases
}
function addQuotes(cell, options) {
options = options || {}
return csv.quoteCell(cell, {
quote: options.quote || node.quo || '"',
separator: options.separator || node.sep || ',',
quoteables: options.quoteables || quoteables
})
}
const hasTemplate = (t) => t?.length > 0 && !(t.length === 1 && t[0] === '')
let template
try {
template = columnStringToTemplateArray(node.template, ',') || ['']
} catch (e) {
node.warn(RED._("csv.errors.bad_template")) // is warning really necessary now we have status?
node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
return // dont hook up the node
}
const noTemplate = hasTemplate(template) === false
node.hdrSent = false
node.on("input", function (msg, send, done) {
node.status({}) // clear status
if (msg.hasOwnProperty("reset")) {
node.hdrSent = false
}
if (msg.hasOwnProperty("payload")) {
let inputData = msg.payload
if (typeof inputData == "object") { // convert object to CSV string
try {
// first determine the payload kind. Array or objects? Array of primitives? Array of arrays? Just an object?
// then, if necessary, convert to an array of objects/arrays
let isObject = !Array.isArray(inputData) && typeof inputData === 'object'
let isArrayOfObjects = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] === 'object'
let isArrayOfArrays = Array.isArray(inputData) && inputData.length > 0 && Array.isArray(inputData[0])
let isArrayOfPrimitives = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] !== 'object'
if (isObject) {
inputData = [inputData]
isArrayOfObjects = true
isObject = false
} else if (isArrayOfPrimitives) {
inputData = [inputData]
isArrayOfArrays = true
isArrayOfPrimitives = false
}
const stringBuilder = []
if (!(noTemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = columnStringToTemplateArray(node.template) || ['']
}
// build header line
if (sendHeaders && node.hdrSent === false) {
if (hasTemplate(template) === false) {
if (msg.hasOwnProperty("columns")) {
template = columnStringToTemplateArray(msg.columns || "", ",") || ['']
}
else {
template = Object.keys(inputData[0]) || ['']
} }
} }
if (last) { newMessage.complete = true; } stringBuilder.push(templateArrayToColumnString(template, true))
send(newMessage); if (sendHeadersOnce) { node.hdrSent = true }
} }
if (has_parts && last && len === 0) {
send({complete:true}); // build csv lines
for (let s = 0; s < inputData.length; s++) {
let row = inputData[s]
if (isArrayOfArrays) {
/*** row is an array of arrays ***/
const _hasTemplate = hasTemplate(template)
const len = _hasTemplate ? template.length : row.length
const result = []
for (let t = 0; t < len; t++) {
let cell = row[t]
if (cell === undefined) { cell = "" }
if(_hasTemplate) {
const header = template[t]
if (header) {
result[t] = addQuotes(RED.util.ensureString(cell))
}
} else {
result[t] = addQuotes(RED.util.ensureString(cell))
}
}
stringBuilder.push(result.join(node.sep))
} else {
/*** row is an object ***/
if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) {
template = columnStringToTemplateArray(msg.columns || "", ",")
}
if (hasTemplate(template) === false) {
/*** row is an object but we still don't have a template ***/
if (badTemplateWarnOnce === true) {
node.warn(RED._("csv.errors.obj_csv"))
badTemplateWarnOnce = false
}
const rowData = []
for (let header in inputData[0]) {
if (row.hasOwnProperty(header)) {
const cell = row[header]
if (typeof cell !== "object") {
let cellValue = ""
if (cell !== undefined) {
cellValue += cell
}
rowData.push(addQuotes(cellValue))
}
}
}
stringBuilder.push(rowData.join(node.sep))
} else {
/*** row is an object and we have a template ***/
const rowData = []
for (let t = 0; t < template.length; t++) {
if (!template[t]) {
rowData.push('')
}
else {
let cellValue = inputData[s][template[t]]
if (cellValue === undefined) { cellValue = "" }
cellValue = RED.util.ensureString(cellValue)
rowData.push(addQuotes(cellValue))
}
}
stringBuilder.push(rowData.join(node.sep)); // add separator
}
}
} }
// join lines, don't forget to add the last new line
msg.payload = stringBuilder.join(node.ret) + node.ret
msg.columns = templateArrayToColumnString(template)
if (msg.payload !== '') { send(msg) }
done()
}
catch (e) {
done(e)
} }
node.linecount = 0;
done();
} }
catch(e) { done(e); } else if (typeof inputData == "string") { // convert CSV string to object
try {
let firstLine = true; // is this the first line
let last = false
let linecount = 0
const has_parts = msg.hasOwnProperty("parts")
// determine if this is a multi part message and if so what part we are processing
if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index
if (msg.parts.index > node.skip) { firstLine = false }
if (msg.parts.hasOwnProperty("count") && (msg.parts.index + 1 >= msg.parts.count)) { last = true }
}
// If skip is set, compute the cursor position to start parsing from
let _cursor = 0
if (node.skip > 0 && linecount < node.skip) {
for (; _cursor < inputData.length; _cursor++) {
if (firstLine && (linecount < node.skip)) {
if (inputData[_cursor] === "\r" || inputData[_cursor] === "\n") {
linecount += 1
}
continue
}
break
}
if (_cursor >= inputData.length) {
return // skip this line
}
}
// count the number of line breaks in the string
const noofCR = ((_cursor ? inputData.slice(_cursor) : inputData).match(/[\r\n]/g) || []).length
// if we have `parts` and we are outputting multiple objects and we have more than one line
// then we need to set firstLine to true so that we process the header line
if (has_parts && node.multi === "mult" && noofCR > 1) {
firstLine = true
}
// if we are processing the first line and the node has been set to extract the header line
// update the template with the header line
if (firstLine && node.hdrin === true) {
/** @type {import('./lib/csv/index.js').CSVParseOptions} */
const csvOptionsForHeaderRow = {
cursor: _cursor,
separator: node.sep,
quote: node.quo,
dataHasHeaderRow: true,
headersOnly: true,
outputStyle: 'array',
strict: true // enforce strict parsing of the header row
}
try {
const csvHeader = csv.parse(inputData, csvOptionsForHeaderRow)
template = csvHeader.headers
_cursor = csvHeader.cursor
} catch (e) {
// node.warn(RED._("csv.errors.bad_template")) // add warning?
node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
throw e
}
}
// now we process the data lines
/** @type {import('./lib/csv/index.js').CSVParseOptions} */
const csvOptions = {
cursor: _cursor,
separator: node.sep,
quote: node.quo,
dataHasHeaderRow: false,
headers: hasTemplate(template) ? template : null,
outputStyle: 'object',
includeNullValues: node.include_null_values,
includeEmptyStrings: node.include_empty_strings,
parseNumeric: node.parsestrings,
strict: false // relax the strictness of the parser for data rows
}
const csvParseResult = csv.parse(inputData, csvOptions)
const data = csvParseResult.data
// output results
if (node.multi !== "one") {
if (has_parts && noofCR <= 1) {
if (data.length > 0) {
node.store.push(...data)
}
if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store
msg.columns = csvParseResult.header
// msg._mode = 'RFC4180 mode'
delete msg.parts
send(msg)
node.store = []
}
}
else {
msg.columns = csvParseResult.header
// msg._mode = 'RFC4180 mode'
msg.payload = data
send(msg); // finally send the array
}
}
else {
const len = data.length
for (let row = 0; row < len; row++) {
const newMessage = RED.util.cloneMessage(msg)
newMessage.columns = csvParseResult.header
newMessage.payload = data[row]
if (!has_parts) {
newMessage.parts = {
id: msg._msgid,
index: row,
count: len
}
}
else {
newMessage.parts.index -= node.skip
newMessage.parts.count -= node.skip
if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1
newMessage.parts.count -= 1
}
}
if (last) { newMessage.complete = true }
// newMessage._mode = 'RFC4180 mode'
send(newMessage)
}
if (has_parts && last && len === 0) {
// send({complete:true, _mode: 'RFC4180 mode'})
send({ complete: true })
}
}
node.linecount = 0
done()
}
catch (e) {
done(e)
}
}
else {
// RFC-vs-legacy mode difference: In RFC mode, we throw catchable errors and provide a status message
const err = new Error(RED._("csv.errors.csv_js"))
node.status({ fill: "red", shape: "dot", text: err.message })
done(err)
}
} }
else { node.warn(RED._("csv.errors.csv_js")); done(); } else {
} if (!msg.hasOwnProperty("reset")) {
else { node.send(msg); // If no payload and not reset - just pass it on.
if (!msg.hasOwnProperty("reset")) { }
node.send(msg); // If no payload and not reset - just pass it on. done()
} }
done(); })
} }
});
} }
RED.nodes.registerType("csv",CSVNode);
RED.nodes.registerType("csv",CSVNode)
} }

324
packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js vendored Normal file
View File

@ -0,0 +1,324 @@
/**
* @typedef {Object} CSVParseOptions
* @property {number} [cursor=0] - an index into the CSV to start parsing from
* @property {string} [separator=','] - the separator character
* @property {string} [quote='"'] - the quote character
* @property {boolean} [headersOnly=false] - only parse the headers and return them
* @property {string[]} [headers=[]] - an array of headers to use instead of the first row of the CSV data
* @property {boolean} [dataHasHeaderRow=true] - whether the CSV data to parse has a header row
* @property {boolean} [outputHeader=true] - whether the output data should include a header row (only applies to array output)
* @property {boolean} [parseNumeric=false] - parse numeric values into numbers
* @property {boolean} [includeNullValues=false] - include null values in the output
* @property {boolean} [includeEmptyStrings=true] - include empty strings in the output
* @property {string} [outputStyle='object'] - output an array of arrays or an array of objects
* @property {boolean} [strict=false] - throw an error if the CSV is malformed
*/
/**
* Parses a CSV string into an array of arrays or an array of objects.
*
* NOTES:
* * Deviations from the RFC4180 spec (for the sake of user fiendliness, system implementations and flexibility), this parser will:
* * accept any separator character, not just `,`
* * accept any quote character, not just `"`
* * parse `\r`, `\n` or `\r\n` as line endings (RRFC4180 2.1 states lines are separated by CRLF)
* * Only single character `quote` is supported
* * `quote` is `"` by default
* * Any cell that contains a `quote` or `separator` will be quoted
* * Any `quote` characters inside a cell will be escaped as per RFC 4180 2.6
* * Only single character `separator` is supported
* * Only `array` and `object` output styles are supported
* * `array` output style is an array of arrays [[],[],[]]
* * `object` output style is an array of objects [{},{},{}]
* * Only `headers` or `dataHasHeaderRow` are supported, not both
* @param {string} csvIn - the CSV string to parse
* @param {CSVParseOptions} parseOptions - options
* @throws {Error}
*/
function parse(csvIn, parseOptions) {
/* Normalise options */
parseOptions = parseOptions || {};
const separator = parseOptions.separator ?? ',';
const quote = parseOptions.quote ?? '"';
const headersOnly = parseOptions.headersOnly ?? false;
const headers = Array.isArray(parseOptions.headers) ? parseOptions.headers : []
const dataHasHeaderRow = parseOptions.dataHasHeaderRow ?? true;
const outputHeader = parseOptions.outputHeader ?? true;
const parseNumeric = parseOptions.parseNumeric ?? false;
const includeNullValues = parseOptions.includeNullValues ?? false;
const includeEmptyStrings = parseOptions.includeEmptyStrings ?? true;
const outputStyle = ['array', 'object'].includes(parseOptions.outputStyle) ? parseOptions.outputStyle : 'object'; // 'array [[],[],[]]' or 'object [{},{},{}]
const strict = parseOptions.strict ?? false
/* Local variables */
const cursorMax = csvIn.length;
const ouputArrays = outputStyle === 'array';
const headersSupplied = headers.length > 0
// The original regex was an "is-a-number" positive logic test. /^ *[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+ *$/i;
// Below, is less strict and inverted logic but coupled with +cast it is 13%+ faster than original regex+parsefloat
// and has the benefit of understanding hexadecimals, binary and octal numbers.
const skipNumberConversion = /^ *(\+|-0\d|0\d)/
const cellBuilder = []
let rowBuilder = []
let cursor = typeof parseOptions.cursor === 'number' ? parseOptions.cursor : 0;
let newCell = true, inQuote = false, closed = false, output = [];
/* inline helper functions */
const finaliseCell = () => {
let cell = cellBuilder.join('')
cellBuilder.length = 0
// push the cell:
// NOTE: if cell is empty but newCell==true, then this cell had zero chars - push `null`
// otherwise push empty string
return rowBuilder.push(cell || (newCell ? null : ''))
}
const finaliseRow = () => {
if (cellBuilder.length) {
finaliseCell()
}
if (rowBuilder.length) {
output.push(rowBuilder)
rowBuilder = []
}
}
/* Main parsing loop */
while (cursor < cursorMax) {
const char = csvIn[cursor]
if (inQuote) {
if (char === quote && csvIn[cursor + 1] === quote) {
cellBuilder.push(quote)
cursor += 2;
newCell = false;
closed = false;
} else if (char === quote) {
inQuote = false;
cursor += 1;
newCell = false;
closed = true;
} else {
cellBuilder.push(char)
newCell = false;
closed = false;
cursor++;
}
} else {
if (char === separator) {
finaliseCell()
cursor += 1;
newCell = true;
closed = false;
} else if (char === quote) {
if (newCell) {
inQuote = true;
cursor += 1;
newCell = false;
closed = false;
}
else if (strict) {
throw new UnquotedQuoteError(cursor)
} else {
// not strict, keep 1 quote if the next char is not a cell/record separator
cursor++
if (csvIn[cursor] && csvIn[cursor] !== '\n' && csvIn[cursor] !== '\r' && csvIn[cursor] !== separator) {
cellBuilder.push(char)
if (csvIn[cursor] === quote) {
cursor++ // skip the next quote
}
}
}
} else {
if (char === '\n' || char === '\r') {
finaliseRow()
if (csvIn[cursor + 1] === '\n') {
cursor += 2;
} else {
cursor++
}
newCell = true;
closed = false;
if (headersOnly) {
break
}
} else {
if (closed) {
if (strict) {
throw new DataAfterCloseError(cursor)
} else {
cursor--; // move back to grab the previously discarded char
closed = false
}
} else {
cellBuilder.push(char)
newCell = false;
cursor++;
}
}
}
}
}
if (strict && inQuote) {
throw new ParseError(`Missing quote, unclosed cell`, cursor)
}
// finalise the last cell/row
finaliseRow()
let firstRowIsHeader = false
// if no headers supplied, generate them
if (output.length >= 1) {
if (headersSupplied) {
// headers already supplied
} else if (dataHasHeaderRow) {
// take the first row as the headers
headers.push(...output[0])
firstRowIsHeader = true
} else {
// generate headers col1, col2, col3, etc
for (let i = 0; i < output[0].length; i++) {
headers.push("col" + (i + 1))
}
}
}
const finalResult = {
/** @type {String[]} headers as an array of string */
headers: headers,
/** @type {String} headers as a comma-separated string */
header: null,
/** @type {Any[]} Result Data (may include header row: check `firstRowIsHeader` flag) */
data: [],
/** @type {Boolean|undefined} flag to indicate if the first row is a header row (only applies when `outputStyle` is 'array') */
firstRowIsHeader: undefined,
/** @type {'array'|'object'} flag to indicate the output style */
outputStyle: outputStyle,
/** @type {Number} The current cursor position */
cursor: cursor,
}
const quotedHeaders = []
for (let i = 0; i < headers.length; i++) {
if (!headers[i]) {
continue
}
quotedHeaders.push(quoteCell(headers[i], { quote, separator: ',' }))
}
finalResult.header = quotedHeaders.join(',') // always quote headers and join with comma
// output is an array of arrays [[],[],[]]
if (ouputArrays || headersOnly) {
if (!firstRowIsHeader && !headersOnly && outputHeader && headers.length > 0) {
if (output.length > 0) {
output.unshift(headers)
} else {
output = [headers]
}
firstRowIsHeader = true
}
if (headersOnly) {
delete finalResult.firstRowIsHeader
return finalResult
}
finalResult.firstRowIsHeader = firstRowIsHeader
finalResult.data = (firstRowIsHeader && !outputHeader) ? output.slice(1) : output
return finalResult
}
// output is an array of objects [{},{},{}]
const outputObjects = []
let i = firstRowIsHeader ? 1 : 0
for (; i < output.length; i++) {
const rowObject = {}
let isEmpty = true
for (let j = 0; j < headers.length; j++) {
if (!headers[j]) {
continue
}
let v = output[i][j] === undefined ? null : output[i][j]
if (v === null && !includeNullValues) {
continue
} else if (v === "" && !includeEmptyStrings) {
continue
} else if (parseNumeric === true && v && !skipNumberConversion.test(v)) {
const vTemp = +v
const isNumber = !isNaN(vTemp)
if(isNumber) {
v = vTemp
}
}
rowObject[headers[j]] = v
isEmpty = false
}
// determine if this row is empty
if (!isEmpty) {
outputObjects.push(rowObject)
}
}
finalResult.data = outputObjects
delete finalResult.firstRowIsHeader
return finalResult
}
/**
* Quotes a cell in a CSV string if necessary. Addiionally, any double quotes inside the cell will be escaped as per RFC 4180 2.6 (https://datatracker.ietf.org/doc/html/rfc4180#section-2).
* @param {string} cell - the string to quote
* @param {*} options - options
* @param {string} [options.quote='"'] - the quote character
* @param {string} [options.separator=','] - the separator character
* @param {string[]} [options.quoteables] - an array of characters that, when encountered, will trigger the application of outer quotes
* @returns
*/
function quoteCell(cell, { quote = '"', separator = ",", quoteables } = {
quote: '"',
separator: ",",
quoteables: [quote, separator, '\r', '\n']
}) {
quoteables = quoteables || [quote, separator, '\r', '\n'];
let doubleUp = false;
if (cell.indexOf(quote) !== -1) { // add double quotes if any quotes
doubleUp = true;
}
const quoteChar = quoteables.some(q => cell.includes(q)) ? quote : '';
return quoteChar + (doubleUp ? cell.replace(/"/g, '""') : cell) + quoteChar;
}
// #region Custom Error Classes
class ParseError extends Error {
/**
* @param {string} message - the error message
* @param {number} cursor - the cursor index where the error occurred
*/
constructor(message, cursor) {
super(message)
this.name = 'ParseError'
this.cursor = cursor
}
}
class UnquotedQuoteError extends ParseError {
/**
* @param {number} cursor - the cursor index where the error occurred
*/
constructor(cursor) {
super('Quote found in the middle of an unquoted field', cursor)
this.name = 'UnquotedQuoteError'
}
}
class DataAfterCloseError extends ParseError {
/**
* @param {number} cursor - the cursor index where the error occurred
*/
constructor(cursor) {
super('Data found after closing quote', cursor)
this.name = 'DataAfterCloseError'
}
}
// #endregion
exports.parse = parse
exports.quoteCell = quoteCell
exports.ParseError = ParseError
exports.UnquotedQuoteError = UnquotedQuoteError
exports.DataAfterCloseError = DataAfterCloseError

9
packages/node_modules/@node-red/nodes/locales/en-US/messages.json vendored
View File

@ -849,7 +849,13 @@
"newline": "Newline", "newline": "Newline",
"usestrings": "parse numerical values", "usestrings": "parse numerical values",
"include_empty_strings": "include empty strings", "include_empty_strings": "include empty strings",
"include_null_values": "include null values" "include_null_values": "include null values",
"spec": "Parser"
},
"spec": {
"rfc": "RFC4180",
"legacy": "Legacy",
"legacy_warning": "Legacy mode will be removed in a future release."
}, },
"placeholder": { "placeholder": {
"columns": "comma-separated column names" "columns": "comma-separated column names"
@ -878,6 +884,7 @@
"once": "send headers once, until msg.reset" "once": "send headers once, until msg.reset"
}, },
"errors": { "errors": {
"bad_template": "Malformed columns template.",
"csv_js": "This node only handles CSV strings or js objects.", "csv_js": "This node only handles CSV strings or js objects.",
"obj_csv": "No columns template specified for object -> CSV.", "obj_csv": "No columns template specified for object -> CSV.",
"bad_csv": "Malformed CSV data - output probably corrupt." "bad_csv": "Malformed CSV data - output probably corrupt."

5
packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html vendored
View File

@ -36,7 +36,9 @@
</dl> </dl>
<h3>Details</h3> <h3>Details</h3>
<p>The column template can contain an ordered list of column names. When converting CSV to an object, the column names <p>The column template can contain an ordered list of column names. When converting CSV to an object, the column names
will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.</p> will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.
<p>When the RFC parser is selected, the column template must be compliant with RFC4180.</p>
</p>
<p>When converting to CSV, the columns template is used to identify which properties to extract from the object and in what order.</p> <p>When converting to CSV, the columns template is used to identify which properties to extract from the object and in what order.</p>
<p>If the columns template is blank then you can use a simple comma separated list of properties supplied in <code>msg.columns</code> to <p>If the columns template is blank then you can use a simple comma separated list of properties supplied in <code>msg.columns</code> to
determine what to extract and in what order. If neither are present then all the object properties are output in the order determine what to extract and in what order. If neither are present then all the object properties are output in the order
@ -49,4 +51,5 @@
<p>If outputting multiple messages they will have their <code>parts</code> property set and form a complete message sequence.</p> <p>If outputting multiple messages they will have their <code>parts</code> property set and form a complete message sequence.</p>
<p>If the node is set to only send column headers once, then setting <code>msg.reset</code> to any value will cause the node to resend the headers.</p> <p>If the node is set to only send column headers once, then setting <code>msg.reset</code> to any value will cause the node to resend the headers.</p>
<p><b>Note:</b> the column template must be comma separated - even if a different separator is chosen for the data.</p> <p><b>Note:</b> the column template must be comma separated - even if a different separator is chosen for the data.</p>
<p><b>Note:</b> in RFC mode, catchable errors will be thrown for malformed CSV headers and invalid input payload data</p>
</script> </script>

1710
test/nodes/core/parsers/70-CSV_spec.js
View File

File diff suppressed because it is too large Load Diff