From 91b7dd988ee6076e5ad21c776bb1e242b14a5e5f Mon Sep 17 00:00:00 2001 From: tmdoit Date: Wed, 18 Mar 2020 16:54:10 +0100 Subject: [PATCH] [CSV node] Add support for parsing empty strings and null values (#2510) * [CSV node] Add support for parsing empty strings and null values * Add new lines at the end and fix script type. * Last one script type fix * Naming change --- .../@node-red/nodes/core/parsers/70-CSV.html | 8 +++- .../@node-red/nodes/core/parsers/70-CSV.js | 38 +++++++++++++------ .../nodes/locales/en-US/messages.json | 7 +++- .../nodes/locales/en-US/parsers/70-CSV.html | 3 ++ 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html index 2e4ca0dd7..c7e8fdc58 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html @@ -33,6 +33,10 @@

+ +
+ +
@@ -74,7 +78,9 @@ ret: {value:'\\n'}, temp: {value:""}, skip: {value:"0"}, - strings: {value:true} + strings: {value:true}, + include_empty_strings: {value:""}, + include_null_values: {value:""} }, inputs:1, outputs:1, diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 290b159a9..0424e1e9b 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -31,6 +31,8 @@ module.exports = function(RED) { this.skip = parseInt(n.skip || 0); this.store = []; this.parsestrings = n.strings; + this.include_empty_strings = n.include_empty_strings || false; + this.include_null_values = n.include_null_values || false; if (this.parsestrings === undefined) { this.parsestrings = true; } var tmpwarn = true; var node = this; @@ -173,20 +175,29 @@ module.exports = function(RED) { } else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } - if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) { - if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } - o[node.template[j]] = k[j]; + if ( node.template[j] && (node.template[j] !== "") ) { + // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null + if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null; + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } + if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; } j += 1; - k[j] = ""; + // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",' + k[j] = line.length - 1 === i ? null : ""; } else if ((line[i] === "\n") || (line[i] === "\r")) { // handle multiple lines //console.log(j,k,o,k[j]); if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } - if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) { - if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } - else { k[j].replace(/\r$/,''); } - o[node.template[j]] = k[j]; + if ( node.template[j] && (node.template[j] !== "") ) { + // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3' + if (line[i-1] === node.sep) k[j] = null; + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } + else { if (k[j] !== null) k[j].replace(/\r$/,''); } + if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; } if (JSON.stringify(o) !== "{}") { // don't send empty objects a.push(o); // add to the array @@ -204,10 +215,13 @@ module.exports = function(RED) { // Finished so finalize and send anything left //console.log(j,k,o,k[j]); if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } - if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) { - if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } - else { k[j].replace(/\r$/,''); } - o[node.template[j]] = k[j]; + + if ( node.template[j] && (node.template[j] !== "") ) { + if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } + else { if (k[j] !== null) k[j].replace(/\r$/,''); } + if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j]; + if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j]; + if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j]; } if (JSON.stringify(o) !== "{}") { // don't send empty objects a.push(o); // add to the array diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index e1d7c6368..06688c326 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -698,7 +698,9 @@ "output": "Output", "includerow": "include column name row", "newline": "Newline", - "usestrings": "parse numerical values" + "usestrings": "parse numerical values", + "include_empty_strings": "include empty strings", + "include_null_values": "include null values" }, "placeholder": { "columns": "comma-separated column names" @@ -894,7 +896,8 @@ "fixup": "Fix-up exp" }, "errors": { - "invalid-expr": "Invalid JSONata expression: __error__" + "invalid-expr": "Invalid JSONata expression: __error__", + "invalid-type": "Cannot join __error__ to buffer" } }, "sort" : { diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html index 5b0731cb1..19ece5ac0 100644 --- a/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html +++ b/packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html @@ -39,6 +39,9 @@ will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.

When converting to CSV, the column template is used to identify which properties to extract from the object and in what order.

If the input is an array then the columns template is only used to optionally generate a row of column titles.

+

If 'parse numerical values' option is checked, string numerical values will be returned as numbers, ie. middle value '1,"1.5",2'.

+

If 'include empty strings' option is checked, empty strings will be returned in result, ie. middle value '"1","",3'.

+

If 'include null values' option is checked, null values will be returned in result, ie. middle value '"1",,3'.

The node can accept a multi-part input as long as the parts property is set correctly.

If outputting multiple messages they will have their parts property set and form a complete message sequence.

Note: the column template must be comma separated - even if a different separator is chosen for the data.