frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

[CSV node] Add support for parsing empty strings and null values (#2510)

Browse Source 
* [CSV node] Add support for parsing empty strings and null values

* Add new lines at the end and fix script type.

* Last one script type fix

* Naming change
This commit is contained in:
tmdoit 2020-03-18 16:54:10 +01:00 committed by GitHub
parent 6912dec166
commit 91b7dd988e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 41 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.html vendored
View File

@ -33,6 +33,10 @@
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-hdrin"><label style="width:auto; margin-top:7px;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span></label><br/> <input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-hdrin"><label style="width:auto; margin-top:7px;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span></label><br/>
<label>&nbsp;</label> <label>&nbsp;</label>
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-strings"><label style="width:auto; margin-top:7px;" for="node-input-strings"><span data-i18n="csv.label.usestrings"></span></label><br/> <input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-strings"><label style="width:auto; margin-top:7px;" for="node-input-strings"><span data-i18n="csv.label.usestrings"></span></label><br/>
<label>&nbsp;</label>
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-include_empty_strings"><label style="width:auto; margin-top:7px;" for="node-input-include_empty_strings"><span data-i18n="csv.label.include_empty_strings"></span></label><br/>
<label>&nbsp;</label>
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-include_null_values"><label style="width:auto; margin-top:7px;" for="node-input-include_null_values"><span data-i18n="csv.label.include_null_values"></span></label><br/>
</div> </div>
<div class="form-row" style="padding-left:20px;"> <div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label> <label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label>
@ -74,7 +78,9 @@
ret: {value:'\\n'}, ret: {value:'\\n'},
temp: {value:""}, temp: {value:""},
skip: {value:"0"}, skip: {value:"0"},
strings: {value:true} strings: {value:true},
include_empty_strings: {value:""},
include_null_values: {value:""}
}, },
inputs:1, inputs:1,
outputs:1, outputs:1,

38
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -31,6 +31,8 @@ module.exports = function(RED) {
this.skip = parseInt(n.skip || 0); this.skip = parseInt(n.skip || 0);
this.store = []; this.store = [];
this.parsestrings = n.strings; this.parsestrings = n.strings;
this.include_empty_strings = n.include_empty_strings || false;
this.include_null_values = n.include_null_values || false;
if (this.parsestrings === undefined) { this.parsestrings = true; } if (this.parsestrings === undefined) { this.parsestrings = true; }
var tmpwarn = true; var tmpwarn = true;
var node = this; var node = this;
@ -173,20 +175,29 @@ module.exports = function(RED) {
} }
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) { if ( node.template[j] && (node.template[j] !== "") ) {
if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } // if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
o[node.template[j]] = k[j]; if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j];
} }
j += 1; j += 1;
k[j] = ""; // if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
k[j] = line.length - 1 === i ? null : "";
} }
else if ((line[i] === "\n") || (line[i] === "\r")) { // handle multiple lines else if ((line[i] === "\n") || (line[i] === "\r")) { // handle multiple lines
//console.log(j,k,o,k[j]); //console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) { if ( node.template[j] && (node.template[j] !== "") ) {
if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } // if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
else { k[j].replace(/\r$/,''); } if (line[i-1] === node.sep) k[j] = null;
o[node.template[j]] = k[j]; if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array a.push(o); // add to the array
@ -204,10 +215,13 @@ module.exports = function(RED) {
// Finished so finalize and send anything left // Finished so finalize and send anything left
//console.log(j,k,o,k[j]); //console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
if ( (node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); } if ( node.template[j] && (node.template[j] !== "") ) {
else { k[j].replace(/\r$/,''); } if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
o[node.template[j]] = k[j]; else { if (k[j] !== null) k[j].replace(/\r$/,''); }
if (node.include_null_values && k[j] === null) o[node.template[j]] = k[j];
if (node.include_empty_strings && k[j] === "") o[node.template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[node.template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array a.push(o); // add to the array

7
packages/node_modules/@node-red/nodes/locales/en-US/messages.json vendored
View File

@ -698,7 +698,9 @@
"output": "Output", "output": "Output",
"includerow": "include column name row", "includerow": "include column name row",
"newline": "Newline", "newline": "Newline",
"usestrings": "parse numerical values" "usestrings": "parse numerical values",
"include_empty_strings": "include empty strings",
"include_null_values": "include null values"
}, },
"placeholder": { "placeholder": {
"columns": "comma-separated column names" "columns": "comma-separated column names"
@ -894,7 +896,8 @@
"fixup": "Fix-up exp" "fixup": "Fix-up exp"
}, },
"errors": { "errors": {
"invalid-expr": "Invalid JSONata expression: __error__" "invalid-expr": "Invalid JSONata expression: __error__",
"invalid-type": "Cannot join __error__ to buffer"
} }
}, },
"sort" : { "sort" : {

3
packages/node_modules/@node-red/nodes/locales/en-US/parsers/70-CSV.html vendored
View File

@ -39,6 +39,9 @@
will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.</p> will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.</p>
<p>When converting to CSV, the column template is used to identify which properties to extract from the object and in what order.</p> <p>When converting to CSV, the column template is used to identify which properties to extract from the object and in what order.</p>
<p>If the input is an array then the columns template is only used to optionally generate a row of column titles.</p> <p>If the input is an array then the columns template is only used to optionally generate a row of column titles.</p>
<p>If 'parse numerical values' option is checked, string numerical values will be returned as numbers, ie. middle value '1,"1.5",2'.</p>
<p>If 'include empty strings' option is checked, empty strings will be returned in result, ie. middle value '"1","",3'.</p>
<p>If 'include null values' option is checked, null values will be returned in result, ie. middle value '"1",,3'.</p>
<p>The node can accept a multi-part input as long as the <code>parts</code> property is set correctly.</p> <p>The node can accept a multi-part input as long as the <code>parts</code> property is set correctly.</p>
<p>If outputting multiple messages they will have their <code>parts</code> property set and form a complete message sequence.</p> <p>If outputting multiple messages they will have their <code>parts</code> property set and form a complete message sequence.</p>
<p><b>Note:</b> the column template must be comma separated - even if a different separator is chosen for the data.</p> <p><b>Note:</b> the column template must be comma separated - even if a different separator is chosen for the data.</p>