frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

CSV better handling of messages with incoming parts - to create array output

Browse Source 
and add tests (apologies for the massive reformat of test file) - but honestly there are two new tests
This commit is contained in:
Dave Conway-Jones 2021-03-11 12:47:54 +00:00
parent 16b9abbe92
commit df90e3414d
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 864 additions and 815 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -158,19 +158,24 @@ module.exports = function(RED) {
var o = {}; // output object to build up var o = {}; // output object to build up
var a = []; // output array is needed for multiline option var a = []; // output array is needed for multiline option
var first = true; // is this the first line var first = true; // is this the first line
var last = false;
var line = msg.payload; var line = msg.payload;
var linecount = 0; var linecount = 0;
var tmp = ""; var tmp = "";
var has_parts = msg.hasOwnProperty("parts");
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i; var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
if (msg.hasOwnProperty("parts")) { if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index; linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; } if (msg.parts.index > node.skip) { first = false; }
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
} }
// For now we are just going to assume that any \r or \n means an end of line... // For now we are just going to assume that any \r or \n means an end of line...
// got to be a weird csv that has singleton \r \n in it for another reason... // got to be a weird csv that has singleton \r \n in it for another reason...
// Now process the whole file/line // Now process the whole file/line
var nocr = (line.match(/[\r\n]/g)||[]).length;
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
for (var i = 0; i < line.length; i++) { for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) { if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; } if (line[i] === "\n") { linecount += 1; }
@ -242,14 +247,14 @@ module.exports = function(RED) {
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j]; if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j]; if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array a.push(o); // add to the array
} }
var has_parts = msg.hasOwnProperty("parts");
if (node.multi !== "one") { if (node.multi !== "one") {
msg.payload = a; msg.payload = a;
if (has_parts) { if (has_parts && nocr <= 1) {
if (JSON.stringify(o) !== "{}") { if (JSON.stringify(o) !== "{}") {
node.store.push(o); node.store.push(o);
} }
@ -287,8 +292,12 @@ module.exports = function(RED) {
newMessage.parts.count -= 1; newMessage.parts.count -= 1;
} }
} }
if (last) { newMessage.complete = true; }
send(newMessage); send(newMessage);
} }
if (has_parts && last && len === 0) {
send({complete:true});
}
} }
node.linecount = 0; node.linecount = 0;
done(); done();

1666
test/nodes/core/parsers/70-CSV_spec.js
View File

File diff suppressed because it is too large Load Diff