frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

CSV better handling of messages with incoming parts - to create array output

Browse Source 
and add tests (apologies for the massive reformat of test file) - but honestly there are two new tests
This commit is contained in:
Dave Conway-Jones 2021-03-11 12:47:54 +00:00
parent 16b9abbe92
commit df90e3414d
No known key found for this signature in database
GPG Key ID: 88BA2B8A411BE9FF
2 changed files with 864 additions and 815 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -158,19 +158,24 @@ module.exports = function(RED) {
var o = {}; // output object to build up
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var last = false;
var line = msg.payload;
var linecount = 0;
var tmp = "";
var has_parts = msg.hasOwnProperty("parts");
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
}
// For now we are just going to assume that any \r or \n means an end of line...
// got to be a weird csv that has singleton \r \n in it for another reason...
// Now process the whole file/line
var nocr = (line.match(/[\r\n]/g)||[]).length;
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
@ -242,14 +247,14 @@ module.exports = function(RED) {
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array
}
var has_parts = msg.hasOwnProperty("parts");
if (node.multi !== "one") {
msg.payload = a;
if (has_parts) {
if (has_parts && nocr <= 1) {
if (JSON.stringify(o) !== "{}") {
node.store.push(o);
}
@ -287,8 +292,12 @@ module.exports = function(RED) {
newMessage.parts.count -= 1;
}
}
if (last) { newMessage.complete = true; }
send(newMessage);
}
if (has_parts && last && len === 0) {
send({complete:true});
}
}
node.linecount = 0;
done();

42
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -15,9 +15,10 @@
* limitations under the License.
**/
var should = require("should");
// var should = require("should");
var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js");
var helper = require("node-red-node-test-helper");
// const { neq } = require("semver");
describe('CSV node', function() {
@ -379,6 +380,45 @@
});
});
it('should be able to create an array from multiple parts', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, multi:"mult", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', [{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6},{"a":7,"b":8,"c":9}]);
msg.should.have.property('columns','a,b,c');
msg.should.not.have.property('parts');
done();
});
n1.emit("input", {"payload":"a,b,c","parts":{"index":0,"ch":"\n","type":"string","id":"1"}});
n1.emit("input", {"payload":"1,2,3","parts":{"index":1,"ch":"\n","type":"string","id":"1"}});
n1.emit("input", {"payload":"4,5,6","parts":{"index":2,"ch":"\n","type":"string","id":"1"}});
n1.emit("input", {"payload":"7,8,9","parts":{"index":3,count:4,"ch":"\n","type":"string","id":"1"}});
});
});
it('should be able to output multiple objects as an array from an input of parts', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, multi:"yes", wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', [{"Col1":"V1","Col2":"V2"},{"Col1":"V3","Col2":"V4"},{"Col1":"V5","Col2":"V6"}]);
msg.should.have.property('columns','Col1,Col2');
msg.should.have.property('parts');
done();
});
//var testString = "1,2,3,4\n5,-6,07,+8\n9,0,a,b\nc,d,e,f";
// n1.emit("input", {payload:testString});
n1.emit("input", {"payload":"Col1,Col2\nV1,V2\nV3,V4\nV5,V6","topic":"","parts":{"id":"3af07e18.865652","type":"array","count":2,"len":1,"index":0}});
//n1.emit("input", {"payload":"Var1,Var2\nW1,W2\nW3,W4\nW5,W6","topic":"","parts":{"id":"3af07e18.865652","type":"array","count":2,"len":1,"index":1}});
});
});
it('should handle numbers in strings but not IP addresses', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e", wires:[["n2"]] },
{id:"n2", type:"helper"} ];