frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Add skip first n lines capability to csv node (#1535)

Browse Source 
* Initial implementation of skip first lines for css node

* add css skip lines tests
This commit is contained in:
Dave Conway-Jones 2018-01-11 22:02:58 +00:00 committed by Nick O'Leary
parent 161c7d30ca
commit 7c0b9ffe06
3 changed files with 141 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
nodes/core/parsers/70-CSV.html
View File

@ -24,29 +24,31 @@
</div>
<hr align="middle"/>
<div class="form-row">
<label style="width:100%; border-bottom: 1px solid #eee;"><span data-i18n="csv.label.c2o"></span></label>
<label style="width:100%; border-bottom:1px solid #eee;"><span data-i18n="csv.label.c2o"></span></label>
</div>
<div class="form-row" style="padding-left: 20px;">
<div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.input"></span></label>
<input style="width:20px; vertical-align:top; margin-right: 5px;" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span>
<span data-i18n="csv.label.skip-s"></span>&nbsp;<input type="text" id="node-input-skip" style="width:30px; height:25px;"/>&nbsp;<span data-i18n="csv.label.skip-e"></span><br/>
<label>&nbsp;</label>
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-hdrin"><label style="width:auto; margin-top:7px;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span>
</div>
<div class="form-row" style="padding-left: 20px;">
<div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label>
<select type="text" id="node-input-multi" style="width: 250px;">
<select type="text" id="node-input-multi" style="width:250px;">
<option value="one" data-i18n="csv.output.row"></option>
<option value="mult" data-i18n="csv.output.array"></option>
</select>
</div>
<div class="form-row" style="margin-top: 20px">
<label style="width:100%; border-bottom: 1px solid #eee;"><span data-i18n="csv.label.o2c"></span></label>
<div class="form-row" style="margin-top:20px">
<label style="width:100%; border-bottom:1px solid #eee;"><span data-i18n="csv.label.o2c"></span></label>
</div>
<div class="form-row" style="padding-left: 20px;">
<div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.output"></span></label>
<input style="width:20px; vertical-align:top; margin-right: 5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span>
<input style="width:20px; vertical-align:top; margin-right:5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span>
</div>
<div class="form-row" style="padding-left: 20px;">
<div class="form-row" style="padding-left:20px;">
<label></label>
<label style="width: auto; margin-right: 10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label>
<label style="width:auto; margin-right:10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label>
<select style="width:150px;" id="node-input-ret">
<option value='\n' data-i18n="csv.newline.linux"></option>
<option value='\r' data-i18n="csv.newline.mac"></option>
@ -95,7 +97,8 @@
hdrout: {value:""},
multi: {value:"one",required:true},
ret: {value:'\\n'},
temp: {value:""}
temp: {value:""},
skip: {value:"0"}
},
inputs:1,
outputs:1,
@ -107,6 +110,9 @@
return this.name?"node_label_italic":"";
},
oneditprepare: function() {
console.log(this.skip,$("#node-input-skip").val());
if (this.skip === undefined) { this.skip = 0; $("#node-input-skip").val("0");}
$("#node-input-skip").spinner({ min:0 });
if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " " || this.sep == "#") {
$("#node-input-select-sep").val(this.sep);
$("#node-input-sep").hide();

43
nodes/core/parsers/70-CSV.js
View File

@ -28,6 +28,8 @@ module.exports = function(RED) {
this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || false;
this.goodtmpl = true;
this.skip = parseInt(n.skip || 0);
this.store = [];
var tmpwarn = true;
var node = this;
@ -134,10 +136,12 @@ module.exports = function(RED) {
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var line = msg.payload;
var linecount = 0;
var tmp = "";
var reg = /^[-]?[0-9]*\.?[0-9]+$/;
if (msg.hasOwnProperty("parts")) {
if (msg.parts.index > 0) { first = false; }
linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
}
// For now we are just going to assume that any \r or \n means an end of line...
@ -145,8 +149,13 @@ module.exports = function(RED) {
// Now process the whole file/line
for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
continue;
}
if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")) { // look for first line break
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; }
node.template = clean(tmp.split(node.sep));
first = false;
}
@ -199,14 +208,27 @@ module.exports = function(RED) {
o[node.template[j]] = k[j];
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the aray
a.push(o); // add to the array
}
var has_parts = msg.hasOwnProperty("parts");
if (node.multi !== "one") {
msg.payload = a;
node.send(msg); // finally send the array
if (has_parts) {
if (JSON.stringify(o) !== "{}") {
node.store.push(o);
}
if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store;
delete msg.parts;
node.send(msg);
node.store = [];
}
}
else {
node.send(msg); // finally send the array
}
}
else {
var has_parts = msg.hasOwnProperty("parts");
var len = a.length;
for (var i = 0; i < len; i++) {
var newMessage = RED.util.cloneMessage(msg);
@ -218,13 +240,18 @@ module.exports = function(RED) {
count: len
};
}
else if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1;
newMessage.parts.count -= 1;
else {
newMessage.parts.index -= node.skip;
newMessage.parts.count -= node.skip;
if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1;
newMessage.parts.count -= 1;
}
}
node.send(newMessage);
}
}
node.linecount = 0;
}
catch(e) { node.error(e,msg); }
}

88
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -265,6 +265,94 @@ describe('CSV node', function() {
});
});
it('should skip several lines from start if requested', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" });
check_parts(msg, 0, 1);
done();
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should skip several lines from start then use next line as a tempate', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", hdrin:true, skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { "9": "C", "0": "D", "A": "E", "B": "F" });
check_parts(msg, 0, 1);
done();
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should skip several lines from start and correct parts', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
var c = 0;
n2.on("input", function(msg) {
if (c===0) {
msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" });
check_parts(msg, 0, 2);
c = c+1;
}
else {
msg.should.have.property('payload', { a: "C", b: "D", c: "E", d: "F" });
check_parts(msg, 1, 2);
done();
}
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should be able to skip and then use the first of multiple parts as a template if parts are present', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, skip:2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
var c = 0;
n2.on("input", function(msg) {
if (c === 0) {
msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 });
check_parts(msg, 0, 2);
c += 1;
}
else {
msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 });
check_parts(msg, 1, 2);
done();
}
});
var testStringA = "foo\n";
var testStringB = "bar\n";
var testString1 = "w,x,y,z\n";
var testString2 = "1,2,3,4\n";
var testString3 = "5,6,7,8\n";
n1.emit("input", {payload:testStringA, parts:{id:"X", index:0, count:5}});
n1.emit("input", {payload:testStringB, parts:{id:"X", index:1, count:5}});
n1.emit("input", {payload:testString1, parts:{id:"X", index:2, count:5}});
n1.emit("input", {payload:testString2, parts:{id:"X", index:3, count:5}});
n1.emit("input", {payload:testString3, parts:{id:"X", index:4, count:5}});
});
});
});
describe('json object to csv', function() {