frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Add skip first n lines capability to csv node (#1535)

Browse Source 
* Initial implementation of skip first lines for css node

* add css skip lines tests
This commit is contained in:
Dave Conway-Jones 2018-01-11 22:02:58 +00:00 committed by Nick O'Leary
parent 161c7d30ca
commit 7c0b9ffe06
3 changed files with 141 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
nodes/core/parsers/70-CSV.html
View File

@ -24,29 +24,31 @@
</div> </div>
<hr align="middle"/> <hr align="middle"/>
<div class="form-row"> <div class="form-row">
<label style="width:100%; border-bottom: 1px solid #eee;"><span data-i18n="csv.label.c2o"></span></label> <label style="width:100%; border-bottom:1px solid #eee;"><span data-i18n="csv.label.c2o"></span></label>
</div> </div>
<div class="form-row" style="padding-left: 20px;"> <div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.input"></span></label> <label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.input"></span></label>
<input style="width:20px; vertical-align:top; margin-right: 5px;" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span> <span data-i18n="csv.label.skip-s"></span>&nbsp;<input type="text" id="node-input-skip" style="width:30px; height:25px;"/>&nbsp;<span data-i18n="csv.label.skip-e"></span><br/>
<label>&nbsp;</label>
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-hdrin"><label style="width:auto; margin-top:7px;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span>
</div> </div>
<div class="form-row" style="padding-left: 20px;"> <div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label> <label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label>
<select type="text" id="node-input-multi" style="width: 250px;"> <select type="text" id="node-input-multi" style="width:250px;">
<option value="one" data-i18n="csv.output.row"></option> <option value="one" data-i18n="csv.output.row"></option>
<option value="mult" data-i18n="csv.output.array"></option> <option value="mult" data-i18n="csv.output.array"></option>
</select> </select>
</div> </div>
<div class="form-row" style="margin-top: 20px"> <div class="form-row" style="margin-top:20px">
<label style="width:100%; border-bottom: 1px solid #eee;"><span data-i18n="csv.label.o2c"></span></label> <label style="width:100%; border-bottom:1px solid #eee;"><span data-i18n="csv.label.o2c"></span></label>
</div> </div>
<div class="form-row" style="padding-left: 20px;"> <div class="form-row" style="padding-left:20px;">
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.output"></span></label> <label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.output"></span></label>
<input style="width:20px; vertical-align:top; margin-right: 5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span> <input style="width:20px; vertical-align:top; margin-right:5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span>
</div> </div>
<div class="form-row" style="padding-left: 20px;"> <div class="form-row" style="padding-left:20px;">
<label></label> <label></label>
<label style="width: auto; margin-right: 10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label> <label style="width:auto; margin-right:10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label>
<select style="width:150px;" id="node-input-ret"> <select style="width:150px;" id="node-input-ret">
<option value='\n' data-i18n="csv.newline.linux"></option> <option value='\n' data-i18n="csv.newline.linux"></option>
<option value='\r' data-i18n="csv.newline.mac"></option> <option value='\r' data-i18n="csv.newline.mac"></option>
@ -95,7 +97,8 @@
hdrout: {value:""}, hdrout: {value:""},
multi: {value:"one",required:true}, multi: {value:"one",required:true},
ret: {value:'\\n'}, ret: {value:'\\n'},
temp: {value:""} temp: {value:""},
skip: {value:"0"}
}, },
inputs:1, inputs:1,
outputs:1, outputs:1,
@ -107,6 +110,9 @@
return this.name?"node_label_italic":""; return this.name?"node_label_italic":"";
}, },
oneditprepare: function() { oneditprepare: function() {
console.log(this.skip,$("#node-input-skip").val());
if (this.skip === undefined) { this.skip = 0; $("#node-input-skip").val("0");}
$("#node-input-skip").spinner({ min:0 });
if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " " || this.sep == "#") { if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " " || this.sep == "#") {
$("#node-input-select-sep").val(this.sep); $("#node-input-select-sep").val(this.sep);
$("#node-input-sep").hide(); $("#node-input-sep").hide();

43
nodes/core/parsers/70-CSV.js
View File

@ -28,6 +28,8 @@ module.exports = function(RED) {
this.hdrin = n.hdrin || false; this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || false; this.hdrout = n.hdrout || false;
this.goodtmpl = true; this.goodtmpl = true;
this.skip = parseInt(n.skip || 0);
this.store = [];
var tmpwarn = true; var tmpwarn = true;
var node = this; var node = this;
@ -134,10 +136,12 @@ module.exports = function(RED) {
var a = []; // output array is needed for multiline option var a = []; // output array is needed for multiline option
var first = true; // is this the first line var first = true; // is this the first line
var line = msg.payload; var line = msg.payload;
var linecount = 0;
var tmp = ""; var tmp = "";
var reg = /^[-]?[0-9]*\.?[0-9]+$/; var reg = /^[-]?[0-9]*\.?[0-9]+$/;
if (msg.hasOwnProperty("parts")) { if (msg.hasOwnProperty("parts")) {
if (msg.parts.index > 0) { first = false; } linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
} }
// For now we are just going to assume that any \r or \n means an end of line... // For now we are just going to assume that any \r or \n means an end of line...
@ -145,8 +149,13 @@ module.exports = function(RED) {
// Now process the whole file/line // Now process the whole file/line
for (var i = 0; i < line.length; i++) { for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
continue;
}
if ((node.hdrin === true) && first) { // if the template is in the first line if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")) { // look for first line break if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; }
node.template = clean(tmp.split(node.sep)); node.template = clean(tmp.split(node.sep));
first = false; first = false;
} }
@ -199,14 +208,27 @@ module.exports = function(RED) {
o[node.template[j]] = k[j]; o[node.template[j]] = k[j];
} }
if (JSON.stringify(o) !== "{}") { // don't send empty objects if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the aray a.push(o); // add to the array
} }
var has_parts = msg.hasOwnProperty("parts");
if (node.multi !== "one") { if (node.multi !== "one") {
msg.payload = a; msg.payload = a;
node.send(msg); // finally send the array if (has_parts) {
if (JSON.stringify(o) !== "{}") {
node.store.push(o);
}
if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store;
delete msg.parts;
node.send(msg);
node.store = [];
}
}
else {
node.send(msg); // finally send the array
}
} }
else { else {
var has_parts = msg.hasOwnProperty("parts");
var len = a.length; var len = a.length;
for (var i = 0; i < len; i++) { for (var i = 0; i < len; i++) {
var newMessage = RED.util.cloneMessage(msg); var newMessage = RED.util.cloneMessage(msg);
@ -218,13 +240,18 @@ module.exports = function(RED) {
count: len count: len
}; };
} }
else if (node.hdrin) { // if we removed the header line then shift the counts by 1 else {
newMessage.parts.index -= 1; newMessage.parts.index -= node.skip;
newMessage.parts.count -= 1; newMessage.parts.count -= node.skip;
if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1;
newMessage.parts.count -= 1;
}
} }
node.send(newMessage); node.send(newMessage);
} }
} }
node.linecount = 0;
} }
catch(e) { node.error(e,msg); } catch(e) { node.error(e,msg); }
} }

88
test/nodes/core/parsers/70-CSV_spec.js
View File

@ -265,6 +265,94 @@ describe('CSV node', function() {
}); });
}); });
it('should skip several lines from start if requested', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" });
check_parts(msg, 0, 1);
done();
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should skip several lines from start then use next line as a tempate', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", hdrin:true, skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
msg.should.have.property('payload', { "9": "C", "0": "D", "A": "E", "B": "F" });
check_parts(msg, 0, 1);
done();
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should skip several lines from start and correct parts', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", skip: 2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
var c = 0;
n2.on("input", function(msg) {
if (c===0) {
msg.should.have.property('payload', { a: 9, b: 0, c: "A", d: "B" });
check_parts(msg, 0, 2);
c = c+1;
}
else {
msg.should.have.property('payload', { a: "C", b: "D", c: "E", d: "F" });
check_parts(msg, 1, 2);
done();
}
});
var testString = "1,2,3,4"+String.fromCharCode(10)+"5,6,7,8"+String.fromCharCode(10)+"9,0,A,B"+String.fromCharCode(10)+"C,D,E,F"+String.fromCharCode(10);
n1.emit("input", {payload:testString});
});
});
it('should be able to skip and then use the first of multiple parts as a template if parts are present', function(done) {
var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, skip:2, wires:[["n2"]] },
{id:"n2", type:"helper"} ];
helper.load(csvNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
var c = 0;
n2.on("input", function(msg) {
if (c === 0) {
msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 });
check_parts(msg, 0, 2);
c += 1;
}
else {
msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 });
check_parts(msg, 1, 2);
done();
}
});
var testStringA = "foo\n";
var testStringB = "bar\n";
var testString1 = "w,x,y,z\n";
var testString2 = "1,2,3,4\n";
var testString3 = "5,6,7,8\n";
n1.emit("input", {payload:testStringA, parts:{id:"X", index:0, count:5}});
n1.emit("input", {payload:testStringB, parts:{id:"X", index:1, count:5}});
n1.emit("input", {payload:testString1, parts:{id:"X", index:2, count:5}});
n1.emit("input", {payload:testString2, parts:{id:"X", index:3, count:5}});
n1.emit("input", {payload:testString3, parts:{id:"X", index:4, count:5}});
});
});
}); });
describe('json object to csv', function() { describe('json object to csv', function() {