mirror of
https://github.com/node-red/node-red.git
synced 2023-10-10 13:36:53 +02:00
Let CSV node only send headers once
(and then reset that on msg.reset) and also accept msg.columns csv string to set column headers if not specified in node. And Add tests
This commit is contained in:
Dave Conway-Jones
parent
fbfc74e5ca
commit
e969a1c97c
@ -28,7 +28,7 @@
|
||||
</div>
|
||||
<div class="form-row" style="padding-left:20px;">
|
||||
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.input"></span></label>
|
||||
<span data-i18n="csv.label.skip-s"></span> <input type="text" id="node-input-skip" style="width:30px; height:25px;"/> <span data-i18n="csv.label.skip-e"></span><br/>
|
||||
<span data-i18n="csv.label.skip-s"></span> <input type="text" id="node-input-skip" style="width:40px; height:25px;"/> <span data-i18n="csv.label.skip-e"></span><br/>
|
||||
<label> </label>
|
||||
<input style="width:20px; vertical-align:baseline; margin-right:5px;" type="checkbox" id="node-input-hdrin"><label style="width:auto; margin-top:7px;" for="node-input-hdrin"><span data-i18n="csv.label.firstrow"></span></label><br/>
|
||||
<label> </label>
|
||||
@ -49,8 +49,13 @@
|
||||
<label style="width:100%;"><span data-i18n="csv.label.o2c"></span></label>
|
||||
</div>
|
||||
<div class="form-row" style="padding-left:20px;">
|
||||
<label><i class="fa fa-sign-in"></i> <span data-i18n="csv.label.output"></span></label>
|
||||
<input style="width:20px; vertical-align:top; margin-right:5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span>
|
||||
<label><i class="fa fa-sign-out"></i> <span data-i18n="csv.label.output"></span></label>
|
||||
<!-- <input style="width:20px; vertical-align:top; margin-right:5px;" type="checkbox" id="node-input-hdrout"><label style="width:auto;" for="node-input-hdrout"><span data-i18n="csv.label.includerow"></span></span> -->
|
||||
<select style="width:60%" id="node-input-hdrout">
|
||||
<option value="none" data-i18n="csv.hdrout.none"></option>
|
||||
<option value="all" data-i18n="csv.hdrout.all"></option>
|
||||
<option value="once" data-i18n="csv.hdrout.once"></option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-row" style="padding-left:20px;">
|
||||
<label></label>
|
||||
@ -73,7 +78,7 @@
|
||||
sep: {value:',',required:true,validate:RED.validators.regex(/^.{1,2}$/)},
|
||||
//quo: {value:'"',required:true},
|
||||
hdrin: {value:""},
|
||||
hdrout: {value:""},
|
||||
hdrout: {value:"none"},
|
||||
multi: {value:"one",required:true},
|
||||
ret: {value:'\\n'},
|
||||
temp: {value:""},
|
||||
@ -92,6 +97,8 @@
|
||||
return this.name?"node_label_italic":"";
|
||||
},
|
||||
oneditprepare: function() {
|
||||
if (this.hdrout === false) { this.hdrout = "none"; $("#node-input-hdrout").val("none"); }
|
||||
if (this.hdrout === true) { this.hdrout = "all"; $("#node-input-hdrout").val("all");}
|
||||
if (this.strings === undefined) { this.strings = true; $("#node-input-strings").prop('checked', true); }
|
||||
if (this.skip === undefined) { this.skip = 0; $("#node-input-skip").val("0");}
|
||||
$("#node-input-skip").spinner({ min:0 });
|
||||
|
||||
@ -26,7 +26,7 @@ module.exports = function(RED) {
|
||||
this.lineend = "\n";
|
||||
this.multi = n.multi || "one";
|
||||
this.hdrin = n.hdrin || false;
|
||||
this.hdrout = n.hdrout || false;
|
||||
this.hdrout = n.hdrout || "none";
|
||||
this.goodtmpl = true;
|
||||
this.skip = parseInt(n.skip || 0);
|
||||
this.store = [];
|
||||
@ -34,6 +34,8 @@ module.exports = function(RED) {
|
||||
this.include_empty_strings = n.include_empty_strings || false;
|
||||
this.include_null_values = n.include_null_values || false;
|
||||
if (this.parsestrings === undefined) { this.parsestrings = true; }
|
||||
if (this.hdrout === false) { this.hdrout = "none"; }
|
||||
if (this.hdrout === true) { this.hdrout = "all"; }
|
||||
var tmpwarn = true;
|
||||
var node = this;
|
||||
|
||||
@ -51,14 +53,22 @@ module.exports = function(RED) {
|
||||
return col;
|
||||
}
|
||||
node.template = clean(node.template);
|
||||
node.hdrSent = false;
|
||||
|
||||
this.on("input", function(msg) {
|
||||
if (msg.hasOwnProperty("reset")) {
|
||||
node.hdrSent = false;
|
||||
}
|
||||
if (msg.hasOwnProperty("payload")) {
|
||||
if (typeof msg.payload == "object") { // convert object to CSV string
|
||||
try {
|
||||
var ou = "";
|
||||
if (node.hdrout) {
|
||||
if (node.hdrout !== "none" && node.hdrSent === false) {
|
||||
if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) {
|
||||
node.template = clean((msg.columns || "").split(","));
|
||||
}
|
||||
ou += node.template.join(node.sep) + node.ret;
|
||||
if (node.hdrout === "once") { node.hdrSent = true; }
|
||||
}
|
||||
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
|
||||
for (var s = 0; s < msg.payload.length; s++) {
|
||||
@ -77,13 +87,15 @@ module.exports = function(RED) {
|
||||
ou += msg.payload[s].join(node.sep) + node.ret;
|
||||
}
|
||||
else {
|
||||
if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) {
|
||||
node.template = clean((msg.columns || "").split(","));
|
||||
}
|
||||
if ((node.template.length === 1) && (node.template[0] === '')) {
|
||||
/* istanbul ignore else */
|
||||
if (tmpwarn === true) { // just warn about missing template once
|
||||
node.warn(RED._("csv.errors.obj_csv"));
|
||||
tmpwarn = false;
|
||||
}
|
||||
ou = "";
|
||||
for (var p in msg.payload[0]) {
|
||||
/* istanbul ignore else */
|
||||
if (msg.payload[0].hasOwnProperty(p)) {
|
||||
@ -127,6 +139,7 @@ module.exports = function(RED) {
|
||||
}
|
||||
}
|
||||
msg.payload = ou;
|
||||
msg.columns = node.template.join(',');
|
||||
if (msg.payload !== '') { node.send(msg); }
|
||||
}
|
||||
catch(e) { node.error(e,msg); }
|
||||
@ -187,7 +200,7 @@ module.exports = function(RED) {
|
||||
// if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
|
||||
k[j] = line.length - 1 === i ? null : "";
|
||||
}
|
||||
else if ((line[i] === "\n") || (line[i] === "\r")) { // handle multiple lines
|
||||
else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
|
||||
//console.log(j,k,o,k[j]);
|
||||
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
|
||||
if ( node.template[j] && (node.template[j] !== "") ) {
|
||||
@ -227,6 +240,7 @@ module.exports = function(RED) {
|
||||
a.push(o); // add to the array
|
||||
}
|
||||
var has_parts = msg.hasOwnProperty("parts");
|
||||
|
||||
if (node.multi !== "one") {
|
||||
msg.payload = a;
|
||||
if (has_parts) {
|
||||
@ -235,12 +249,15 @@ module.exports = function(RED) {
|
||||
}
|
||||
if (msg.parts.index + 1 === msg.parts.count) {
|
||||
msg.payload = node.store;
|
||||
console.log(node.template)
|
||||
msg.columns = node.template.filter(val => val).join(',');
|
||||
delete msg.parts;
|
||||
node.send(msg);
|
||||
node.store = [];
|
||||
}
|
||||
}
|
||||
else {
|
||||
msg.columns = node.template.filter(val => val).join(',');
|
||||
node.send(msg); // finally send the array
|
||||
}
|
||||
}
|
||||
@ -248,6 +265,7 @@ module.exports = function(RED) {
|
||||
var len = a.length;
|
||||
for (var i = 0; i < len; i++) {
|
||||
var newMessage = RED.util.cloneMessage(msg);
|
||||
newMessage.columns = node.template.filter(val => val).join(',');
|
||||
newMessage.payload = a[i];
|
||||
if (!has_parts) {
|
||||
newMessage.parts = {
|
||||
@ -273,7 +291,11 @@ module.exports = function(RED) {
|
||||
}
|
||||
else { node.warn(RED._("csv.errors.csv_js")); }
|
||||
}
|
||||
else { node.send(msg); } // If no payload - just pass it on.
|
||||
else {
|
||||
if (!msg.hasOwnProperty("reset")) {
|
||||
node.send(msg); // If no payload and not reset - just pass it on.
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
RED.nodes.registerType("csv",CSVNode);
|
||||
|
||||
@ -723,6 +723,11 @@
|
||||
"mac": "Mac (\\r)",
|
||||
"windows": "Windows (\\r\\n)"
|
||||
},
|
||||
"hdrout": {
|
||||
"none": "never send column headers",
|
||||
"all": "always send column headers",
|
||||
"once": "send headers once, until msg.reset"
|
||||
},
|
||||
"errors": {
|
||||
"csv_js": "This node only handles CSV strings or js objects.",
|
||||
"obj_csv": "No columns template specified for object -> CSV."
|
||||
|
||||
@ -38,11 +38,13 @@
|
||||
<p>The column template can contain an ordered list of column names. When converting CSV to an object, the column names
|
||||
will be used as the property names. Alternatively, the column names can be taken from the first row of the CSV.</p>
|
||||
<p>When converting to CSV, the column template is used to identify which properties to extract from the object and in what order.</p>
|
||||
<p>If the template is blank then the node can use a simple comma separated list of properties supplied in <code>msg.columns</code> to
|
||||
determine what to extract. If that is not present then all the object properties are ouput in the order in which they are found.</p>
|
||||
<p>If the input is an array then the columns template is only used to optionally generate a row of column titles.</p>
|
||||
<p>If 'parse numerical values' option is checked, string numerical values will be returned as numbers, ie. middle value '1,"1.5",2'.</p>
|
||||
<p>If 'include empty strings' option is checked, empty strings will be returned in result, ie. middle value '"1","",3'.</p>
|
||||
<p>If 'include null values' option is checked, null values will be returned in result, ie. middle value '"1",,3'.</p>
|
||||
<p>The node can accept a multi-part input as long as the <code>parts</code> property is set correctly.</p>
|
||||
<p>The node can accept a multi-part input as long as the <code>parts</code> property is set correctly, for example from a file-in node or split node.</p>
|
||||
<p>If outputting multiple messages they will have their <code>parts</code> property set and form a complete message sequence.</p>
|
||||
<p><b>Note:</b> the column template must be comma separated - even if a different separator is chosen for the data.</p>
|
||||
</script>
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
/* eslint-disable no-undef */
|
||||
/**
|
||||
* Copyright JS Foundation and other contributors, http://js.foundation
|
||||
*
|
||||
@ -76,6 +77,7 @@ describe('CSV node', function() {
|
||||
var n2 = helper.getNode("n2");
|
||||
n2.on("input", function(msg) {
|
||||
msg.should.have.property('payload', { a: 1, b: 2, c: 3, d: 4 });
|
||||
msg.should.have.property('columns', "a,b,c,d");
|
||||
check_parts(msg, 0, 1);
|
||||
done();
|
||||
});
|
||||
@ -108,6 +110,7 @@ describe('CSV node', function() {
|
||||
var n2 = helper.getNode("n2");
|
||||
n2.on("input", function(msg) {
|
||||
msg.should.have.property('payload', { col1: 1, col2: 2, col3: 3, col4: 4 });
|
||||
msg.should.have.property('columns', "col1,col2,col3,col4");
|
||||
check_parts(msg, 0, 1);
|
||||
done();
|
||||
});
|
||||
@ -124,6 +127,7 @@ describe('CSV node', function() {
|
||||
var n2 = helper.getNode("n2");
|
||||
n2.on("input", function(msg) {
|
||||
msg.should.have.property('payload', { a: 1, d: 4 });
|
||||
msg.should.have.property('columns', 'a,d');
|
||||
check_parts(msg, 0, 1);
|
||||
done();
|
||||
});
|
||||
@ -283,6 +287,7 @@ describe('CSV node', function() {
|
||||
var n2 = helper.getNode("n2");
|
||||
n2.on("input", function(msg) {
|
||||
msg.should.have.property('payload', [ { a: 1, b: 2, c: 3, d: 4 },{ a: 5, b: -6, c: '07', d: '+8' },{ a: 9, b: 0, c: 'a', d: 'b' },{ a: 'c', b: 'd', c: 'e', d: 'f' } ]);
|
||||
msg.should.have.property('columns','a,b,c,d');
|
||||
msg.should.not.have.property('parts');
|
||||
done();
|
||||
});
|
||||
@ -367,7 +372,7 @@ describe('CSV node', function() {
|
||||
});
|
||||
});
|
||||
|
||||
it('should skip several lines from start then use next line as a tempate', function(done) {
|
||||
it('should skip several lines from start then use next line as a template', function(done) {
|
||||
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", hdrin:true, skip: 2, wires:[["n2"]] },
|
||||
{id:"n2", type:"helper"} ];
|
||||
helper.load(csvNode, flow, function() {
|
||||
@ -417,11 +422,13 @@ describe('CSV node', function() {
|
||||
n2.on("input", function(msg) {
|
||||
if (c === 0) {
|
||||
msg.should.have.property('payload', { w: 1, x: 2, y: 3, z: 4 });
|
||||
msg.should.have.property('columns', 'w,x,y,z');
|
||||
check_parts(msg, 0, 2);
|
||||
c += 1;
|
||||
}
|
||||
else {
|
||||
msg.should.have.property('payload', { w: 5, x: 6, y: 7, z: 8 });
|
||||
msg.should.have.property('columns', 'w,x,y,z');
|
||||
check_parts(msg, 1, 2);
|
||||
done();
|
||||
}
|
||||
@ -569,6 +576,33 @@ describe('CSV node', function() {
|
||||
});
|
||||
});
|
||||
|
||||
it('should be able to pass in column names', function(done) {
|
||||
var flow = [ { id:"n1", type:"csv", temp:"", hdrout:"once", ret:"\r\n", wires:[["n2"]] },
|
||||
{id:"n2", type:"helper"} ];
|
||||
helper.load(csvNode, flow, function() {
|
||||
var n1 = helper.getNode("n1");
|
||||
var n2 = helper.getNode("n2");
|
||||
var count = 0;
|
||||
n2.on("input", function(msg) {
|
||||
count += 1;
|
||||
try {
|
||||
if (count === 1) {
|
||||
msg.should.have.property('payload', 'a,,b,a\r\n4,,3,4\r\n');
|
||||
}
|
||||
if (count === 3) {
|
||||
msg.should.have.property('payload', '4,,3,4\r\n');
|
||||
done()
|
||||
}
|
||||
}
|
||||
catch(e) { done(e); }
|
||||
});
|
||||
var testJson = [{ d: 1, b: 3, c: 2, a: 4 }];
|
||||
n1.emit("input", {payload:testJson, columns:"a,,b,a"});
|
||||
n1.emit("input", {payload:testJson});
|
||||
n1.emit("input", {payload:testJson});
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle quotes and sub-properties', function(done) {
|
||||
var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d", wires:[["n2"]] },
|
||||
{id:"n2", type:"helper"} ];
|
||||
|
||||
Loading…
Reference in New Issue
Block a user