mirror of
https://github.com/node-red/node-red.git
synced 2023-10-10 13:36:53 +02:00
Major Update to CSV node.
now handles lines, files, column names in first row, etc etc
This commit is contained in:
Dave C-J
parent
429a87f88a
commit
7e2dbb13e4
@ -1,3 +1,4 @@
|
||||
|
||||
<!--
|
||||
Copyright 2014 IBM Corp.
|
||||
|
||||
@ -17,29 +18,60 @@
|
||||
<script type="text/x-red" data-template-name="csv">
|
||||
<div class="form-row">
|
||||
<label for="node-input-temp"><i class="fa fa-list"></i> Columns</label>
|
||||
<input type="text" id="node-input-temp" placeholder="A,B,C,D">
|
||||
<input type="text" id="node-input-temp" placeholder="comma-separated column names">
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label for="node-input-sep"><i class="fa fa-chevron-right"></i> Separator</label>
|
||||
<input type="text" id="node-input-sep" placeholder=','>
|
||||
<label for="node-input-select-sep"><i class="fa fa-text-width"></i> Separator</label>
|
||||
<select style="width: 150px" id="node-input-select-sep">
|
||||
<option value=",">comma</option>
|
||||
<option value="\t">tab</option>
|
||||
<option value=" ">space</option>
|
||||
<option value=";">semicolon</option>
|
||||
<option value=":">colon</option>
|
||||
<option value="">other...</option>
|
||||
</select>
|
||||
<input style="width: 40px;" type="text" id="node-input-sep" pattern=".">
|
||||
</div>
|
||||
<!-- <div class="form-row">
|
||||
<label for="node-input-quo"><i class="fa fa-tag"></i> Escape</label>
|
||||
<input type="text" id="node-input-quo" placeholder='"'>
|
||||
</div> -->
|
||||
<br/>
|
||||
|
||||
<div class="form-row">
|
||||
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
|
||||
<input type="text" id="node-input-name" placeholder="Name">
|
||||
</div>
|
||||
<div id="node-input-tip" class="form-tips">Tip: you can use "\t" for tab separator.</div>
|
||||
<hr align="middle"/>
|
||||
<div class="form-row">
|
||||
<label style="width: 100%;"><i class="fa fa-gears"></i> CSV-to-Object options</label>
|
||||
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Input</label>
|
||||
<input style="width: 30px" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin">first row contains column names</span>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-out"></i> Output</label>
|
||||
<select type="text" id="node-input-multi" style="width: 250px;">
|
||||
<option value="one">a message per row</option>
|
||||
<option value="mult">a single message [array]</option>
|
||||
</select>
|
||||
</div>
|
||||
<hr align="middle"/>
|
||||
<div class="form-row">
|
||||
<label style="width: 100%;"><i class="fa fa-gears"></i> Object-to-CSV options</label>
|
||||
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Output</label>
|
||||
<input style="width: 30px" type="checkbox" id="node-input-hdrout"><label style="width: auto;" for="node-input-hdrout">include column name row</span>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-align-left"></i> Newline</label>
|
||||
<select style="width: 150px" id="node-input-ret">
|
||||
<option value='\n'>Linux (\n)</option>
|
||||
<option value='\r'>Mac (\r)</option>
|
||||
<option value='\r\n'>Windows (\r\n)</option>
|
||||
</select>
|
||||
</div>
|
||||
</script>
|
||||
|
||||
<script type="text/x-red" data-help-name="csv">
|
||||
<p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object. Places the result in the payload.</p>
|
||||
<p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object.
|
||||
Places the result in the payload.</p>
|
||||
<p>If the input is a string it tries to parse it as CSV and creates a javascript object.</p>
|
||||
<p>If the input is a javascript object it tries to build a CSV string.</p>
|
||||
<p>A columns template MUST be specified that contains the ordered list of column headers. For csv input these become the property names.
|
||||
<p>The columns template should contain an ordered list of column headers. For csv input these become the property names.
|
||||
For csv output these specify the properties to extract from the object and the order for the csv.</p>
|
||||
<p><b>Note:</b> the columns should always be specified comma separated - even if another separator is chosen for the data.</p>
|
||||
</script>
|
||||
@ -50,9 +82,13 @@
|
||||
color:"#DEBD5C",
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
sep: {value:',',required:true},
|
||||
sep: {value:',',required:true,validate:RED.validators.regex(/^.{1,2}$/)},
|
||||
//quo: {value:'"',required:true},
|
||||
temp: {value:"",required:true}
|
||||
hdrin: {value:""},
|
||||
hdrout: {value:""},
|
||||
multi: {value:"one",required:true},
|
||||
ret: {value:'\\n'},
|
||||
temp: {value:""}
|
||||
},
|
||||
inputs:1,
|
||||
outputs:1,
|
||||
@ -62,6 +98,26 @@
|
||||
},
|
||||
labelStyle: function() {
|
||||
return this.name?"node_label_italic":"";
|
||||
},
|
||||
oneditprepare: function() {
|
||||
if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " ") {
|
||||
$("#node-input-select-sep").val(this.sep);
|
||||
$("#node-input-sep").hide();
|
||||
} else {
|
||||
$("#node-input-select-sep").val("");
|
||||
$("#node-input-sep").val(this.sep);
|
||||
$("#node-input-sep").show();
|
||||
}
|
||||
$("#node-input-select-sep").change(function() {
|
||||
var v = $("#node-input-select-sep option:selected").val();
|
||||
$("#node-input-sep").val(v);
|
||||
if (v == "") {
|
||||
$("#node-input-sep").val("");
|
||||
$("#node-input-sep").show().focus();
|
||||
} else {
|
||||
$("#node-input-sep").hide();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
@ -19,64 +19,131 @@ module.exports = function(RED) {
|
||||
function CSVNode(n) {
|
||||
RED.nodes.createNode(this,n);
|
||||
this.template = n.temp.split(",");
|
||||
this.sep = n.sep || ',';
|
||||
this.sep = this.sep.replace("\\n","\n").replace("\\r","\r").replace("\\t","\t");
|
||||
this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
|
||||
this.quo = '"';
|
||||
this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
|
||||
this.winflag = (this.ret === "\r\n");
|
||||
this.lineend = "\n";
|
||||
this.multi = n.multi || "one";
|
||||
this.hdrin = n.hdrin || false;
|
||||
this.hdrout = n.hdrout || false;
|
||||
this.goodtmpl = true;
|
||||
var node = this;
|
||||
|
||||
for (var t = 0; t < node.template.length; t++) {
|
||||
node.template[t] = node.template[t].trim(); // remove leading and trailing whitespace
|
||||
if (node.template[t].charAt(0) === '"' && node.template[t].charAt(node.template[t].length -1) === '"') {
|
||||
// remove leading and trialing quotes (if they exist) - and remove whitepace again.
|
||||
node.template[t] = node.template[t].substr(1,node.template[t].length -2).trim();
|
||||
// pass in an array of column names to be trimed, de-quoted and retrimed
|
||||
var clean = function(col) {
|
||||
for (var t = 0; t < col.length; t++) {
|
||||
col[t] = col[t].trim(); // remove leading and trailing whitespace
|
||||
if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
|
||||
// remove leading and trailing quotes (if they exist) - and remove whitepace again.
|
||||
col[t] = col[t].substr(1,col[t].length -2).trim();
|
||||
}
|
||||
}
|
||||
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
|
||||
else { node.goodtmpl = true; }
|
||||
return col;
|
||||
}
|
||||
node.template = clean(node.template);
|
||||
|
||||
this.on("input", function(msg) {
|
||||
if (msg.hasOwnProperty("payload")) {
|
||||
if (typeof msg.payload == "object") { // convert to csv
|
||||
if (typeof msg.payload == "object") { // convert object to CSV string
|
||||
try {
|
||||
var ou = "";
|
||||
for (var t in node.template) {
|
||||
if (msg.payload.hasOwnProperty(node.template[t])) {
|
||||
if (msg.payload[node.template[t]].indexOf(node.sep) != -1) {
|
||||
ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
|
||||
}
|
||||
else if (msg.payload[node.template[t]].indexOf(node.quo) != -1) {
|
||||
msg.payload[node.template[t]] = msg.payload[node.template[t]].replace(/"/g, '""');
|
||||
ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
|
||||
}
|
||||
else { ou += msg.payload[node.template[t]] + node.sep; }
|
||||
}
|
||||
if (node.hdrout) {
|
||||
ou += node.template.join(node.sep) + node.ret;
|
||||
}
|
||||
msg.payload = ou.slice(0,-1);
|
||||
node.send(msg);
|
||||
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
|
||||
for (var s = 0; s < msg.payload.length; s++) {
|
||||
for (var t=0; t < node.template.length; t++) {
|
||||
|
||||
// aaargh - resorting to eval here - but fairly contained front and back.
|
||||
var p = RED.util.ensureString(eval("msg.payload[s]."+node.template[t]));
|
||||
|
||||
if (p === "undefined") { p = ""; }
|
||||
if (p.indexOf(node.sep) != -1) { // add quotes if any "commas"
|
||||
ou += node.quo + p + node.quo + node.sep;
|
||||
}
|
||||
else if (p.indexOf(node.quo) != -1) { // add double quotes if any quotes
|
||||
p = p.replace(/"/g, '""');
|
||||
ou += node.quo + p + node.quo + node.sep;
|
||||
}
|
||||
else { ou += p + node.sep; } // otherwise just add
|
||||
}
|
||||
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
|
||||
}
|
||||
node.send({payload:ou});
|
||||
}
|
||||
catch(e) { node.log(e); }
|
||||
}
|
||||
else if (typeof msg.payload == "string") { // convert to object
|
||||
else if (typeof msg.payload == "string") { // convert CSV string to object
|
||||
try {
|
||||
var f = true;
|
||||
var j = 0;
|
||||
var k = [""];
|
||||
var o = {};
|
||||
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
|
||||
var j = 0; // pointer into array of template items
|
||||
var k = [""]; // array of data for each of the template items
|
||||
var o = {}; // output object to build up
|
||||
var a = []; // output array is needed for multiline option
|
||||
var first = true; // is this the first line
|
||||
var tmp = "";
|
||||
|
||||
// For now we are just going to assume that any r or /n means an end of line...
|
||||
// got to be a weird csv that has singleton \r \n in it for another reason...
|
||||
|
||||
// Now process the whole file/line
|
||||
for (var i = 0; i < msg.payload.length; i++) {
|
||||
if (msg.payload[i] === node.quo) {
|
||||
f = !f;
|
||||
if (msg.payload[i-1] === node.quo) { k[j] += '\"'; }
|
||||
}
|
||||
else if ((msg.payload[i] === node.sep) && f) {
|
||||
if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
|
||||
j += 1;
|
||||
k[j] = "";
|
||||
if ((node.hdrin === true) && first) { // if the template is in the first line
|
||||
if ((msg.payload[i] === "\n")||(msg.payload[i] === "\r")) { // look for first line break
|
||||
node.template = clean(tmp.split(node.sep));
|
||||
first = false;
|
||||
}
|
||||
else { tmp += msg.payload[i]; }
|
||||
}
|
||||
else {
|
||||
k[j] += msg.payload[i];
|
||||
if (msg.payload[i] === node.quo) { // if it's a quote toggle inside or outside
|
||||
f = !f;
|
||||
if (msg.payload[i-1] === node.quo) { k[j] += '\"'; } // if it's a quotequote then it's actually a quote
|
||||
}
|
||||
else if ((msg.payload[i] === node.sep) && f) { // if we are outside of quote (ie valid separator
|
||||
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
|
||||
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) {
|
||||
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
|
||||
o[node.template[j]] = k[j];
|
||||
}
|
||||
j += 1;
|
||||
k[j] = "";
|
||||
}
|
||||
else if (f && ((msg.payload[i] === "\n") || (msg.payload[i] === "\r"))) { // handle multiple lines
|
||||
//console.log(j,k,o,k[j]);
|
||||
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
|
||||
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
|
||||
o[node.template[j]] = k[j].replace(/\r$/,'');
|
||||
}
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
if (node.multi === "one") { node.send({payload:o}); } // either send
|
||||
else { a.push(o); } // or add to the array
|
||||
}
|
||||
j = 0;
|
||||
k = [""];
|
||||
o = {};
|
||||
}
|
||||
else { // just add to the part of the message
|
||||
k[j] += msg.payload[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
|
||||
// Finished so finalize and send anything left
|
||||
//console.log(j,k,o,k[j]);
|
||||
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
|
||||
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
|
||||
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
|
||||
o[node.template[j]] = k[j].replace(/\r$/,'');
|
||||
}
|
||||
msg.payload = o;
|
||||
node.send(msg);
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
if (node.multi === "one") { node.send({payload:o}); } // either send
|
||||
else { a.push(o); } // or add to the aray
|
||||
}
|
||||
if (node.multi !== "one") { node.send({payload:a}); } // finally send the array
|
||||
}
|
||||
catch(e) { node.log(e); }
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user