frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Major Update to CSV node. 

now handles lines, files, column names in first row, etc etc
This commit is contained in:
Dave C-J 2014-09-12 16:50:01 +01:00
parent 429a87f88a
commit 7e2dbb13e4
2 changed files with 173 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
nodes/core/parsers/70-CSV.html
View File

@ -1,3 +1,4 @@
<!--
Copyright 2014 IBM Corp.
@ -17,29 +18,60 @@
<script type="text/x-red" data-template-name="csv">
<div class="form-row">
<label for="node-input-temp"><i class="fa fa-list"></i> Columns</label>
<input type="text" id="node-input-temp" placeholder="A,B,C,D">
<input type="text" id="node-input-temp" placeholder="comma-separated column names">
</div>
<div class="form-row">
<label for="node-input-sep"><i class="fa fa-chevron-right"></i> Separator</label>
<input type="text" id="node-input-sep" placeholder=','>
<label for="node-input-select-sep"><i class="fa fa-text-width"></i> Separator</label>
<select style="width: 150px" id="node-input-select-sep">
<option value=",">comma</option>
<option value="\t">tab</option>
<option value=" ">space</option>
<option value=";">semicolon</option>
<option value=":">colon</option>
<option value="">other...</option>
</select>
<input style="width: 40px;" type="text" id="node-input-sep" pattern=".">
</div>
<!-- <div class="form-row">
<label for="node-input-quo"><i class="fa fa-tag"></i> Escape</label>
<input type="text" id="node-input-quo" placeholder='"'>
</div> -->
<br/>
<div class="form-row">
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
<input type="text" id="node-input-name" placeholder="Name">
</div>
<div id="node-input-tip" class="form-tips">Tip: you can use "\t" for tab separator.</div>
<hr align="middle"/>
<div class="form-row">
<label style="width: 100%;"><i class="fa fa-gears"></i> CSV-to-Object options</label>
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Input</label>
<input style="width: 30px" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin">first row contains column names</span>
</div>
<div class="form-row">
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-out"></i> Output</label>
<select type="text" id="node-input-multi" style="width: 250px;">
<option value="one">a message per row</option>
<option value="mult">a single message [array]</option>
</select>
</div>
<hr align="middle"/>
<div class="form-row">
<label style="width: 100%;"><i class="fa fa-gears"></i> Object-to-CSV options</label>
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Output</label>
<input style="width: 30px" type="checkbox" id="node-input-hdrout"><label style="width: auto;" for="node-input-hdrout">include column name row</span>
</div>
<div class="form-row">
<label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-align-left"></i> Newline</label>
<select style="width: 150px" id="node-input-ret">
<option value='\n'>Linux (\n)</option>
<option value='\r'>Mac (\r)</option>
<option value='\r\n'>Windows (\r\n)</option>
</select>
</div>
</script>
<script type="text/x-red" data-help-name="csv">
<p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object. Places the result in the payload.</p>
<p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object.
Places the result in the payload.</p>
<p>If the input is a string it tries to parse it as CSV and creates a javascript object.</p>
<p>If the input is a javascript object it tries to build a CSV string.</p>
<p>A columns template MUST be specified that contains the ordered list of column headers. For csv input these become the property names.
<p>The columns template should contain an ordered list of column headers. For csv input these become the property names.
For csv output these specify the properties to extract from the object and the order for the csv.</p>
<p><b>Note:</b> the columns should always be specified comma separated - even if another separator is chosen for the data.</p>
</script>
@ -50,9 +82,13 @@
color:"#DEBD5C",
defaults: {
name: {value:""},
sep: {value:',',required:true},
sep: {value:',',required:true,validate:RED.validators.regex(/^.{1,2}$/)},
//quo: {value:'"',required:true},
temp: {value:"",required:true}
hdrin: {value:""},
hdrout: {value:""},
multi: {value:"one",required:true},
ret: {value:'\\n'},
temp: {value:""}
},
inputs:1,
outputs:1,
@ -62,6 +98,26 @@
},
labelStyle: function() {
return this.name?"node_label_italic":"";
},
oneditprepare: function() {
if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " ") {
$("#node-input-select-sep").val(this.sep);
$("#node-input-sep").hide();
} else {
$("#node-input-select-sep").val("");
$("#node-input-sep").val(this.sep);
$("#node-input-sep").show();
}
$("#node-input-select-sep").change(function() {
var v = $("#node-input-select-sep option:selected").val();
$("#node-input-sep").val(v);
if (v == "") {
$("#node-input-sep").val("");
$("#node-input-sep").show().focus();
} else {
$("#node-input-sep").hide();
}
});
}
});
</script>

141
nodes/core/parsers/70-CSV.js
View File

@ -19,64 +19,131 @@ module.exports = function(RED) {
function CSVNode(n) {
RED.nodes.createNode(this,n);
this.template = n.temp.split(",");
this.sep = n.sep || ',';
this.sep = this.sep.replace("\\n","\n").replace("\\r","\r").replace("\\t","\t");
this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
this.quo = '"';
this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
this.winflag = (this.ret === "\r\n");
this.lineend = "\n";
this.multi = n.multi || "one";
this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || false;
this.goodtmpl = true;
var node = this;
for (var t = 0; t < node.template.length; t++) {
node.template[t] = node.template[t].trim(); // remove leading and trailing whitespace
if (node.template[t].charAt(0) === '"' && node.template[t].charAt(node.template[t].length -1) === '"') {
// remove leading and trialing quotes (if they exist) - and remove whitepace again.
node.template[t] = node.template[t].substr(1,node.template[t].length -2).trim();
// pass in an array of column names to be trimed, de-quoted and retrimed
var clean = function(col) {
for (var t = 0; t < col.length; t++) {
col[t] = col[t].trim(); // remove leading and trailing whitespace
if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
// remove leading and trailing quotes (if they exist) - and remove whitepace again.
col[t] = col[t].substr(1,col[t].length -2).trim();
}
}
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; }
return col;
}
node.template = clean(node.template);
this.on("input", function(msg) {
if (msg.hasOwnProperty("payload")) {
if (typeof msg.payload == "object") { // convert to csv
if (typeof msg.payload == "object") { // convert object to CSV string
try {
var ou = "";
for (var t in node.template) {
if (msg.payload.hasOwnProperty(node.template[t])) {
if (msg.payload[node.template[t]].indexOf(node.sep) != -1) {
ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
}
else if (msg.payload[node.template[t]].indexOf(node.quo) != -1) {
msg.payload[node.template[t]] = msg.payload[node.template[t]].replace(/"/g, '""');
ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
}
else { ou += msg.payload[node.template[t]] + node.sep; }
}
if (node.hdrout) {
ou += node.template.join(node.sep) + node.ret;
}
msg.payload = ou.slice(0,-1);
node.send(msg);
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
for (var s = 0; s < msg.payload.length; s++) {
for (var t=0; t < node.template.length; t++) {
// aaargh - resorting to eval here - but fairly contained front and back.
var p = RED.util.ensureString(eval("msg.payload[s]."+node.template[t]));
if (p === "undefined") { p = ""; }
if (p.indexOf(node.sep) != -1) { // add quotes if any "commas"
ou += node.quo + p + node.quo + node.sep;
}
else if (p.indexOf(node.quo) != -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""');
ou += node.quo + p + node.quo + node.sep;
}
else { ou += p + node.sep; } // otherwise just add
}
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
}
node.send({payload:ou});
}
catch(e) { node.log(e); }
}
else if (typeof msg.payload == "string") { // convert to object
else if (typeof msg.payload == "string") { // convert CSV string to object
try {
var f = true;
var j = 0;
var k = [""];
var o = {};
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
var j = 0; // pointer into array of template items
var k = [""]; // array of data for each of the template items
var o = {}; // output object to build up
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var tmp = "";
// For now we are just going to assume that any r or /n means an end of line...
// got to be a weird csv that has singleton \r \n in it for another reason...
// Now process the whole file/line
for (var i = 0; i < msg.payload.length; i++) {
if (msg.payload[i] === node.quo) {
f = !f;
if (msg.payload[i-1] === node.quo) { k[j] += '\"'; }
}
else if ((msg.payload[i] === node.sep) && f) {
if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
j += 1;
k[j] = "";
if ((node.hdrin === true) && first) { // if the template is in the first line
if ((msg.payload[i] === "\n")||(msg.payload[i] === "\r")) { // look for first line break
node.template = clean(tmp.split(node.sep));
first = false;
}
else { tmp += msg.payload[i]; }
}
else {
k[j] += msg.payload[i];
if (msg.payload[i] === node.quo) { // if it's a quote toggle inside or outside
f = !f;
if (msg.payload[i-1] === node.quo) { k[j] += '\"'; } // if it's a quotequote then it's actually a quote
}
else if ((msg.payload[i] === node.sep) && f) { // if we are outside of quote (ie valid separator
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) {
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
o[node.template[j]] = k[j];
}
j += 1;
k[j] = "";
}
else if (f && ((msg.payload[i] === "\n") || (msg.payload[i] === "\r"))) { // handle multiple lines
//console.log(j,k,o,k[j]);
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
o[node.template[j]] = k[j].replace(/\r$/,'');
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
if (node.multi === "one") { node.send({payload:o}); } // either send
else { a.push(o); } // or add to the array
}
j = 0;
k = [""];
o = {};
}
else { // just add to the part of the message
k[j] += msg.payload[i];
}
}
}
if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
// Finished so finalize and send anything left
//console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
o[node.template[j]] = k[j].replace(/\r$/,'');
}
msg.payload = o;
node.send(msg);
if (JSON.stringify(o) !== "{}") { // don't send empty objects
if (node.multi === "one") { node.send({payload:o}); } // either send
else { a.push(o); } // or add to the aray
}
if (node.multi !== "one") { node.send({payload:a}); } // finally send the array
}
catch(e) { node.log(e); }
}