Major Update to CSV node.

now handles lines, files, column names in first row, etc etc

nodes/core/parsers/70-CSV.html

@@ -1,3 +1,4 @@
+
 <!--
   Copyright 2014 IBM Corp.
 
@@ -17,29 +18,60 @@
 <script type="text/x-red" data-template-name="csv">
     <div class="form-row">
         <label for="node-input-temp"><i class="fa fa-list"></i> Columns</label>
-        <input type="text" id="node-input-temp" placeholder="A,B,C,D">
+        <input type="text" id="node-input-temp" placeholder="comma-separated column names">
     </div>
     <div class="form-row">
-        <label for="node-input-sep"><i class="fa fa-chevron-right"></i> Separator</label>
+        <label for="node-input-select-sep"><i class="fa fa-text-width"></i> Separator</label>
-        <input type="text" id="node-input-sep" placeholder=','>
+            <select style="width: 150px" id="node-input-select-sep">
+                <option value=",">comma</option>
+                <option value="\t">tab</option>
+                <option value=" ">space</option>
+                <option value=";">semicolon</option>
+                <option value=":">colon</option>
+                <option value="">other...</option>
+           </select>
+           <input style="width: 40px;" type="text" id="node-input-sep" pattern=".">
     </div>
-    <!-- <div class="form-row">
+
-        <label for="node-input-quo"><i class="fa fa-tag"></i> Escape</label>
-        <input type="text" id="node-input-quo" placeholder='"'>
-    </div> -->
-    <br/>
     <div class="form-row">
         <label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
         <input type="text" id="node-input-name" placeholder="Name">
     </div>
-    <div id="node-input-tip" class="form-tips">Tip: you can use "\t" for tab separator.</div>
+    <hr align="middle"/>
+    <div class="form-row">
+        <label style="width: 100%;"><i class="fa fa-gears"></i> CSV-to-Object options</label>
+        <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Input</label>
+        <input style="width: 30px" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin">first row contains column names</span>
+    </div>
+    <div class="form-row">
+        <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-out"></i> Output</label>
+        <select type="text" id="node-input-multi" style="width: 250px;">
+            <option value="one">a message per row</option>
+            <option value="mult">a single message [array]</option>
+        </select>
+    </div>
+    <hr align="middle"/>
+    <div class="form-row">
+        <label style="width: 100%;"><i class="fa fa-gears"></i> Object-to-CSV options</label>
+        <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Output</label>
+        <input style="width: 30px" type="checkbox" id="node-input-hdrout"><label style="width: auto;" for="node-input-hdrout">include column name row</span>
+    </div>
+    <div class="form-row">
+        <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-align-left"></i> Newline</label>
+        <select style="width: 150px" id="node-input-ret">
+            <option value='\n'>Linux (\n)</option>
+            <option value='\r'>Mac (\r)</option>
+            <option value='\r\n'>Windows (\r\n)</option>
+       </select>
+    </div>
 </script>
 
 <script type="text/x-red" data-help-name="csv">
-    <p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object. Places the result in the payload.</p>
+    <p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object.
+    Places the result in the payload.</p>
     <p>If the input is a string it tries to parse it as CSV and creates a javascript object.</p>
     <p>If the input is a javascript object it tries to build a CSV string.</p>
-    <p>A columns template MUST be specified that contains the ordered list of column headers. For csv input these become the property names.
+    <p>The columns template should contain an ordered list of column headers. For csv input these become the property names.
     For csv output these specify the properties to extract from the object and the order for the csv.</p>
     <p><b>Note:</b> the columns should always be specified comma separated - even if another separator is chosen for the data.</p>
     </script>
@@ -50,9 +82,13 @@
         color:"#DEBD5C",
         defaults: {
             name: {value:""},
-            sep: {value:',',required:true},
+            sep: {value:',',required:true,validate:RED.validators.regex(/^.{1,2}$/)},
             //quo: {value:'"',required:true},
-            temp: {value:"",required:true}
+            hdrin: {value:""},
+            hdrout: {value:""},
+            multi: {value:"one",required:true},
+            ret: {value:'\\n'},
+            temp: {value:""}
         },
         inputs:1,
         outputs:1,
@@ -62,6 +98,26 @@
         },
         labelStyle: function() {
             return this.name?"node_label_italic":"";
+        },
+        oneditprepare: function() {
+            if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " ") {
+                $("#node-input-select-sep").val(this.sep);
+                $("#node-input-sep").hide();
+            } else {
+                $("#node-input-select-sep").val("");
+                $("#node-input-sep").val(this.sep);
+                $("#node-input-sep").show();
+            }
+            $("#node-input-select-sep").change(function() {
+                var v = $("#node-input-select-sep option:selected").val();
+                $("#node-input-sep").val(v);
+                if (v == "") {
+                    $("#node-input-sep").val("");
+                    $("#node-input-sep").show().focus();
+                } else {
+                    $("#node-input-sep").hide();
+                }
+            });
         }
     });
 </script>

nodes/core/parsers/70-CSV.js

@@ -19,64 +19,131 @@ module.exports = function(RED) {
     function CSVNode(n) {
         RED.nodes.createNode(this,n);
         this.template = n.temp.split(",");
-        this.sep = n.sep || ',';
+        this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
-        this.sep = this.sep.replace("\\n","\n").replace("\\r","\r").replace("\\t","\t");
         this.quo = '"';
+        this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
+        this.winflag = (this.ret === "\r\n");
+        this.lineend = "\n";
+        this.multi = n.multi || "one";
+        this.hdrin = n.hdrin || false;
+        this.hdrout = n.hdrout || false;
+        this.goodtmpl = true;
         var node = this;
 
-        for (var t = 0; t < node.template.length; t++) {
+        // pass in an array of column names to be trimed, de-quoted and retrimed
-            node.template[t] = node.template[t].trim(); // remove leading and trailing whitespace
+        var clean = function(col) {
-            if (node.template[t].charAt(0) === '"' && node.template[t].charAt(node.template[t].length -1) === '"') {
+            for (var t = 0; t < col.length; t++) {
-                // remove leading and trialing quotes (if they exist) - and remove whitepace again.
+                col[t] = col[t].trim(); // remove leading and trailing whitespace
-                node.template[t] = node.template[t].substr(1,node.template[t].length -2).trim();
+                if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
+                    // remove leading and trailing quotes (if they exist) - and remove whitepace again.
+                    col[t] = col[t].substr(1,col[t].length -2).trim();
                 }
             }
+            if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
+            else { node.goodtmpl = true; }
+            return col;
+        }
+        node.template = clean(node.template);
 
         this.on("input", function(msg) {
             if (msg.hasOwnProperty("payload")) {
-                if (typeof msg.payload == "object") { // convert to csv
+                if (typeof msg.payload == "object") { // convert object to CSV string
                     try {
                         var ou = "";
-                        for (var t in node.template) {
+                        if (node.hdrout) {
-                            if (msg.payload.hasOwnProperty(node.template[t])) {
+                            ou += node.template.join(node.sep) + node.ret;
-                                if (msg.payload[node.template[t]].indexOf(node.sep) != -1) {
-                                    ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
                         }
-                                else if (msg.payload[node.template[t]].indexOf(node.quo) != -1) {
+                        if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
-                                    msg.payload[node.template[t]] = msg.payload[node.template[t]].replace(/"/g, '""');
+                        for (var s = 0; s < msg.payload.length; s++) {
-                                    ou += node.quo + msg.payload[node.template[t]] + node.quo + node.sep;
+                            for (var t=0; t < node.template.length; t++) {
+
+                                // aaargh - resorting to eval here - but fairly contained front and back.
+                                var p = RED.util.ensureString(eval("msg.payload[s]."+node.template[t]));
+
+                                if (p === "undefined") { p = ""; }
+                                if (p.indexOf(node.sep) != -1) { // add quotes if any "commas"
+                                    ou += node.quo + p + node.quo + node.sep;
                                 }
-                                else { ou += msg.payload[node.template[t]] + node.sep; }
+                                else if (p.indexOf(node.quo) != -1) { // add double quotes if any quotes
+                                    p = p.replace(/"/g, '""');
+                                    ou += node.quo + p + node.quo + node.sep;
                                 }
+                                else { ou += p + node.sep; } // otherwise just add
                             }
-                        msg.payload = ou.slice(0,-1);
+                            ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
-                        node.send(msg);
+                        }
+                        node.send({payload:ou});
                     }
                     catch(e) { node.log(e); }
                 }
-                else if (typeof msg.payload == "string") { // convert to object
+                else if (typeof msg.payload == "string") { // convert CSV string to object
                     try {
-                        var f = true;
+                        var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
-                        var j = 0;
+                        var j = 0; // pointer into array of template items
-                        var k = [""];
+                        var k = [""]; // array of data for each of the template items
-                        var o = {};
+                        var o = {}; // output object to build up
+                        var a = []; // output array is needed for multiline option
+                        var first = true; // is this the first line
+                        var tmp = "";
+
+                        // For now we are just going to assume that any r or /n means an end of line...
+                        //   got to be a weird csv that has singleton \r \n in it for another reason...
+
+                        // Now process the whole file/line
                         for (var i = 0; i < msg.payload.length; i++) {
-                            if (msg.payload[i] === node.quo) {
+                            if ((node.hdrin === true) && first) { // if the template is in the first line
-                                f = !f;
+                                if ((msg.payload[i] === "\n")||(msg.payload[i] === "\r")) { // look for first line break
-                                if (msg.payload[i-1] === node.quo) { k[j] += '\"'; }
+                                    node.template = clean(tmp.split(node.sep));
+                                    first = false;
+                                }
+                                else { tmp += msg.payload[i]; }
+                            }
+                            else {
+                                if (msg.payload[i] === node.quo) { // if it's a quote toggle inside or outside
+                                    f = !f;
+                                    if (msg.payload[i-1] === node.quo) { k[j] += '\"'; } // if it's a quotequote then it's actually a quote
+                                }
+                                else if ((msg.payload[i] === node.sep) && f) { // if we are outside of quote (ie valid separator
+                                    if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
+                                    if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) {
+                                        if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
+                                        o[node.template[j]] = k[j];
                                     }
-                            else if ((msg.payload[i] === node.sep) && f) {
-                                if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
                                     j += 1;
                                     k[j] = "";
                                 }
-                            else {
+                                else if (f && ((msg.payload[i] === "\n") || (msg.payload[i] === "\r"))) { // handle multiple lines
+                                    //console.log(j,k,o,k[j]);
+                                    if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
+                                        if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
+                                        o[node.template[j]] = k[j].replace(/\r$/,'');
+                                    }
+                                    if (JSON.stringify(o) !== "{}") { // don't send empty objects
+                                        if (node.multi === "one") { node.send({payload:o}); } // either send
+                                        else { a.push(o); } // or add to the array
+                                    }
+                                    j = 0;
+                                    k = [""];
+                                    o = {};
+                                }
+                                else { // just add to the part of the message
                                     k[j] += msg.payload[i];
                                 }
                             }
-                        if ( node.template[j] && (node.template[j] !== "") ) { o[node.template[j]] = k[j]; }
+                        }
+                        // Finished so finalize and send anything left
+                        //console.log(j,k,o,k[j]);
+                        if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
+                        if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
+                            if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
+                            o[node.template[j]] = k[j].replace(/\r$/,'');
+                        }
                         msg.payload = o;
-                        node.send(msg);
+                        if (JSON.stringify(o) !== "{}") { // don't send empty objects
+                            if (node.multi === "one") { node.send({payload:o}); } // either send
+                            else { a.push(o); } // or add to the aray
+                        }
+                        if (node.multi !== "one") { node.send({payload:a}); } // finally send the array
                     }
                     catch(e) { node.log(e); }
                 }