1
0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
node-red/nodes/core/parsers/70-CSV.js
Dave Conway-Jones 7c0b9ffe06 Add skip first n lines capability to csv node (#1535)
* Initial implementation of skip first lines for css node

* add css skip lines tests
2018-01-11 22:02:58 +00:00

265 lines
15 KiB
JavaScript

/**
* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
module.exports = function(RED) {
"use strict";
function CSVNode(n) {
RED.nodes.createNode(this,n);
this.template = (n.temp || "").split(",");
this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
this.quo = '"';
this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
this.winflag = (this.ret === "\r\n");
this.lineend = "\n";
this.multi = n.multi || "one";
this.hdrin = n.hdrin || false;
this.hdrout = n.hdrout || false;
this.goodtmpl = true;
this.skip = parseInt(n.skip || 0);
this.store = [];
var tmpwarn = true;
var node = this;
// pass in an array of column names to be trimed, de-quoted and retrimed
var clean = function(col) {
for (var t = 0; t < col.length; t++) {
col[t] = col[t].trim(); // remove leading and trailing whitespace
if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
// remove leading and trailing quotes (if they exist) - and remove whitepace again.
col[t] = col[t].substr(1,col[t].length -2).trim();
}
}
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
else { node.goodtmpl = true; }
return col;
}
node.template = clean(node.template);
this.on("input", function(msg) {
if (msg.hasOwnProperty("payload")) {
if (typeof msg.payload == "object") { // convert object to CSV string
try {
var ou = "";
if (node.hdrout) {
ou += node.template.join(node.sep) + node.ret;
}
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
for (var s = 0; s < msg.payload.length; s++) {
if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
for (var t = 0; t < msg.payload[s].length; t++) {
if (!msg.payload[s][t] && (msg.payload[s][t] !== 0)) { msg.payload[s][t] = ""; }
if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
}
ou += msg.payload[s].join(node.sep) + node.ret;
}
else {
if ((node.template.length === 1) && (node.template[0] === '')) {
/* istanbul ignore else */
if (tmpwarn === true) { // just warn about missing template once
node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false;
}
ou = "";
for (var p in msg.payload[0]) {
/* istanbul ignore else */
if (msg.payload[0].hasOwnProperty(p)) {
/* istanbul ignore else */
if (typeof msg.payload[0][p] !== "object") {
var q = "" + msg.payload[0][p];
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""');
ou += node.quo + q + node.quo + node.sep;
}
else if (q.indexOf(node.sep) !== -1) { // add quotes if any "commas"
ou += node.quo + q + node.quo + node.sep;
}
else { ou += q + node.sep; } // otherwise just add
}
}
}
ou = ou.slice(0,-1) + node.ret;
}
else {
for (var t=0; t < node.template.length; t++) {
if (node.template[t] === '') {
ou += node.sep;
}
else {
var p = RED.util.ensureString(RED.util.getMessageProperty(msg,"payload["+s+"]['"+node.template[t]+"']"));
/* istanbul ignore else */
if (p === "undefined") { p = ""; }
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""');
ou += node.quo + p + node.quo + node.sep;
}
else if (p.indexOf(node.sep) !== -1) { // add quotes if any "commas"
ou += node.quo + p + node.quo + node.sep;
}
else { ou += p + node.sep; } // otherwise just add
}
}
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
}
}
}
msg.payload = ou;
if (msg.payload !== '') { node.send(msg); }
}
catch(e) { node.error(e,msg); }
}
else if (typeof msg.payload == "string") { // convert CSV string to object
try {
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
var j = 0; // pointer into array of template items
var k = [""]; // array of data for each of the template items
var o = {}; // output object to build up
var a = []; // output array is needed for multiline option
var first = true; // is this the first line
var line = msg.payload;
var linecount = 0;
var tmp = "";
var reg = /^[-]?[0-9]*\.?[0-9]+$/;
if (msg.hasOwnProperty("parts")) {
linecount = msg.parts.index;
if (msg.parts.index > node.skip) { first = false; }
}
// For now we are just going to assume that any \r or \n means an end of line...
// got to be a weird csv that has singleton \r \n in it for another reason...
// Now process the whole file/line
for (var i = 0; i < line.length; i++) {
if (first && (linecount < node.skip)) {
if (line[i] === "\n") { linecount += 1; }
continue;
}
if ((node.hdrin === true) && first) { // if the template is in the first line
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
if (line.length - i === 1) { tmp += line[i]; }
node.template = clean(tmp.split(node.sep));
first = false;
}
else { tmp += line[i]; }
}
else {
if (line[i] === node.quo) { // if it's a quote toggle inside or outside
f = !f;
if (line[i-1] === node.quo) {
if (f === false) { k[j] += '\"'; }
} // if it's a quotequote then it's actually a quote
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
}
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) {
if ( reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
o[node.template[j]] = k[j];
}
j += 1;
k[j] = "";
}
else if ((line[i] === "\n") || (line[i] === "\r")) { // handle multiple lines
//console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
if ( reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
else { k[j].replace(/\r$/,''); }
o[node.template[j]] = k[j];
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array
}
j = 0;
k = [""];
o = {};
f = true; // reset in/out flag ready for next line.
}
else { // just add to the part of the message
k[j] += line[i];
}
}
}
// Finished so finalize and send anything left
//console.log(j,k,o,k[j]);
if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
if ( reg.test(k[j]) ) { k[j] = parseFloat(k[j]); }
else { k[j].replace(/\r$/,''); }
o[node.template[j]] = k[j];
}
if (JSON.stringify(o) !== "{}") { // don't send empty objects
a.push(o); // add to the array
}
var has_parts = msg.hasOwnProperty("parts");
if (node.multi !== "one") {
msg.payload = a;
if (has_parts) {
if (JSON.stringify(o) !== "{}") {
node.store.push(o);
}
if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store;
delete msg.parts;
node.send(msg);
node.store = [];
}
}
else {
node.send(msg); // finally send the array
}
}
else {
var len = a.length;
for (var i = 0; i < len; i++) {
var newMessage = RED.util.cloneMessage(msg);
newMessage.payload = a[i];
if (!has_parts) {
newMessage.parts = {
id: msg._msgid,
index: i,
count: len
};
}
else {
newMessage.parts.index -= node.skip;
newMessage.parts.count -= node.skip;
if (node.hdrin) { // if we removed the header line then shift the counts by 1
newMessage.parts.index -= 1;
newMessage.parts.count -= 1;
}
}
node.send(newMessage);
}
}
node.linecount = 0;
}
catch(e) { node.error(e,msg); }
}
else { node.warn(RED._("csv.errors.csv_js")); }
}
else { node.send(msg); } // If no payload - just pass it on.
});
}
RED.nodes.registerType("csv",CSVNode);
}