frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

PERF : make csv way faster by not allocating and handling huge string

This commit is contained in:
Franck 2023-09-18 15:16:15 +02:00
parent d9bbac20f3
commit 27ca30aa82
1 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -63,7 +63,7 @@ module.exports = function(RED) {
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template);
}
var ou = "";
const ou = [];
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
if (node.hdrout !== "none" && node.hdrSent === false) {
if ((template.length === 1) && (template[0] === '')) {
@ -74,7 +74,7 @@ module.exports = function(RED) {
template = Object.keys(msg.payload[0]);
}
}
ou += template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
if (node.hdrout === "once") { node.hdrSent = true; }
}
for (var s = 0; s < msg.payload.length; s++) {
@ -93,7 +93,7 @@ module.exports = function(RED) {
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
}
ou += msg.payload[s].join(node.sep) + node.ret;
ou.push(msg.payload[s].join(node.sep));
}
else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
@ -105,6 +105,7 @@ module.exports = function(RED) {
node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false;
}
const row = [];
for (var p in msg.payload[0]) {
/* istanbul ignore else */
if (msg.payload[s].hasOwnProperty(p)) {
@ -118,21 +119,22 @@ module.exports = function(RED) {
}
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""');
ou += node.quo + q + node.quo + node.sep;
row.push(node.quo + q + node.quo);
}
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + q + node.quo + node.sep;
row.push(node.quo + q + node.quo);
}
else { ou += q + node.sep; } // otherwise just add
else { row.push(q); } // otherwise just add
}
}
}
ou = ou.slice(0,-1) + node.ret;
ou.push(row.join(node.sep)); // add separator
}
else {
const row = [];
for (var t=0; t < template.length; t++) {
if (template[t] === '') {
ou += node.sep;
row.push('');
}
else {
var tt = template[t];
@ -146,19 +148,20 @@ module.exports = function(RED) {
p = RED.util.ensureString(p);
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""');
ou += node.quo + p + node.quo + node.sep;
row.push(node.quo + p + node.quo);
}
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + p + node.quo + node.sep;
row.push(node.quo + p + node.quo);
}
else { ou += p + node.sep; } // otherwise just add
else { row.push(p); } // otherwise just add
}
}
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
ou.push(row.join(node.sep)); // add separator
}
}
}
msg.payload = ou;
// join lines, don't forget to add the last new line
msg.payload = ou.join(node.ret) + node.ret;
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); }
done();