frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

PERF : make csv way faster by not allocating and handling huge string

This commit is contained in:
Franck 2023-09-18 15:16:15 +02:00
parent d9bbac20f3
commit 27ca30aa82
1 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -63,7 +63,7 @@ module.exports = function(RED) {
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) { if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template); template = clean(node.template);
} }
var ou = ""; const ou = [];
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
if (node.hdrout !== "none" && node.hdrSent === false) { if (node.hdrout !== "none" && node.hdrSent === false) {
if ((template.length === 1) && (template[0] === '')) { if ((template.length === 1) && (template[0] === '')) {
@ -74,7 +74,7 @@ module.exports = function(RED) {
template = Object.keys(msg.payload[0]); template = Object.keys(msg.payload[0]);
} }
} }
ou += template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret; ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
if (node.hdrout === "once") { node.hdrSent = true; } if (node.hdrout === "once") { node.hdrSent = true; }
} }
for (var s = 0; s < msg.payload.length; s++) { for (var s = 0; s < msg.payload.length; s++) {
@ -93,7 +93,7 @@ module.exports = function(RED) {
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo; msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
} }
} }
ou += msg.payload[s].join(node.sep) + node.ret; ou.push(msg.payload[s].join(node.sep));
} }
else { else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) { if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
@ -105,6 +105,7 @@ module.exports = function(RED) {
node.warn(RED._("csv.errors.obj_csv")); node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false; tmpwarn = false;
} }
const row = [];
for (var p in msg.payload[0]) { for (var p in msg.payload[0]) {
/* istanbul ignore else */ /* istanbul ignore else */
if (msg.payload[s].hasOwnProperty(p)) { if (msg.payload[s].hasOwnProperty(p)) {
@ -118,21 +119,22 @@ module.exports = function(RED) {
} }
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""'); q = q.replace(/"/g, '""');
ou += node.quo + q + node.quo + node.sep; row.push(node.quo + q + node.quo);
} }
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + q + node.quo + node.sep; row.push(node.quo + q + node.quo);
} }
else { ou += q + node.sep; } // otherwise just add else { row.push(q); } // otherwise just add
} }
} }
} }
ou = ou.slice(0,-1) + node.ret; ou.push(row.join(node.sep)); // add separator
} }
else { else {
const row = [];
for (var t=0; t < template.length; t++) { for (var t=0; t < template.length; t++) {
if (template[t] === '') { if (template[t] === '') {
ou += node.sep; row.push('');
} }
else { else {
var tt = template[t]; var tt = template[t];
@ -146,19 +148,20 @@ module.exports = function(RED) {
p = RED.util.ensureString(p); p = RED.util.ensureString(p);
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""'); p = p.replace(/"/g, '""');
ou += node.quo + p + node.quo + node.sep; row.push(node.quo + p + node.quo);
} }
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n" else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + p + node.quo + node.sep; row.push(node.quo + p + node.quo);
} }
else { ou += p + node.sep; } // otherwise just add else { row.push(p); } // otherwise just add
} }
} }
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline" ou.push(row.join(node.sep)); // add separator
} }
} }
} }
msg.payload = ou; // join lines, don't forget to add the last new line
msg.payload = ou.join(node.ret) + node.ret;
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); } if (msg.payload !== '') { send(msg); }
done(); done();