frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Merge pull request #4349 from braincube-io/feat/fmo/makeCsvExportFast

Browse Source 
Performance : make CSV export way faster by not re-allocating and handling huge string
This commit is contained in:
Nick O'Leary 2023-09-19 09:22:54 +01:00 committed by GitHub
commit fc54848318
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -63,7 +63,7 @@ module.exports = function(RED) {
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
template = clean(node.template);
}
var ou = "";
const ou = [];
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
if (node.hdrout !== "none" && node.hdrSent === false) {
if ((template.length === 1) && (template[0] === '')) {
@ -74,7 +74,7 @@ module.exports = function(RED) {
template = Object.keys(msg.payload[0]);
}
}
ou += template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
if (node.hdrout === "once") { node.hdrSent = true; }
}
for (var s = 0; s < msg.payload.length; s++) {
@ -93,7 +93,7 @@ module.exports = function(RED) {
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
}
}
ou += msg.payload[s].join(node.sep) + node.ret;
ou.push(msg.payload[s].join(node.sep));
}
else {
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
@ -105,6 +105,7 @@ module.exports = function(RED) {
node.warn(RED._("csv.errors.obj_csv"));
tmpwarn = false;
}
const row = [];
for (var p in msg.payload[0]) {
/* istanbul ignore else */
if (msg.payload[s].hasOwnProperty(p)) {
@ -118,21 +119,22 @@ module.exports = function(RED) {
}
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
q = q.replace(/"/g, '""');
ou += node.quo + q + node.quo + node.sep;
row.push(node.quo + q + node.quo);
}
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + q + node.quo + node.sep;
row.push(node.quo + q + node.quo);
}
else { ou += q + node.sep; } // otherwise just add
else { row.push(q); } // otherwise just add
}
}
}
ou = ou.slice(0,-1) + node.ret;
ou.push(row.join(node.sep)); // add separator
}
else {
const row = [];
for (var t=0; t < template.length; t++) {
if (template[t] === '') {
ou += node.sep;
row.push('');
}
else {
var tt = template[t];
@ -146,19 +148,20 @@ module.exports = function(RED) {
p = RED.util.ensureString(p);
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
p = p.replace(/"/g, '""');
ou += node.quo + p + node.quo + node.sep;
row.push(node.quo + p + node.quo);
}
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
ou += node.quo + p + node.quo + node.sep;
row.push(node.quo + p + node.quo);
}
else { ou += p + node.sep; } // otherwise just add
else { row.push(p); } // otherwise just add
}
}
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
ou.push(row.join(node.sep)); // add separator
}
}
}
msg.payload = ou;
// join lines, don't forget to add the last new line
msg.payload = ou.join(node.ret) + node.ret;
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
if (msg.payload !== '') { send(msg); }
done();