frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2025-03-01 10:36:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

update CSV to adhere to strict rfc compliance on msg.columns

Browse Source
This commit is contained in:
Steve-Mcl 2024-12-12 16:42:11 +00:00
parent 6af3c8c2a9
commit b139eb4a18
1 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -367,20 +367,21 @@ module.exports = function(RED) {
const sendHeadersAlways = node.hdrout === "all" const sendHeadersAlways = node.hdrout === "all"
const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways) const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
const quoteables = [node.sep, node.quo, "\n", "\r"] const quoteables = [node.sep, node.quo, "\n", "\r"]
const templateQuoteables = [node.sep, '"', "\n", "\r"] const templateQuoteables = [node.sep, node.quo, "\n", "\r"]
const templateQuoteablesStrict = [',', '"', "\n", "\r"]
let badTemplateWarnOnce = true let badTemplateWarnOnce = true
const columnStringToTemplateArray = function (col, sep) { const columnStringToTemplateArray = function (col, sep) {
// NOTE: enforce strict column template parsing in RFC4180 mode // NOTE: enforce strict column template parsing in RFC4180 mode
const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true }) const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false } if (parsed.data?.length === 1) { node.goodtmpl = true } else { node.goodtmpl = false }
return parsed.headers.length ? parsed.headers : null return node.goodtmpl ? parsed.data[0] : null
} }
const templateArrayToColumnString = function (template, keepEmptyColumns) { const templateArrayToColumnString = function (template, keepEmptyColumns, separator = ',', quotables = templateQuoteablesStrict) {
// NOTE: enforce strict column template parsing in RFC4180 mode // NOTE: defaults to strict column template parsing (commas and double quotes)
const parsed = csv.parse('', {headers: template, headersOnly:true, separator: node.sep, quote: node.quo, outputStyle: 'array', strict: true }) const parsed = csv.parse('', {headers: template, headersOnly:true, separator, quote: node.quo, outputStyle: 'array', strict: true })
return keepEmptyColumns return keepEmptyColumns
? parsed.headers.map(e => addQuotes(e || '', { separator: node.sep, quoteables: templateQuoteables})).join(node.sep) ? parsed.headers.map(e => addQuotes(e || '', { separator, quoteables: quotables })).join(separator)
: parsed.header // exclues empty columns : parsed.header // exclues empty columns
// TODO: resolve inconsistency between CSV->JSON and JSON->CSV // TODO: resolve inconsistency between CSV->JSON and JSON->CSV
// CSV->JSON: empty columns are excluded // CSV->JSON: empty columns are excluded
@ -441,13 +442,13 @@ module.exports = function(RED) {
if (sendHeaders && node.hdrSent === false) { if (sendHeaders && node.hdrSent === false) {
if (hasTemplate(template) === false) { if (hasTemplate(template) === false) {
if (msg.hasOwnProperty("columns")) { if (msg.hasOwnProperty("columns")) {
template = columnStringToTemplateArray(msg.columns || "", node.sep) || [''] template = columnStringToTemplateArray(msg.columns || "", ",") || ['']
} }
else { else {
template = Object.keys(inputData[0]) || [''] template = Object.keys(inputData[0]) || ['']
} }
} }
stringBuilder.push(templateArrayToColumnString(template, true)) stringBuilder.push(templateArrayToColumnString(template, true, node.sep, templateQuoteables)) // use user set separator for output data.
if (sendHeadersOnce) { node.hdrSent = true } if (sendHeadersOnce) { node.hdrSent = true }
} }
@ -475,7 +476,7 @@ module.exports = function(RED) {
} else { } else {
/*** row is an object ***/ /*** row is an object ***/
if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) { if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) {
template = columnStringToTemplateArray(msg.columns || "", node.sep) template = columnStringToTemplateArray(msg.columns || "", ",")
} }
if (hasTemplate(template) === false) { if (hasTemplate(template) === false) {
/*** row is an object but we still don't have a template ***/ /*** row is an object but we still don't have a template ***/
@ -483,6 +484,7 @@ module.exports = function(RED) {
node.warn(RED._("csv.errors.obj_csv")) node.warn(RED._("csv.errors.obj_csv"))
badTemplateWarnOnce = false badTemplateWarnOnce = false
} }
template = Object.keys(row) || ['']
const rowData = [] const rowData = []
for (let header in inputData[0]) { for (let header in inputData[0]) {
if (row.hasOwnProperty(header)) { if (row.hasOwnProperty(header)) {
@ -518,7 +520,7 @@ module.exports = function(RED) {
// join lines, don't forget to add the last new line // join lines, don't forget to add the last new line
msg.payload = stringBuilder.join(node.ret) + node.ret msg.payload = stringBuilder.join(node.ret) + node.ret
msg.columns = templateArrayToColumnString(template) msg.columns = templateArrayToColumnString(template) // always strict commas + double quotes for
if (msg.payload !== '') { send(msg) } if (msg.payload !== '') { send(msg) }
done() done()
} }
@ -615,16 +617,15 @@ module.exports = function(RED) {
} }
if (msg.parts.index + 1 === msg.parts.count) { if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store msg.payload = node.store
msg.columns = csvParseResult.header // msg.columns = csvParseResult.header
// msg._mode = 'RFC4180 mode' msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
delete msg.parts delete msg.parts
send(msg) send(msg)
node.store = [] node.store = []
} }
} }
else { else {
msg.columns = csvParseResult.header msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
// msg._mode = 'RFC4180 mode'
msg.payload = data msg.payload = data
send(msg); // finally send the array send(msg); // finally send the array
} }
@ -633,7 +634,8 @@ module.exports = function(RED) {
const len = data.length const len = data.length
for (let row = 0; row < len; row++) { for (let row = 0; row < len; row++) {
const newMessage = RED.util.cloneMessage(msg) const newMessage = RED.util.cloneMessage(msg)
newMessage.columns = csvParseResult.header // newMessage.columns = csvParseResult.header
newMessage.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
newMessage.payload = data[row] newMessage.payload = data[row]
if (!has_parts) { if (!has_parts) {
newMessage.parts = { newMessage.parts = {