mirror of
https://github.com/node-red/node-red.git
synced 2025-03-01 10:36:34 +00:00
Merge branch 'dev' into make-split/join-more-flexible
This commit is contained in:
Nick O'Leary
GitHub
@@ -320,7 +320,7 @@
|
||||
}
|
||||
// but replace with repeat one if set to repeat
|
||||
if ((this.repeat && this.repeat != 0) || this.crontab) {
|
||||
suffix = " ↻";
|
||||
suffix = "\t↻";
|
||||
}
|
||||
if (this.name) {
|
||||
return this.name+suffix;
|
||||
|
||||
@@ -109,9 +109,8 @@ module.exports = function(RED) {
|
||||
}
|
||||
const p = props.shift()
|
||||
const property = p.p;
|
||||
const value = p.v ? p.v : '';
|
||||
const valueType = p.vt ? p.vt : 'str';
|
||||
|
||||
const value = p.v !== undefined ? p.v : '';
|
||||
const valueType = p.vt !== undefined ? p.vt : 'str';
|
||||
if (property) {
|
||||
if (valueType === "jsonata") {
|
||||
if (p.v) {
|
||||
|
||||
@@ -86,7 +86,7 @@
|
||||
},
|
||||
label: function() {
|
||||
var suffix = "";
|
||||
if (this.console === true || this.console === "true") { suffix = " ⇲"; }
|
||||
if (this.console === true || this.console === "true") { suffix = "\t⇲"; }
|
||||
if (this.targetType === "jsonata") {
|
||||
return (this.name || "JSONata") + suffix;
|
||||
}
|
||||
@@ -195,6 +195,119 @@
|
||||
node.dirty = true;
|
||||
});
|
||||
RED.view.redraw();
|
||||
},
|
||||
requestDebugNodeList: function(filteredNodes) {
|
||||
var workspaceOrder = RED.nodes.getWorkspaceOrder();
|
||||
var workspaceOrderMap = {};
|
||||
workspaceOrder.forEach(function(ws,i) {
|
||||
workspaceOrderMap[ws] = i;
|
||||
});
|
||||
|
||||
var candidateNodes = [];
|
||||
var candidateSFs = [];
|
||||
var subflows = {};
|
||||
RED.nodes.eachNode(function (n) {
|
||||
var nt = n.type;
|
||||
if (nt === "debug") {
|
||||
if (n.z in workspaceOrderMap) {
|
||||
candidateNodes.push(n);
|
||||
}
|
||||
else {
|
||||
var sf = RED.nodes.subflow(n.z);
|
||||
if (sf) {
|
||||
subflows[sf.id] = {
|
||||
debug: true,
|
||||
subflows: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(nt.substring(0, 8) === "subflow:") {
|
||||
if (n.z in workspaceOrderMap) {
|
||||
candidateSFs.push(n);
|
||||
}
|
||||
else {
|
||||
var psf = RED.nodes.subflow(n.z);
|
||||
if (psf) {
|
||||
var sid = nt.substring(8);
|
||||
var item = subflows[psf.id];
|
||||
if (!item) {
|
||||
item = {
|
||||
debug: undefined,
|
||||
subflows: {}
|
||||
};
|
||||
subflows[psf.id] = item;
|
||||
}
|
||||
item.subflows[sid] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
candidateSFs.forEach(function (sf) {
|
||||
var sid = sf.type.substring(8);
|
||||
if (containsDebug(sid, subflows)) {
|
||||
candidateNodes.push(sf);
|
||||
}
|
||||
});
|
||||
|
||||
candidateNodes.sort(function(A,B) {
|
||||
var wsA = workspaceOrderMap[A.z];
|
||||
var wsB = workspaceOrderMap[B.z];
|
||||
if (wsA !== wsB) {
|
||||
return wsA-wsB;
|
||||
}
|
||||
var labelA = RED.utils.getNodeLabel(A,A.id);
|
||||
var labelB = RED.utils.getNodeLabel(B,B.id);
|
||||
return labelA.localeCompare(labelB);
|
||||
});
|
||||
var currentWs = null;
|
||||
var data = [];
|
||||
var currentFlow;
|
||||
var currentSelectedCount = 0;
|
||||
candidateNodes.forEach(function(node) {
|
||||
if (currentWs !== node.z) {
|
||||
if (currentFlow && currentFlow.checkbox) {
|
||||
currentFlow.selected = currentSelectedCount === currentFlow.children.length
|
||||
}
|
||||
currentSelectedCount = 0;
|
||||
currentWs = node.z;
|
||||
var parent = RED.nodes.workspace(currentWs) || RED.nodes.subflow(currentWs);
|
||||
currentFlow = {
|
||||
label: RED.utils.getNodeLabel(parent, currentWs),
|
||||
}
|
||||
if (!parent.disabled) {
|
||||
currentFlow.children = [];
|
||||
currentFlow.checkbox = true;
|
||||
} else {
|
||||
currentFlow.class = "disabled"
|
||||
}
|
||||
data.push(currentFlow);
|
||||
}
|
||||
if (currentFlow.children) {
|
||||
if (!filteredNodes[node.id]) {
|
||||
currentSelectedCount++;
|
||||
}
|
||||
currentFlow.children.push({
|
||||
label: RED.utils.getNodeLabel(node,node.id),
|
||||
node: {
|
||||
id: node.id
|
||||
},
|
||||
checkbox: true,
|
||||
selected: !filteredNodes[node.id]
|
||||
});
|
||||
}
|
||||
});
|
||||
if (currentFlow && currentFlow.checkbox) {
|
||||
currentFlow.selected = currentSelectedCount === currentFlow.children.length
|
||||
}
|
||||
if (subWindow) {
|
||||
try {
|
||||
subWindow.postMessage({event:"refreshDebugNodeList", nodes:data},"*");
|
||||
} catch(err) {
|
||||
console.log(err);
|
||||
}
|
||||
}
|
||||
RED.debug.refreshDebugNodeList(data)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -396,6 +509,26 @@
|
||||
}
|
||||
}
|
||||
|
||||
function containsDebug(sid, map) {
|
||||
var item = map[sid];
|
||||
if (item) {
|
||||
if (item.debug === undefined) {
|
||||
var sfs = Object.keys(item.subflows);
|
||||
var contain = false;
|
||||
for (var i = 0; i < sfs.length; i++) {
|
||||
var sf = sfs[i];
|
||||
if (containsDebug(sf, map)) {
|
||||
contain = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
item.debug = contain;
|
||||
}
|
||||
return item.debug;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
$("#red-ui-sidebar-debug-open").on("click", function(e) {
|
||||
e.preventDefault();
|
||||
subWindow = window.open(document.location.toString().replace(/[?#].*$/,"")+"debug/view/view.html"+document.location.search,"nodeREDDebugView","menubar=no,location=no,toolbar=no,chrome,height=500,width=600");
|
||||
@@ -427,6 +560,8 @@
|
||||
options.messageSourceClick(msg.id,msg._alias,msg.path);
|
||||
} else if (msg.event === "clear") {
|
||||
options.clear();
|
||||
} else if (msg.event === "requestDebugNodeList") {
|
||||
options.requestDebugNodeList(msg.filteredNodes)
|
||||
}
|
||||
};
|
||||
window.addEventListener('message',this.handleWindowMessage);
|
||||
|
||||
@@ -5,6 +5,7 @@ module.exports = function(RED) {
|
||||
const fs = require("fs-extra");
|
||||
const path = require("path");
|
||||
var debuglength = RED.settings.debugMaxLength || 1000;
|
||||
var statuslength = RED.settings.debugStatusLength || 32;
|
||||
var useColors = RED.settings.debugUseColors || false;
|
||||
util.inspect.styles.boolean = "red";
|
||||
const { hasOwnProperty } = Object.prototype;
|
||||
@@ -164,7 +165,7 @@ module.exports = function(RED) {
|
||||
}
|
||||
}
|
||||
|
||||
if (st.length > 32) { st = st.substr(0,32) + "..."; }
|
||||
if (st.length > statuslength) { st = st.substr(0,statuslength) + "..."; }
|
||||
|
||||
var newStatus = {fill:fill, shape:shape, text:st};
|
||||
if (JSON.stringify(newStatus) !== node.oldState) { // only send if we have to
|
||||
|
||||
@@ -275,7 +275,7 @@
|
||||
value: [],
|
||||
type: "link in[]",
|
||||
validate: function (v, opt) {
|
||||
if ((this.linkType === "static" && v.length > 0)
|
||||
if (((this.linkType || "static") === "static" && v.length > 0)
|
||||
|| this.linkType === "dynamic") {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -167,19 +167,13 @@ RED.debug = (function() {
|
||||
var menu = RED.popover.menu({
|
||||
options: options,
|
||||
onselect: function(item) {
|
||||
if (item.value !== filterType) {
|
||||
filterType = item.value;
|
||||
$('#red-ui-sidebar-debug-filter span').text(RED._('node-red:debug.sidebar.'+filterType));
|
||||
refreshMessageList();
|
||||
RED.settings.set("debug.filter",filterType)
|
||||
}
|
||||
setFilterType(item.value)
|
||||
if (filterType === 'filterSelected') {
|
||||
refreshDebugNodeList();
|
||||
config.requestDebugNodeList(filteredNodes);
|
||||
filterDialog.slideDown(200);
|
||||
filterDialogShown = true;
|
||||
debugNodeTreeList.focus();
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
menu.show({
|
||||
@@ -254,131 +248,7 @@ RED.debug = (function() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
function containsDebug(sid, map) {
|
||||
var item = map[sid];
|
||||
if (item) {
|
||||
if (item.debug === undefined) {
|
||||
var sfs = Object.keys(item.subflows);
|
||||
var contain = false;
|
||||
for (var i = 0; i < sfs.length; i++) {
|
||||
var sf = sfs[i];
|
||||
if (containsDebug(sf, map)) {
|
||||
contain = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
item.debug = contain;
|
||||
}
|
||||
return item.debug;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
function refreshDebugNodeList() {
|
||||
var workspaceOrder = RED.nodes.getWorkspaceOrder();
|
||||
var workspaceOrderMap = {};
|
||||
workspaceOrder.forEach(function(ws,i) {
|
||||
workspaceOrderMap[ws] = i;
|
||||
});
|
||||
|
||||
var candidateNodes = [];
|
||||
var candidateSFs = [];
|
||||
var subflows = {};
|
||||
RED.nodes.eachNode(function (n) {
|
||||
var nt = n.type;
|
||||
if (nt === "debug") {
|
||||
if (n.z in workspaceOrderMap) {
|
||||
candidateNodes.push(n);
|
||||
}
|
||||
else {
|
||||
var sf = RED.nodes.subflow(n.z);
|
||||
if (sf) {
|
||||
subflows[sf.id] = {
|
||||
debug: true,
|
||||
subflows: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(nt.substring(0, 8) === "subflow:") {
|
||||
if (n.z in workspaceOrderMap) {
|
||||
candidateSFs.push(n);
|
||||
}
|
||||
else {
|
||||
var psf = RED.nodes.subflow(n.z);
|
||||
if (psf) {
|
||||
var sid = nt.substring(8);
|
||||
var item = subflows[psf.id];
|
||||
if (!item) {
|
||||
item = {
|
||||
debug: undefined,
|
||||
subflows: {}
|
||||
};
|
||||
subflows[psf.id] = item;
|
||||
}
|
||||
item.subflows[sid] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
candidateSFs.forEach(function (sf) {
|
||||
var sid = sf.type.substring(8);
|
||||
if (containsDebug(sid, subflows)) {
|
||||
candidateNodes.push(sf);
|
||||
}
|
||||
});
|
||||
|
||||
candidateNodes.sort(function(A,B) {
|
||||
var wsA = workspaceOrderMap[A.z];
|
||||
var wsB = workspaceOrderMap[B.z];
|
||||
if (wsA !== wsB) {
|
||||
return wsA-wsB;
|
||||
}
|
||||
var labelA = RED.utils.getNodeLabel(A,A.id);
|
||||
var labelB = RED.utils.getNodeLabel(B,B.id);
|
||||
return labelA.localeCompare(labelB);
|
||||
});
|
||||
var currentWs = null;
|
||||
var data = [];
|
||||
var currentFlow;
|
||||
var currentSelectedCount = 0;
|
||||
candidateNodes.forEach(function(node) {
|
||||
if (currentWs !== node.z) {
|
||||
if (currentFlow && currentFlow.checkbox) {
|
||||
currentFlow.selected = currentSelectedCount === currentFlow.children.length
|
||||
}
|
||||
currentSelectedCount = 0;
|
||||
currentWs = node.z;
|
||||
var parent = RED.nodes.workspace(currentWs) || RED.nodes.subflow(currentWs);
|
||||
currentFlow = {
|
||||
label: RED.utils.getNodeLabel(parent, currentWs),
|
||||
}
|
||||
if (!parent.disabled) {
|
||||
currentFlow.children = [];
|
||||
currentFlow.checkbox = true;
|
||||
} else {
|
||||
currentFlow.class = "disabled"
|
||||
}
|
||||
data.push(currentFlow);
|
||||
}
|
||||
if (currentFlow.children) {
|
||||
if (!filteredNodes[node.id]) {
|
||||
currentSelectedCount++;
|
||||
}
|
||||
currentFlow.children.push({
|
||||
label: RED.utils.getNodeLabel(node,node.id),
|
||||
node: node,
|
||||
checkbox: true,
|
||||
selected: !filteredNodes[node.id]
|
||||
});
|
||||
}
|
||||
});
|
||||
if (currentFlow && currentFlow.checkbox) {
|
||||
currentFlow.selected = currentSelectedCount === currentFlow.children.length
|
||||
}
|
||||
|
||||
function refreshDebugNodeList(data) {
|
||||
debugNodeTreeList.treeList("data", data);
|
||||
}
|
||||
|
||||
@@ -401,7 +271,7 @@ RED.debug = (function() {
|
||||
},200);
|
||||
}
|
||||
function _refreshMessageList(_activeWorkspace) {
|
||||
if (_activeWorkspace) {
|
||||
if (typeof _activeWorkspace === 'string') {
|
||||
activeWorkspace = _activeWorkspace.replace(/\./g,"_");
|
||||
}
|
||||
if (filterType === "filterAll") {
|
||||
@@ -479,12 +349,12 @@ RED.debug = (function() {
|
||||
filteredNodes[n.id] = true;
|
||||
});
|
||||
delete filteredNodes[sourceId];
|
||||
$("#red-ui-sidebar-debug-filterSelected").trigger("click");
|
||||
RED.settings.set('debug.filteredNodes',Object.keys(filteredNodes))
|
||||
setFilterType('filterSelected')
|
||||
refreshMessageList();
|
||||
}},
|
||||
{id:"red-ui-debug-msg-menu-item-clear-filter",label:RED._("node-red:debug.messageMenu.clearFilter"),onselect:function(){
|
||||
$("#red-ui-sidebar-debug-filterAll").trigger("click");
|
||||
clearFilterSettings()
|
||||
refreshMessageList();
|
||||
}}
|
||||
);
|
||||
@@ -713,9 +583,17 @@ RED.debug = (function() {
|
||||
if (!!clearFilter) {
|
||||
clearFilterSettings();
|
||||
}
|
||||
refreshDebugNodeList();
|
||||
config.requestDebugNodeList(filteredNodes);
|
||||
}
|
||||
|
||||
function setFilterType(type) {
|
||||
if (type !== filterType) {
|
||||
filterType = type;
|
||||
$('#red-ui-sidebar-debug-filter span').text(RED._('node-red:debug.sidebar.'+filterType));
|
||||
refreshMessageList();
|
||||
RED.settings.set("debug.filter",filterType)
|
||||
}
|
||||
}
|
||||
function clearFilterSettings() {
|
||||
filteredNodes = {};
|
||||
filterType = 'filterAll';
|
||||
@@ -728,6 +606,7 @@ RED.debug = (function() {
|
||||
init: init,
|
||||
refreshMessageList:refreshMessageList,
|
||||
handleDebugMessage: handleDebugMessage,
|
||||
clearMessageList: clearMessageList
|
||||
clearMessageList: clearMessageList,
|
||||
refreshDebugNodeList: refreshDebugNodeList
|
||||
}
|
||||
})();
|
||||
|
||||
@@ -12,6 +12,9 @@ $(function() {
|
||||
},
|
||||
clear: function() {
|
||||
window.opener.postMessage({event:"clear"},'*');
|
||||
},
|
||||
requestDebugNodeList: function(filteredNodes) {
|
||||
window.opener.postMessage({event: 'requestDebugNodeList', filteredNodes},'*')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +29,8 @@ $(function() {
|
||||
RED.debug.refreshMessageList(evt.data.activeWorkspace);
|
||||
} else if (evt.data.event === "projectChange") {
|
||||
RED.debug.clearMessageList(true);
|
||||
} else if (evt.data.event === "refreshDebugNodeList") {
|
||||
RED.debug.refreshDebugNodeList(evt.data.nodes)
|
||||
}
|
||||
},false);
|
||||
} catch(err) {
|
||||
|
||||
@@ -315,7 +315,7 @@ module.exports = function(RED) {
|
||||
var spec = module.module;
|
||||
if (spec && (spec !== "")) {
|
||||
moduleLoadPromises.push(RED.import(module.module).then(lib => {
|
||||
sandbox[vname] = lib.default;
|
||||
sandbox[vname] = lib.default || lib;
|
||||
}).catch(err => {
|
||||
node.error(RED._("function.error.moduleLoadError",{module:module.spec, error:err.toString()}))
|
||||
throw err;
|
||||
|
||||
@@ -103,7 +103,6 @@
|
||||
} else if (type === "istype") {
|
||||
r.v = rule.find(".node-input-rule-type-value").typedInput('type');
|
||||
r.vt = rule.find(".node-input-rule-type-value").typedInput('type');
|
||||
r.vt = (r.vt === "number") ? "num" : "str";
|
||||
} else if (type === "jsonata_exp") {
|
||||
r.v = rule.find(".node-input-rule-exp-value").typedInput('value');
|
||||
r.vt = rule.find(".node-input-rule-exp-value").typedInput('type');
|
||||
@@ -168,7 +167,35 @@
|
||||
label:RED._("node-red:common.label.payload"),
|
||||
validate: RED.validators.typedInput("propertyType", false)},
|
||||
propertyType: { value:"msg" },
|
||||
rules: {value:[{t:"eq", v:"", vt:"str"}]},
|
||||
rules: {
|
||||
value:[{t:"eq", v:"", vt:"str"}],
|
||||
validate: function (rules, opt) {
|
||||
let msg;
|
||||
const errors = []
|
||||
if (!rules || rules.length === 0) { return true }
|
||||
for (var i=0;i<rules.length;i++) {
|
||||
const opt = { label: RED._('node-red:switch.label.rule')+' '+(i+1) }
|
||||
const r = rules[i];
|
||||
if (r.t !== 'istype') {
|
||||
if (r.hasOwnProperty('v')) {
|
||||
if ((msg = RED.utils.validateTypedProperty(r.v,r.vt,opt)) !== true) {
|
||||
errors.push(msg)
|
||||
}
|
||||
}
|
||||
if (r.hasOwnProperty('v2')) {
|
||||
if ((msg = RED.utils.validateTypedProperty(r.v2,r.v2t,opt)) !== true) {
|
||||
errors.push(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (errors.length) {
|
||||
console.log(errors)
|
||||
return errors
|
||||
}
|
||||
return true;
|
||||
}
|
||||
},
|
||||
checkall: {value:"true", required:true},
|
||||
repair: {value:false},
|
||||
outputs: {value:1}
|
||||
@@ -218,7 +245,11 @@
|
||||
if (i > 0) {
|
||||
var lastRule = $("#node-input-rule-container").editableList('getItemAt',i-1);
|
||||
var exportedRule = exportRule(lastRule.element);
|
||||
opt.r.vt = exportedRule.vt;
|
||||
if (exportedRule.t === "istype") {
|
||||
opt.r.vt = (exportedRule.vt === "number") ? "num" : "str";
|
||||
} else {
|
||||
opt.r.vt = exportedRule.vt;
|
||||
}
|
||||
opt.r.v = "";
|
||||
// We could copy the value over as well and preselect it (see the 'activeElement' code below)
|
||||
// But not sure that feels right. Is copying over the last value 'expected' behaviour?
|
||||
|
||||
@@ -19,71 +19,42 @@
|
||||
|
||||
<script type="text/javascript">
|
||||
(function() {
|
||||
function isInvalidProperty(v,vt) {
|
||||
if (/msg|flow|global/.test(vt)) {
|
||||
if (!RED.utils.validatePropertyExpression(v)) {
|
||||
return RED._("node-red:change.errors.invalid-prop", {
|
||||
property: v
|
||||
});
|
||||
}
|
||||
} else if (vt === "jsonata") {
|
||||
try{ jsonata(v); } catch(e) {
|
||||
return RED._("node-red:change.errors.invalid-expr", {
|
||||
error: e.message
|
||||
});
|
||||
}
|
||||
} else if (vt === "json") {
|
||||
try{ JSON.parse(v); } catch(e) {
|
||||
return RED._("node-red:change.errors.invalid-json-data", {
|
||||
error: e.message
|
||||
});
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
RED.nodes.registerType('change', {
|
||||
color: "#E2D96E",
|
||||
category: 'function',
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
rules:{value:[{t:"set",p:"payload",pt:"msg",to:"",tot:"str"}],validate: function(rules, opt) {
|
||||
var msg;
|
||||
if (!rules || rules.length === 0) { return true }
|
||||
for (var i=0;i<rules.length;i++) {
|
||||
var r = rules[i];
|
||||
if (r.t === 'set') {
|
||||
if (msg = isInvalidProperty(r.p,r.pt)) {
|
||||
return msg;
|
||||
rules:{
|
||||
value:[{t:"set",p:"payload",pt:"msg",to:"",tot:"str"}],
|
||||
validate: function(rules, opt) {
|
||||
let msg;
|
||||
const errors = []
|
||||
if (!rules || rules.length === 0) { return true }
|
||||
for (var i=0;i<rules.length;i++) {
|
||||
const opt = { label: RED._('node-red:change.label.rule')+' '+(i+1) }
|
||||
const r = rules[i];
|
||||
if (r.t === 'set' || r.t === 'change' || r.t === 'delete' || r.t === 'move') {
|
||||
if ((msg = RED.utils.validateTypedProperty(r.p,r.pt,opt)) !== true) {
|
||||
errors.push(msg)
|
||||
}
|
||||
}
|
||||
if (msg = isInvalidProperty(r.to,r.tot)) {
|
||||
return msg;
|
||||
if (r.t === 'set' || r.t === 'change' || r.t === 'move') {
|
||||
if ((msg = RED.utils.validateTypedProperty(r.to,r.tot,opt)) !== true) {
|
||||
errors.push(msg)
|
||||
}
|
||||
}
|
||||
} else if (r.t === 'change') {
|
||||
if (msg = isInvalidProperty(r.p,r.pt)) {
|
||||
return msg;
|
||||
}
|
||||
if(msg = isInvalidProperty(r.from,r.fromt)) {
|
||||
return msg;
|
||||
}
|
||||
if(msg = isInvalidProperty(r.to,r.tot)) {
|
||||
return msg;
|
||||
}
|
||||
} else if (r.t === 'delete') {
|
||||
if (msg = isInvalidProperty(r.p,r.pt)) {
|
||||
return msg;
|
||||
}
|
||||
} else if (r.t === 'move') {
|
||||
if (msg = isInvalidProperty(r.p,r.pt)) {
|
||||
return msg;
|
||||
}
|
||||
if (msg = isInvalidProperty(r.to,r.tot)) {
|
||||
return msg;
|
||||
if (r.t === 'change') {
|
||||
if ((msg = RED.utils.validateTypedProperty(r.from,r.fromt,opt)) !== true) {
|
||||
errors.push(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (errors.length) {
|
||||
return errors
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}},
|
||||
},
|
||||
// legacy
|
||||
action: {value:""},
|
||||
property: {value:""},
|
||||
|
||||
@@ -117,7 +117,7 @@ module.exports = function(RED) {
|
||||
});
|
||||
return
|
||||
} else if (rule.tot === 'date') {
|
||||
value = Date.now();
|
||||
value = RED.util.evaluateNodeProperty(rule.to, rule.tot, node)
|
||||
} else if (rule.tot === 'jsonata') {
|
||||
RED.util.evaluateJSONataExpression(rule.to,msg, (err, value) => {
|
||||
if (err) {
|
||||
@@ -233,7 +233,9 @@ module.exports = function(RED) {
|
||||
// only replace if they match exactly
|
||||
RED.util.setMessageProperty(msg,property,value);
|
||||
} else {
|
||||
current = current.replace(fromRE,value);
|
||||
// if target is boolean then just replace it
|
||||
if (rule.tot === "bool") { current = value; }
|
||||
else { current = current.replace(fromRE,value); }
|
||||
RED.util.setMessageProperty(msg,property,current);
|
||||
}
|
||||
} else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') {
|
||||
|
||||
@@ -57,7 +57,9 @@
|
||||
action: {value:"scale"},
|
||||
round: {value:false},
|
||||
property: {value:"payload",required:true,
|
||||
label:RED._("node-red:common.label.property")},
|
||||
label:RED._("node-red:common.label.property"),
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowBlank: true })
|
||||
},
|
||||
name: {value:""}
|
||||
},
|
||||
inputs: 1,
|
||||
|
||||
@@ -153,7 +153,7 @@
|
||||
}
|
||||
var editorRow = $("#dialog-form>div.node-text-editor-row");
|
||||
height -= (parseInt(editorRow.css("marginTop"))+parseInt(editorRow.css("marginBottom")));
|
||||
$(".node-text-editor").css("height",height+"px");
|
||||
$("#dialog-form .node-text-editor").css("height",height+"px");
|
||||
this.editor.resize();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -284,7 +284,7 @@ module.exports = function(RED) {
|
||||
done();
|
||||
}
|
||||
}
|
||||
else {
|
||||
else if (!msg.hasOwnProperty("reset")) {
|
||||
if (maxKeptMsgsCount(node) > 0) {
|
||||
if (node.intervalID === -1) {
|
||||
node.send(msg);
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
color:"darksalmon",
|
||||
defaults: {
|
||||
command: {value:""},
|
||||
addpay: {value:""},
|
||||
addpay: {value:"", validate: RED.validators.typedInput({ type: 'msg', allowBlank: true })},
|
||||
append: {value:""},
|
||||
useSpawn: {value:"false"},
|
||||
timer: {value:""},
|
||||
|
||||
@@ -56,9 +56,11 @@
|
||||
inout: {value:"out"},
|
||||
septopics: {value:true},
|
||||
property: {value:"payload", required:true,
|
||||
label:RED._("node-red:rbe.label.property")},
|
||||
label:RED._("node-red:rbe.label.property"),
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true })},
|
||||
topi: {value:"topic", required:true,
|
||||
label:RED._("node-red:rbe.label.topic")}
|
||||
label:RED._("node-red:rbe.label.topic"),
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true })}
|
||||
},
|
||||
inputs:1,
|
||||
outputs:1,
|
||||
|
||||
@@ -104,6 +104,7 @@ module.exports = function(RED) {
|
||||
* @returns `true` if it is a valid topic
|
||||
*/
|
||||
function isValidPublishTopic(topic) {
|
||||
if (topic.length === 0) return false;
|
||||
return !/[\+#\b\f\n\r\t\v\0]/.test(topic);
|
||||
}
|
||||
|
||||
@@ -219,8 +220,8 @@ module.exports = function(RED) {
|
||||
*/
|
||||
function subscriptionHandler(node, datatype ,topic, payload, packet) {
|
||||
const msg = {topic:topic, payload:null, qos:packet.qos, retain:packet.retain};
|
||||
const v5 = (node && node.brokerConn)
|
||||
? node.brokerConn.v5()
|
||||
const v5 = (node && node.brokerConn)
|
||||
? node.brokerConn.v5()
|
||||
: Object.prototype.hasOwnProperty.call(packet, "properties");
|
||||
if(v5 && packet.properties) {
|
||||
setStrProp(packet.properties, msg, "responseTopic");
|
||||
@@ -451,7 +452,7 @@ module.exports = function(RED) {
|
||||
|
||||
/**
|
||||
* Perform the disconnect action
|
||||
* @param {MQTTInNode|MQTTOutNode} node
|
||||
* @param {MQTTInNode|MQTTOutNode} node
|
||||
* @param {Function} done
|
||||
*/
|
||||
function handleDisconnectAction(node, done) {
|
||||
@@ -611,7 +612,7 @@ module.exports = function(RED) {
|
||||
node.brokerurl = node.url;
|
||||
} else {
|
||||
// if the broker is ws:// or wss:// or tcp://
|
||||
if (node.broker.indexOf("://") > -1) {
|
||||
if ((typeof node.broker === 'string') && node.broker.indexOf("://") > -1) {
|
||||
node.brokerurl = node.broker;
|
||||
// Only for ws or wss, check if proxy env var for additional configuration
|
||||
if (node.brokerurl.indexOf("wss://") > -1 || node.brokerurl.indexOf("ws://") > -1) {
|
||||
@@ -865,7 +866,7 @@ module.exports = function(RED) {
|
||||
* Call end and wait for the client to end (or timeout)
|
||||
* @param {mqtt.MqttClient} client The broker client
|
||||
* @param {number} ms The time to wait for the client to end
|
||||
* @returns
|
||||
* @returns
|
||||
*/
|
||||
let waitEnd = (client, ms) => {
|
||||
return new Promise( (resolve, reject) => {
|
||||
@@ -905,7 +906,7 @@ module.exports = function(RED) {
|
||||
node.subid = 1;
|
||||
|
||||
//typedef for subscription object:
|
||||
/**
|
||||
/**
|
||||
* @typedef {Object} Subscription
|
||||
* @property {String} topic - topic to subscribe to
|
||||
* @property {Object} [options] - options object
|
||||
@@ -933,7 +934,7 @@ module.exports = function(RED) {
|
||||
const ref = _ref || 0;
|
||||
let options
|
||||
let qos = 1 // default to QoS 1 (AWS and several other brokers don't support QoS 2)
|
||||
|
||||
|
||||
// if options is an object, then clone it
|
||||
if (typeof _options == "object") {
|
||||
options = RED.util.cloneMessage(_options || {})
|
||||
@@ -947,7 +948,7 @@ module.exports = function(RED) {
|
||||
if (typeof qos === "number" && qos >= 0 && qos <= 2) {
|
||||
options.qos = qos;
|
||||
}
|
||||
|
||||
|
||||
subscription.topic = _topic;
|
||||
subscription.qos = qos;
|
||||
subscription.options = RED.util.cloneMessage(options);
|
||||
@@ -957,16 +958,16 @@ module.exports = function(RED) {
|
||||
}
|
||||
|
||||
/**
|
||||
* If topic is a subscription object, then use that, otherwise look up the topic in
|
||||
* If topic is a subscription object, then use that, otherwise look up the topic in
|
||||
* the subscriptions object. If the topic is not found, then create a new subscription
|
||||
* object and add it to the subscriptions object.
|
||||
* @param {Subscription|String} topic
|
||||
* @param {*} options
|
||||
* @param {*} callback
|
||||
* @param {*} ref
|
||||
* @param {Subscription|String} topic
|
||||
* @param {*} options
|
||||
* @param {*} callback
|
||||
* @param {*} ref
|
||||
*/
|
||||
node.subscribe = function (topic, options, callback, ref) {
|
||||
/** @type {Subscription} */
|
||||
/** @type {Subscription} */
|
||||
let subscription
|
||||
let doCompare = false
|
||||
let changesFound = false
|
||||
@@ -1004,7 +1005,7 @@ module.exports = function(RED) {
|
||||
_brokerConn.unsubscribe(sub.topic, sub.ref, true)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
// if subscription is found (or sent in as a parameter), then check for changes.
|
||||
// if there are any changes requested, tidy up the old subscription
|
||||
if (subscription) {
|
||||
@@ -1091,7 +1092,7 @@ module.exports = function(RED) {
|
||||
delete sub[ref]
|
||||
}
|
||||
}
|
||||
// if instructed to remove the actual MQTT client subscription
|
||||
// if instructed to remove the actual MQTT client subscription
|
||||
if (unsub) {
|
||||
// if there are no more subscriptions for the topic, then remove the topic
|
||||
if (Object.keys(sub).length === 0) {
|
||||
|
||||
@@ -141,15 +141,7 @@ in your Node-RED user directory (${RED.settings.userDir}).
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* @param {Object} headersObject
|
||||
* @param {string} name
|
||||
* @return {any} value
|
||||
*/
|
||||
const getHeaderValue = (headersObject, name) => {
|
||||
const asLowercase = name.toLowercase();
|
||||
return headersObject[Object.keys(headersObject).find(k => k.toLowerCase() === asLowercase)];
|
||||
}
|
||||
|
||||
this.on("input",function(msg,nodeSend,nodeDone) {
|
||||
checkNodeAgentPatch();
|
||||
//reset redirectList on each request
|
||||
@@ -300,7 +292,7 @@ in your Node-RED user directory (${RED.settings.userDir}).
|
||||
}
|
||||
|
||||
opts.headers = {};
|
||||
//add msg.headers
|
||||
//add msg.headers
|
||||
//NOTE: ui headers will take precidence over msg.headers
|
||||
if (msg.headers) {
|
||||
if (msg.headers.hasOwnProperty('x-node-red-request-node')) {
|
||||
@@ -452,10 +444,6 @@ in your Node-RED user directory (${RED.settings.userDir}).
|
||||
formData.append(opt, val);
|
||||
} else if (typeof val === 'object' && val.hasOwnProperty('value')) {
|
||||
formData.append(opt,val.value,val.options || {});
|
||||
} else if (Array.isArray(val)) {
|
||||
for (var i=0; i<val.length; i++) {
|
||||
formData.append(opt, val[i])
|
||||
}
|
||||
} else {
|
||||
formData.append(opt,JSON.stringify(val));
|
||||
}
|
||||
@@ -637,7 +625,7 @@ in your Node-RED user directory (${RED.settings.userDir}).
|
||||
msg.payload = msg.payload.toString('utf8'); // txt
|
||||
|
||||
if (node.ret === "obj") {
|
||||
if (msg.statusCode == 204){msg.payload= "{}"};
|
||||
if (msg.statusCode == 204){msg.payload= "{}"};
|
||||
try { msg.payload = JSON.parse(msg.payload); } // obj
|
||||
catch(e) { node.warn(RED._("httpin.errors.json-error")); }
|
||||
}
|
||||
@@ -744,7 +732,7 @@ in your Node-RED user directory (${RED.settings.userDir}).
|
||||
*
|
||||
* If the algorithm directive's value ends with "-sess", then HA1 is
|
||||
* HA1=digestCompute(digestCompute(username:realm:password):nonce:cnonce)
|
||||
*
|
||||
*
|
||||
* If the algorithm directive's value does not end with "-sess", then HA1 is
|
||||
* HA1=digestCompute(username:realm:password)
|
||||
*/
|
||||
|
||||
@@ -411,23 +411,33 @@ module.exports = function(RED) {
|
||||
if (msg._session && msg._session.type == "tcp") {
|
||||
var client = connectionPool[msg._session.id];
|
||||
if (client) {
|
||||
if (Buffer.isBuffer(msg.payload)) {
|
||||
client.write(msg.payload);
|
||||
} else if (typeof msg.payload === "string" && node.base64) {
|
||||
client.write(Buffer.from(msg.payload,'base64'));
|
||||
} else {
|
||||
client.write(Buffer.from(""+msg.payload));
|
||||
if (msg?.reset === true) {
|
||||
client.destroy();
|
||||
}
|
||||
else {
|
||||
if (Buffer.isBuffer(msg.payload)) {
|
||||
client.write(msg.payload);
|
||||
} else if (typeof msg.payload === "string" && node.base64) {
|
||||
client.write(Buffer.from(msg.payload,'base64'));
|
||||
} else {
|
||||
client.write(Buffer.from(""+msg.payload));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (var i in connectionPool) {
|
||||
if (Buffer.isBuffer(msg.payload)) {
|
||||
connectionPool[i].write(msg.payload);
|
||||
} else if (typeof msg.payload === "string" && node.base64) {
|
||||
connectionPool[i].write(Buffer.from(msg.payload,'base64'));
|
||||
} else {
|
||||
connectionPool[i].write(Buffer.from(""+msg.payload));
|
||||
if (msg?.reset === true) {
|
||||
connectionPool[i].destroy();
|
||||
}
|
||||
else {
|
||||
if (Buffer.isBuffer(msg.payload)) {
|
||||
connectionPool[i].write(msg.payload);
|
||||
} else if (typeof msg.payload === "string" && node.base64) {
|
||||
connectionPool[i].write(Buffer.from(msg.payload,'base64'));
|
||||
} else {
|
||||
connectionPool[i].write(Buffer.from(""+msg.payload));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -547,13 +557,33 @@ module.exports = function(RED) {
|
||||
|
||||
this.on("input", function(msg, nodeSend, nodeDone) {
|
||||
var i = 0;
|
||||
if ((!Buffer.isBuffer(msg.payload)) && (typeof msg.payload !== "string")) {
|
||||
if (msg.payload !== undefined && (!Buffer.isBuffer(msg.payload)) && (typeof msg.payload !== "string")) {
|
||||
msg.payload = msg.payload.toString();
|
||||
}
|
||||
|
||||
var host = node.server || msg.host;
|
||||
var port = node.port || msg.port;
|
||||
|
||||
if (node.out === "sit" && msg?.reset) {
|
||||
if (msg.reset === true) { // kill all connections
|
||||
for (var cl in clients) {
|
||||
if (clients[cl].hasOwnProperty("client")) {
|
||||
clients[cl].client.destroy();
|
||||
delete clients[cl];
|
||||
}
|
||||
}
|
||||
}
|
||||
if (typeof(msg.reset) === "string" && msg.reset.includes(":")) { // just kill connection host:port
|
||||
if (clients.hasOwnProperty(msg.reset) && clients[msg.reset].hasOwnProperty("client")) {
|
||||
clients[msg.reset].client.destroy();
|
||||
delete clients[msg.reset];
|
||||
}
|
||||
}
|
||||
const cc = Object.keys(clients).length;
|
||||
node.status({fill:"green",shape:cc===0?"ring":"dot",text:RED._("tcpin.status.connections",{count:cc})});
|
||||
if ((host === undefined || port === undefined) && !msg.hasOwnProperty("payload")) { return; }
|
||||
}
|
||||
|
||||
// Store client information independently
|
||||
// the clients object will have:
|
||||
// clients[id].client, clients[id].msg, clients[id].timeout
|
||||
@@ -621,13 +651,16 @@ module.exports = function(RED) {
|
||||
clients[connection_id].connecting = true;
|
||||
clients[connection_id].client.connect(connOpts, function() {
|
||||
//node.log(RED._("tcpin.errors.client-connected"));
|
||||
node.status({fill:"green",shape:"dot",text:"common.status.connected"});
|
||||
// node.status({fill:"green",shape:"dot",text:"common.status.connected"});
|
||||
node.status({fill:"green",shape:"dot",text:RED._("tcpin.status.connections",{count:Object.keys(clients).length})});
|
||||
if (clients[connection_id] && clients[connection_id].client) {
|
||||
clients[connection_id].connected = true;
|
||||
clients[connection_id].connecting = false;
|
||||
let event;
|
||||
while (event = dequeue(clients[connection_id].msgQueue)) {
|
||||
clients[connection_id].client.write(event.msg.payload);
|
||||
if (event.msg.payload !== undefined) {
|
||||
clients[connection_id].client.write(event.msg.payload);
|
||||
}
|
||||
event.nodeDone();
|
||||
}
|
||||
if (node.out === "time" && node.splitc < 0) {
|
||||
@@ -823,7 +856,9 @@ module.exports = function(RED) {
|
||||
else if (!clients[connection_id].connecting && clients[connection_id].connected) {
|
||||
if (clients[connection_id] && clients[connection_id].client) {
|
||||
let event = dequeue(clients[connection_id].msgQueue)
|
||||
clients[connection_id].client.write(event.msg.payload);
|
||||
if (event.msg.payload !== undefined ) {
|
||||
clients[connection_id].client.write(event.msg.payload);
|
||||
}
|
||||
event.nodeDone();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,20 @@
|
||||
</select>
|
||||
<input style="width:40px;" type="text" id="node-input-sep" pattern=".">
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
<label><i class="fa fa-code"></i> <span data-i18n="csv.label.spec"></span></label>
|
||||
<div style="display: inline-grid;width: 70%;">
|
||||
<select style="width:100%" id="csv-option-spec">
|
||||
<option value="rfc" data-i18n="csv.spec.rfc"></option>
|
||||
<option value="" data-i18n="csv.spec.legacy"></option>
|
||||
</select>
|
||||
<div>
|
||||
<div class="form-tips csv-lecacy-warning" data-i18n="node-red:csv.spec.legacy_warning"
|
||||
style="width: calc(100% - 18px); margin-top: 4px; max-width: unset;">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label for="node-input-name"><i class="fa fa-tag"></i> <span data-i18n="common.label.name"></span></label>
|
||||
<input type="text" id="node-input-name" data-i18n="[placeholder]common.label.name">
|
||||
@@ -60,10 +73,10 @@
|
||||
<div class="form-row" style="padding-left:20px;">
|
||||
<label></label>
|
||||
<label style="width:auto; margin-right:10px;" for="node-input-ret"><span data-i18n="csv.label.newline"></span></label>
|
||||
<select style="width:150px;" id="node-input-ret">
|
||||
<select style="width:calc(70% - 108px);" id="node-input-ret">
|
||||
<option value='\r\n' data-i18n="csv.newline.windows"></option>
|
||||
<option value='\n' data-i18n="csv.newline.linux"></option>
|
||||
<option value='\r' data-i18n="csv.newline.mac"></option>
|
||||
<option value='\r\n' data-i18n="csv.newline.windows"></option>
|
||||
</select>
|
||||
</div>
|
||||
</script>
|
||||
@@ -75,6 +88,7 @@
|
||||
color:"#DEBD5C",
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
spec: {value:"rfc"},
|
||||
sep: {
|
||||
value:',', required:true,
|
||||
label:RED._("node-red:csv.label.separator"),
|
||||
@@ -83,7 +97,7 @@
|
||||
hdrin: {value:""},
|
||||
hdrout: {value:"none"},
|
||||
multi: {value:"one",required:true},
|
||||
ret: {value:'\\n'},
|
||||
ret: {value:'\\r\\n'}, // default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)")
|
||||
temp: {value:""},
|
||||
skip: {value:"0"},
|
||||
strings: {value:true},
|
||||
@@ -123,6 +137,27 @@
|
||||
$("#node-input-sep").hide();
|
||||
}
|
||||
});
|
||||
|
||||
$("#csv-option-spec").on("change", function() {
|
||||
if ($("#csv-option-spec").val() == "rfc") {
|
||||
$(".form-tips.csv-lecacy-warning").hide();
|
||||
} else {
|
||||
$(".form-tips.csv-lecacy-warning").show();
|
||||
}
|
||||
});
|
||||
// new nodes will have `spec` set to "rfc" (default), but existing nodes will either not have
|
||||
// a spec value or it will be empty - we need to maintain the legacy behaviour for existing
|
||||
// flows but default to rfc for new nodes
|
||||
let spec = !this.spec ? "" : "rfc"
|
||||
$("#csv-option-spec").val(spec).trigger("change")
|
||||
},
|
||||
oneditsave: function() {
|
||||
const specFormVal = $("#csv-option-spec").val() || '' // empty === legacy
|
||||
const spectNodeVal = this.spec || '' // empty === legacy, null/undefined means in-place node upgrade (keep as is)
|
||||
if (specFormVal !== spectNodeVal) {
|
||||
// only update the flow value if changed (avoid marking the node dirty unnecessarily)
|
||||
this.spec = specFormVal
|
||||
}
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
@@ -15,319 +15,674 @@
|
||||
**/
|
||||
|
||||
module.exports = function(RED) {
|
||||
const csv = require('./lib/csv')
|
||||
|
||||
"use strict";
|
||||
function CSVNode(n) {
|
||||
RED.nodes.createNode(this,n);
|
||||
this.template = (n.temp || "");
|
||||
this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
|
||||
this.quo = '"';
|
||||
this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
|
||||
this.winflag = (this.ret === "\r\n");
|
||||
this.lineend = "\n";
|
||||
this.multi = n.multi || "one";
|
||||
this.hdrin = n.hdrin || false;
|
||||
this.hdrout = n.hdrout || "none";
|
||||
this.goodtmpl = true;
|
||||
this.skip = parseInt(n.skip || 0);
|
||||
this.store = [];
|
||||
this.parsestrings = n.strings;
|
||||
this.include_empty_strings = n.include_empty_strings || false;
|
||||
this.include_null_values = n.include_null_values || false;
|
||||
if (this.parsestrings === undefined) { this.parsestrings = true; }
|
||||
if (this.hdrout === false) { this.hdrout = "none"; }
|
||||
if (this.hdrout === true) { this.hdrout = "all"; }
|
||||
var tmpwarn = true;
|
||||
var node = this;
|
||||
var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
|
||||
RED.nodes.createNode(this,n)
|
||||
const node = this
|
||||
const RFC4180Mode = n.spec === 'rfc'
|
||||
const legacyMode = !RFC4180Mode
|
||||
|
||||
// pass in an array of column names to be trimmed, de-quoted and retrimmed
|
||||
var clean = function(col,sep) {
|
||||
if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
|
||||
col = col.trim().split(re) || [""];
|
||||
col = col.map(x => x.replace(/"/g,'').trim());
|
||||
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
|
||||
else { node.goodtmpl = true; }
|
||||
return col;
|
||||
}
|
||||
var template = clean(node.template,',');
|
||||
var notemplate = template.length === 1 && template[0] === '';
|
||||
node.hdrSent = false;
|
||||
node.status({}) // clear status
|
||||
|
||||
this.on("input", function(msg, send, done) {
|
||||
if (msg.hasOwnProperty("reset")) {
|
||||
node.hdrSent = false;
|
||||
if (legacyMode) {
|
||||
this.template = (n.temp || "");
|
||||
this.sep = (n.sep || ',').replace(/\\t/g,"\t").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
|
||||
this.quo = '"';
|
||||
this.ret = (n.ret || "\n").replace(/\\n/g,"\n").replace(/\\r/g,"\r");
|
||||
this.winflag = (this.ret === "\r\n");
|
||||
this.lineend = "\n";
|
||||
this.multi = n.multi || "one";
|
||||
this.hdrin = n.hdrin || false;
|
||||
this.hdrout = n.hdrout || "none";
|
||||
this.goodtmpl = true;
|
||||
this.skip = parseInt(n.skip || 0);
|
||||
this.store = [];
|
||||
this.parsestrings = n.strings;
|
||||
this.include_empty_strings = n.include_empty_strings || false;
|
||||
this.include_null_values = n.include_null_values || false;
|
||||
if (this.parsestrings === undefined) { this.parsestrings = true; }
|
||||
if (this.hdrout === false) { this.hdrout = "none"; }
|
||||
if (this.hdrout === true) { this.hdrout = "all"; }
|
||||
var tmpwarn = true;
|
||||
// var node = this;
|
||||
var re = new RegExp(node.sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') + '(?=(?:(?:[^"]*"){2})*[^"]*$)','g');
|
||||
|
||||
// pass in an array of column names to be trimmed, de-quoted and retrimmed
|
||||
var clean = function(col,sep) {
|
||||
if (sep) { re = new RegExp(sep.replace(/[-[\]{}()*+!<=:?.\/\\^$|#\s,]/g,'\\$&') +'(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); }
|
||||
col = col.trim().split(re) || [""];
|
||||
col = col.map(x => x.replace(/"/g,'').trim());
|
||||
if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
|
||||
else { node.goodtmpl = true; }
|
||||
return col;
|
||||
}
|
||||
if (msg.hasOwnProperty("payload")) {
|
||||
if (typeof msg.payload == "object") { // convert object to CSV string
|
||||
try {
|
||||
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
|
||||
template = clean(node.template);
|
||||
}
|
||||
var ou = "";
|
||||
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
|
||||
if (node.hdrout !== "none" && node.hdrSent === false) {
|
||||
if ((template.length === 1) && (template[0] === '')) {
|
||||
if (msg.hasOwnProperty("columns")) {
|
||||
template = clean(msg.columns || "",",");
|
||||
}
|
||||
else {
|
||||
template = Object.keys(msg.payload[0]);
|
||||
}
|
||||
var template = clean(node.template,',');
|
||||
var notemplate = template.length === 1 && template[0] === '';
|
||||
node.hdrSent = false;
|
||||
|
||||
this.on("input", function(msg, send, done) {
|
||||
if (msg.hasOwnProperty("reset")) {
|
||||
node.hdrSent = false;
|
||||
}
|
||||
if (msg.hasOwnProperty("payload")) {
|
||||
if (typeof msg.payload == "object") { // convert object to CSV string
|
||||
try {
|
||||
if (!(notemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
|
||||
template = clean(node.template);
|
||||
}
|
||||
ou += template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret;
|
||||
if (node.hdrout === "once") { node.hdrSent = true; }
|
||||
}
|
||||
for (var s = 0; s < msg.payload.length; s++) {
|
||||
if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
|
||||
if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
|
||||
for (var t = 0; t < msg.payload[s].length; t++) {
|
||||
if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
|
||||
if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
}
|
||||
ou += msg.payload[s].join(node.sep) + node.ret;
|
||||
}
|
||||
else {
|
||||
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
|
||||
template = clean(msg.columns || "",",");
|
||||
}
|
||||
const ou = [];
|
||||
if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
|
||||
if (node.hdrout !== "none" && node.hdrSent === false) {
|
||||
if ((template.length === 1) && (template[0] === '')) {
|
||||
/* istanbul ignore else */
|
||||
if (tmpwarn === true) { // just warn about missing template once
|
||||
node.warn(RED._("csv.errors.obj_csv"));
|
||||
tmpwarn = false;
|
||||
if (msg.hasOwnProperty("columns")) {
|
||||
template = clean(msg.columns || "",",");
|
||||
}
|
||||
for (var p in msg.payload[0]) {
|
||||
else {
|
||||
template = Object.keys(msg.payload[0]);
|
||||
}
|
||||
}
|
||||
ou.push(template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep));
|
||||
if (node.hdrout === "once") { node.hdrSent = true; }
|
||||
}
|
||||
for (var s = 0; s < msg.payload.length; s++) {
|
||||
if ((Array.isArray(msg.payload[s])) || (typeof msg.payload[s] !== "object")) {
|
||||
if (typeof msg.payload[s] !== "object") { msg.payload = [ msg.payload ]; }
|
||||
for (var t = 0; t < msg.payload[s].length; t++) {
|
||||
if (msg.payload[s][t] === undefined) { msg.payload[s][t] = ""; }
|
||||
if (msg.payload[s][t].toString().indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
msg.payload[s][t] = msg.payload[s][t].toString().replace(/"/g, '""');
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
else if (msg.payload[s][t].toString().indexOf(node.sep) !== -1) { // add quotes if any "commas"
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
else if (msg.payload[s][t].toString().indexOf("\n") !== -1) { // add quotes if any "\n"
|
||||
msg.payload[s][t] = node.quo + msg.payload[s][t].toString() + node.quo;
|
||||
}
|
||||
}
|
||||
ou.push(msg.payload[s].join(node.sep));
|
||||
}
|
||||
else {
|
||||
if ((template.length === 1) && (template[0] === '') && (msg.hasOwnProperty("columns"))) {
|
||||
template = clean(msg.columns || "",",");
|
||||
}
|
||||
if ((template.length === 1) && (template[0] === '')) {
|
||||
/* istanbul ignore else */
|
||||
if (msg.payload[s].hasOwnProperty(p)) {
|
||||
if (tmpwarn === true) { // just warn about missing template once
|
||||
node.warn(RED._("csv.errors.obj_csv"));
|
||||
tmpwarn = false;
|
||||
}
|
||||
const row = [];
|
||||
for (var p in msg.payload[0]) {
|
||||
/* istanbul ignore else */
|
||||
if (typeof msg.payload[s][p] !== "object") {
|
||||
// Fix to honour include null values flag
|
||||
//if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) {
|
||||
var q = "";
|
||||
if (msg.payload[s][p] !== undefined) {
|
||||
q += msg.payload[s][p];
|
||||
if (msg.payload[s].hasOwnProperty(p)) {
|
||||
/* istanbul ignore else */
|
||||
if (typeof msg.payload[s][p] !== "object") {
|
||||
// Fix to honour include null values flag
|
||||
//if (typeof msg.payload[s][p] !== "object" || (node.include_null_values === true && msg.payload[s][p] === null)) {
|
||||
var q = "";
|
||||
if (msg.payload[s][p] !== undefined) {
|
||||
q += msg.payload[s][p];
|
||||
}
|
||||
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
q = q.replace(/"/g, '""');
|
||||
row.push(node.quo + q + node.quo);
|
||||
}
|
||||
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
|
||||
row.push(node.quo + q + node.quo);
|
||||
}
|
||||
else { row.push(q); } // otherwise just add
|
||||
}
|
||||
if (q.indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
q = q.replace(/"/g, '""');
|
||||
ou += node.quo + q + node.quo + node.sep;
|
||||
}
|
||||
else if (q.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
|
||||
ou += node.quo + q + node.quo + node.sep;
|
||||
}
|
||||
else { ou += q + node.sep; } // otherwise just add
|
||||
}
|
||||
}
|
||||
ou.push(row.join(node.sep)); // add separator
|
||||
}
|
||||
ou = ou.slice(0,-1) + node.ret;
|
||||
else {
|
||||
const row = [];
|
||||
for (var t=0; t < template.length; t++) {
|
||||
if (template[t] === '') {
|
||||
row.push('');
|
||||
}
|
||||
else {
|
||||
var tt = template[t];
|
||||
if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; }
|
||||
else { tt = '"'+tt+'"'; }
|
||||
var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
|
||||
/* istanbul ignore else */
|
||||
if (p === undefined) { p = ""; }
|
||||
// fix to honour include null values flag
|
||||
//if (p === null && node.include_null_values !== true) { p = "";}
|
||||
p = RED.util.ensureString(p);
|
||||
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
p = p.replace(/"/g, '""');
|
||||
row.push(node.quo + p + node.quo);
|
||||
}
|
||||
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
|
||||
row.push(node.quo + p + node.quo);
|
||||
}
|
||||
else { row.push(p); } // otherwise just add
|
||||
}
|
||||
}
|
||||
ou.push(row.join(node.sep)); // add separator
|
||||
}
|
||||
}
|
||||
}
|
||||
// join lines, don't forget to add the last new line
|
||||
msg.payload = ou.join(node.ret) + node.ret;
|
||||
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
|
||||
if (msg.payload !== '') {
|
||||
send(msg);
|
||||
}
|
||||
done();
|
||||
}
|
||||
catch(e) { done(e); }
|
||||
}
|
||||
else if (typeof msg.payload == "string") { // convert CSV string to object
|
||||
try {
|
||||
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
|
||||
var j = 0; // pointer into array of template items
|
||||
var k = [""]; // array of data for each of the template items
|
||||
var o = {}; // output object to build up
|
||||
var a = []; // output array is needed for multiline option
|
||||
var first = true; // is this the first line
|
||||
var last = false;
|
||||
var line = msg.payload;
|
||||
var linecount = 0;
|
||||
var tmp = "";
|
||||
var has_parts = msg.hasOwnProperty("parts");
|
||||
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
|
||||
if (msg.hasOwnProperty("parts")) {
|
||||
linecount = msg.parts.index;
|
||||
if (msg.parts.index > node.skip) { first = false; }
|
||||
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
|
||||
}
|
||||
|
||||
// For now we are just going to assume that any \r or \n means an end of line...
|
||||
// got to be a weird csv that has singleton \r \n in it for another reason...
|
||||
|
||||
// Now process the whole file/line
|
||||
var nocr = (line.match(/[\r\n]/g)||[]).length;
|
||||
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
|
||||
for (var i = 0; i < line.length; i++) {
|
||||
if (first && (linecount < node.skip)) {
|
||||
if (line[i] === "\n") { linecount += 1; }
|
||||
continue;
|
||||
}
|
||||
if ((node.hdrin === true) && first) { // if the template is in the first line
|
||||
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
|
||||
if (line.length - i === 1) { tmp += line[i]; }
|
||||
template = clean(tmp,node.sep);
|
||||
first = false;
|
||||
}
|
||||
else { tmp += line[i]; }
|
||||
}
|
||||
else {
|
||||
for (var t=0; t < template.length; t++) {
|
||||
if (template[t] === '') {
|
||||
ou += node.sep;
|
||||
if (line[i] === node.quo) { // if it's a quote toggle inside or outside
|
||||
f = !f;
|
||||
if (line[i-1] === node.quo) {
|
||||
if (f === false) { k[j] += '\"'; }
|
||||
} // if it's a quotequote then it's actually a quote
|
||||
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
|
||||
}
|
||||
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
// if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
|
||||
if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
else {
|
||||
var tt = template[t];
|
||||
if (template[t].indexOf('"') >=0 ) { tt = "'"+tt+"'"; }
|
||||
else { tt = '"'+tt+'"'; }
|
||||
var p = RED.util.getMessageProperty(msg,'payload["'+s+'"]['+tt+']');
|
||||
/* istanbul ignore else */
|
||||
if (p === undefined) { p = ""; }
|
||||
// fix to honour include null values flag
|
||||
//if (p === null && node.include_null_values !== true) { p = "";}
|
||||
p = RED.util.ensureString(p);
|
||||
if (p.indexOf(node.quo) !== -1) { // add double quotes if any quotes
|
||||
p = p.replace(/"/g, '""');
|
||||
ou += node.quo + p + node.quo + node.sep;
|
||||
}
|
||||
else if (p.indexOf(node.sep) !== -1 || p.indexOf("\n") !== -1) { // add quotes if any "commas" or "\n"
|
||||
ou += node.quo + p + node.quo + node.sep;
|
||||
}
|
||||
else { ou += p + node.sep; } // otherwise just add
|
||||
j += 1;
|
||||
// if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
|
||||
k[j] = line.length - 1 === i ? null : "";
|
||||
}
|
||||
else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
|
||||
//console.log(j,k,o,k[j]);
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
// if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
|
||||
if (line[i-1] === node.sep) k[j] = null;
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
a.push(o); // add to the array
|
||||
}
|
||||
j = 0;
|
||||
k = [""];
|
||||
o = {};
|
||||
f = true; // reset in/out flag ready for next line.
|
||||
}
|
||||
else { // just add to the part of the message
|
||||
k[j] += line[i];
|
||||
}
|
||||
ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
|
||||
}
|
||||
}
|
||||
}
|
||||
msg.payload = ou;
|
||||
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(',');
|
||||
if (msg.payload !== '') { send(msg); }
|
||||
done();
|
||||
}
|
||||
catch(e) { done(e); }
|
||||
}
|
||||
else if (typeof msg.payload == "string") { // convert CSV string to object
|
||||
try {
|
||||
var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
|
||||
var j = 0; // pointer into array of template items
|
||||
var k = [""]; // array of data for each of the template items
|
||||
var o = {}; // output object to build up
|
||||
var a = []; // output array is needed for multiline option
|
||||
var first = true; // is this the first line
|
||||
var last = false;
|
||||
var line = msg.payload;
|
||||
var linecount = 0;
|
||||
var tmp = "";
|
||||
var has_parts = msg.hasOwnProperty("parts");
|
||||
var reg = /^[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+$/i;
|
||||
if (msg.hasOwnProperty("parts")) {
|
||||
linecount = msg.parts.index;
|
||||
if (msg.parts.index > node.skip) { first = false; }
|
||||
if (msg.parts.hasOwnProperty("count") && (msg.parts.index+1 >= msg.parts.count)) { last = true; }
|
||||
}
|
||||
// Finished so finalize and send anything left
|
||||
if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
|
||||
// For now we are just going to assume that any \r or \n means an end of line...
|
||||
// got to be a weird csv that has singleton \r \n in it for another reason...
|
||||
|
||||
// Now process the whole file/line
|
||||
var nocr = (line.match(/[\r\n]/g)||[]).length;
|
||||
if (has_parts && node.multi === "mult" && nocr > 1) { tmp = ""; first = true; }
|
||||
for (var i = 0; i < line.length; i++) {
|
||||
if (first && (linecount < node.skip)) {
|
||||
if (line[i] === "\n") { linecount += 1; }
|
||||
continue;
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
if ((node.hdrin === true) && first) { // if the template is in the first line
|
||||
if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break
|
||||
if (line.length - i === 1) { tmp += line[i]; }
|
||||
template = clean(tmp,node.sep);
|
||||
first = false;
|
||||
}
|
||||
else { tmp += line[i]; }
|
||||
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
a.push(o); // add to the array
|
||||
}
|
||||
else {
|
||||
if (line[i] === node.quo) { // if it's a quote toggle inside or outside
|
||||
f = !f;
|
||||
if (line[i-1] === node.quo) {
|
||||
if (f === false) { k[j] += '\"'; }
|
||||
} // if it's a quotequote then it's actually a quote
|
||||
//if ((line[i-1] !== node.sep) && (line[i+1] !== node.sep)) { k[j] += line[i]; }
|
||||
}
|
||||
else if ((line[i] === node.sep) && f) { // if it is the end of the line then finish
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
// if no value between separators ('1,,"3"...') or if the line beings with separator (',1,"2"...') treat value as null
|
||||
if (line[i-1] === node.sep || line[i-1].includes('\n','\r')) k[j] = null;
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
j += 1;
|
||||
// if separator is last char in processing string line (without end of line), add null value at the end - example: '1,2,3\n3,"3",'
|
||||
k[j] = line.length - 1 === i ? null : "";
|
||||
}
|
||||
else if (((line[i] === "\n") || (line[i] === "\r")) && f) { // handle multiple lines
|
||||
//console.log(j,k,o,k[j]);
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
// if separator before end of line, set null value ie. '1,2,"3"\n1,2,\n1,2,3'
|
||||
if (line[i-1] === node.sep) k[j] = null;
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
a.push(o); // add to the array
|
||||
}
|
||||
j = 0;
|
||||
k = [""];
|
||||
o = {};
|
||||
f = true; // reset in/out flag ready for next line.
|
||||
}
|
||||
else { // just add to the part of the message
|
||||
k[j] += line[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
// Finished so finalize and send anything left
|
||||
if (f === false) { node.warn(RED._("csv.errors.bad_csv")); }
|
||||
if (!node.goodtmpl) { template[j] = "col"+(j+1); }
|
||||
|
||||
if ( template[j] && (template[j] !== "") ) {
|
||||
if ( (k[j] !== null && node.parsestrings === true) && reg.test(k[j].trim()) ) { k[j] = parseFloat(k[j].trim()); }
|
||||
else { if (k[j] !== null) k[j].replace(/\r$/,''); }
|
||||
if (node.include_null_values && k[j] === null) o[template[j]] = k[j];
|
||||
if (node.include_empty_strings && k[j] === "") o[template[j]] = k[j];
|
||||
if (k[j] !== null && k[j] !== "") o[template[j]] = k[j];
|
||||
}
|
||||
|
||||
if (JSON.stringify(o) !== "{}") { // don't send empty objects
|
||||
a.push(o); // add to the array
|
||||
}
|
||||
|
||||
if (node.multi !== "one") {
|
||||
msg.payload = a;
|
||||
if (has_parts && nocr <= 1) {
|
||||
if (JSON.stringify(o) !== "{}") {
|
||||
node.store.push(o);
|
||||
if (node.multi !== "one") {
|
||||
msg.payload = a;
|
||||
if (has_parts && nocr <= 1) {
|
||||
if (JSON.stringify(o) !== "{}") {
|
||||
node.store.push(o);
|
||||
}
|
||||
if (msg.parts.index + 1 === msg.parts.count) {
|
||||
msg.payload = node.store;
|
||||
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
|
||||
delete msg.parts;
|
||||
send(msg);
|
||||
node.store = [];
|
||||
}
|
||||
}
|
||||
if (msg.parts.index + 1 === msg.parts.count) {
|
||||
msg.payload = node.store;
|
||||
else {
|
||||
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
|
||||
delete msg.parts;
|
||||
send(msg);
|
||||
node.store = [];
|
||||
send(msg); // finally send the array
|
||||
}
|
||||
}
|
||||
else {
|
||||
msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
|
||||
send(msg); // finally send the array
|
||||
}
|
||||
}
|
||||
else {
|
||||
var len = a.length;
|
||||
for (var i = 0; i < len; i++) {
|
||||
var newMessage = RED.util.cloneMessage(msg);
|
||||
newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
|
||||
newMessage.payload = a[i];
|
||||
if (!has_parts) {
|
||||
newMessage.parts = {
|
||||
id: msg._msgid,
|
||||
index: i,
|
||||
count: len
|
||||
};
|
||||
var len = a.length;
|
||||
for (var i = 0; i < len; i++) {
|
||||
var newMessage = RED.util.cloneMessage(msg);
|
||||
newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(',');
|
||||
newMessage.payload = a[i];
|
||||
if (!has_parts) {
|
||||
newMessage.parts = {
|
||||
id: msg._msgid,
|
||||
index: i,
|
||||
count: len
|
||||
};
|
||||
}
|
||||
else {
|
||||
newMessage.parts.index -= node.skip;
|
||||
newMessage.parts.count -= node.skip;
|
||||
if (node.hdrin) { // if we removed the header line then shift the counts by 1
|
||||
newMessage.parts.index -= 1;
|
||||
newMessage.parts.count -= 1;
|
||||
}
|
||||
}
|
||||
if (last) { newMessage.complete = true; }
|
||||
send(newMessage);
|
||||
}
|
||||
else {
|
||||
newMessage.parts.index -= node.skip;
|
||||
newMessage.parts.count -= node.skip;
|
||||
if (node.hdrin) { // if we removed the header line then shift the counts by 1
|
||||
newMessage.parts.index -= 1;
|
||||
newMessage.parts.count -= 1;
|
||||
if (has_parts && last && len === 0) {
|
||||
send({complete:true});
|
||||
}
|
||||
}
|
||||
node.linecount = 0;
|
||||
done();
|
||||
}
|
||||
catch(e) { done(e); }
|
||||
}
|
||||
else { node.warn(RED._("csv.errors.csv_js")); done(); }
|
||||
}
|
||||
else {
|
||||
if (!msg.hasOwnProperty("reset")) {
|
||||
node.send(msg); // If no payload and not reset - just pass it on.
|
||||
}
|
||||
done();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if(RFC4180Mode) {
|
||||
node.template = (n.temp || "")
|
||||
node.sep = (n.sep || ',').replace(/\\t/g, "\t").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
|
||||
node.quo = '"'
|
||||
// default to CRLF (RFC4180 Sec 2.1: "Each record is located on a separate line, delimited by a line break (CRLF)")
|
||||
node.ret = (n.ret || "\r\n").replace(/\\n/g, "\n").replace(/\\r/g, "\r")
|
||||
node.multi = n.multi || "one"
|
||||
node.hdrin = n.hdrin || false
|
||||
node.hdrout = n.hdrout || "none"
|
||||
node.goodtmpl = true
|
||||
node.skip = parseInt(n.skip || 0)
|
||||
node.store = []
|
||||
node.parsestrings = n.strings
|
||||
node.include_empty_strings = n.include_empty_strings || false
|
||||
node.include_null_values = n.include_null_values || false
|
||||
if (node.parsestrings === undefined) { node.parsestrings = true }
|
||||
if (node.hdrout === false) { node.hdrout = "none" }
|
||||
if (node.hdrout === true) { node.hdrout = "all" }
|
||||
const dontSendHeaders = node.hdrout === "none"
|
||||
const sendHeadersOnce = node.hdrout === "once"
|
||||
const sendHeadersAlways = node.hdrout === "all"
|
||||
const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
|
||||
const quoteables = [node.sep, node.quo, "\n", "\r"]
|
||||
const templateQuoteables = [',', '"', "\n", "\r"]
|
||||
let badTemplateWarnOnce = true
|
||||
|
||||
const columnStringToTemplateArray = function (col, sep) {
|
||||
// NOTE: enforce strict column template parsing in RFC4180 mode
|
||||
const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
|
||||
if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false }
|
||||
return parsed.headers.length ? parsed.headers : null
|
||||
}
|
||||
const templateArrayToColumnString = function (template, keepEmptyColumns) {
|
||||
// NOTE: enforce strict column template parsing in RFC4180 mode
|
||||
const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true })
|
||||
return keepEmptyColumns
|
||||
? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables}))
|
||||
: parsed.header // exclues empty columns
|
||||
// TODO: resolve inconsistency between CSV->JSON and JSON->CSV
|
||||
// CSV->JSON: empty columns are excluded
|
||||
// JSON->CSV: empty columns are kept in some cases
|
||||
}
|
||||
function addQuotes(cell, options) {
|
||||
options = options || {}
|
||||
return csv.quoteCell(cell, {
|
||||
quote: options.quote || node.quo || '"',
|
||||
separator: options.separator || node.sep || ',',
|
||||
quoteables: options.quoteables || quoteables
|
||||
})
|
||||
}
|
||||
const hasTemplate = (t) => t?.length > 0 && !(t.length === 1 && t[0] === '')
|
||||
let template
|
||||
try {
|
||||
template = columnStringToTemplateArray(node.template, ',') || ['']
|
||||
} catch (e) {
|
||||
node.warn(RED._("csv.errors.bad_template")) // is warning really necessary now we have status?
|
||||
node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
|
||||
return // dont hook up the node
|
||||
}
|
||||
const noTemplate = hasTemplate(template) === false
|
||||
node.hdrSent = false
|
||||
|
||||
node.on("input", function (msg, send, done) {
|
||||
node.status({}) // clear status
|
||||
if (msg.hasOwnProperty("reset")) {
|
||||
node.hdrSent = false
|
||||
}
|
||||
if (msg.hasOwnProperty("payload")) {
|
||||
let inputData = msg.payload
|
||||
if (typeof inputData == "object") { // convert object to CSV string
|
||||
try {
|
||||
// first determine the payload kind. Array or objects? Array of primitives? Array of arrays? Just an object?
|
||||
// then, if necessary, convert to an array of objects/arrays
|
||||
let isObject = !Array.isArray(inputData) && typeof inputData === 'object'
|
||||
let isArrayOfObjects = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] === 'object'
|
||||
let isArrayOfArrays = Array.isArray(inputData) && inputData.length > 0 && Array.isArray(inputData[0])
|
||||
let isArrayOfPrimitives = Array.isArray(inputData) && inputData.length > 0 && typeof inputData[0] !== 'object'
|
||||
|
||||
if (isObject) {
|
||||
inputData = [inputData]
|
||||
isArrayOfObjects = true
|
||||
isObject = false
|
||||
} else if (isArrayOfPrimitives) {
|
||||
inputData = [inputData]
|
||||
isArrayOfArrays = true
|
||||
isArrayOfPrimitives = false
|
||||
}
|
||||
|
||||
const stringBuilder = []
|
||||
if (!(noTemplate && (msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("index") && msg.parts.index > 0))) {
|
||||
template = columnStringToTemplateArray(node.template) || ['']
|
||||
}
|
||||
|
||||
// build header line
|
||||
if (sendHeaders && node.hdrSent === false) {
|
||||
if (hasTemplate(template) === false) {
|
||||
if (msg.hasOwnProperty("columns")) {
|
||||
template = columnStringToTemplateArray(msg.columns || "", ",") || ['']
|
||||
}
|
||||
else {
|
||||
template = Object.keys(inputData[0]) || ['']
|
||||
}
|
||||
}
|
||||
if (last) { newMessage.complete = true; }
|
||||
send(newMessage);
|
||||
stringBuilder.push(templateArrayToColumnString(template, true))
|
||||
if (sendHeadersOnce) { node.hdrSent = true }
|
||||
}
|
||||
if (has_parts && last && len === 0) {
|
||||
send({complete:true});
|
||||
|
||||
// build csv lines
|
||||
for (let s = 0; s < inputData.length; s++) {
|
||||
let row = inputData[s]
|
||||
if (isArrayOfArrays) {
|
||||
/*** row is an array of arrays ***/
|
||||
const _hasTemplate = hasTemplate(template)
|
||||
const len = _hasTemplate ? template.length : row.length
|
||||
const result = []
|
||||
for (let t = 0; t < len; t++) {
|
||||
let cell = row[t]
|
||||
if (cell === undefined) { cell = "" }
|
||||
if(_hasTemplate) {
|
||||
const header = template[t]
|
||||
if (header) {
|
||||
result[t] = addQuotes(RED.util.ensureString(cell))
|
||||
}
|
||||
} else {
|
||||
result[t] = addQuotes(RED.util.ensureString(cell))
|
||||
}
|
||||
}
|
||||
stringBuilder.push(result.join(node.sep))
|
||||
} else {
|
||||
/*** row is an object ***/
|
||||
if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) {
|
||||
template = columnStringToTemplateArray(msg.columns || "", ",")
|
||||
}
|
||||
if (hasTemplate(template) === false) {
|
||||
/*** row is an object but we still don't have a template ***/
|
||||
if (badTemplateWarnOnce === true) {
|
||||
node.warn(RED._("csv.errors.obj_csv"))
|
||||
badTemplateWarnOnce = false
|
||||
}
|
||||
const rowData = []
|
||||
for (let header in inputData[0]) {
|
||||
if (row.hasOwnProperty(header)) {
|
||||
const cell = row[header]
|
||||
if (typeof cell !== "object") {
|
||||
let cellValue = ""
|
||||
if (cell !== undefined) {
|
||||
cellValue += cell
|
||||
}
|
||||
rowData.push(addQuotes(cellValue))
|
||||
}
|
||||
}
|
||||
}
|
||||
stringBuilder.push(rowData.join(node.sep))
|
||||
} else {
|
||||
/*** row is an object and we have a template ***/
|
||||
const rowData = []
|
||||
for (let t = 0; t < template.length; t++) {
|
||||
if (!template[t]) {
|
||||
rowData.push('')
|
||||
}
|
||||
else {
|
||||
let cellValue = inputData[s][template[t]]
|
||||
if (cellValue === undefined) { cellValue = "" }
|
||||
cellValue = RED.util.ensureString(cellValue)
|
||||
rowData.push(addQuotes(cellValue))
|
||||
}
|
||||
}
|
||||
stringBuilder.push(rowData.join(node.sep)); // add separator
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// join lines, don't forget to add the last new line
|
||||
msg.payload = stringBuilder.join(node.ret) + node.ret
|
||||
msg.columns = templateArrayToColumnString(template)
|
||||
if (msg.payload !== '') { send(msg) }
|
||||
done()
|
||||
}
|
||||
catch (e) {
|
||||
done(e)
|
||||
}
|
||||
node.linecount = 0;
|
||||
done();
|
||||
}
|
||||
catch(e) { done(e); }
|
||||
else if (typeof inputData == "string") { // convert CSV string to object
|
||||
try {
|
||||
let firstLine = true; // is this the first line
|
||||
let last = false
|
||||
let linecount = 0
|
||||
const has_parts = msg.hasOwnProperty("parts")
|
||||
|
||||
// determine if this is a multi part message and if so what part we are processing
|
||||
if (msg.hasOwnProperty("parts")) {
|
||||
linecount = msg.parts.index
|
||||
if (msg.parts.index > node.skip) { firstLine = false }
|
||||
if (msg.parts.hasOwnProperty("count") && (msg.parts.index + 1 >= msg.parts.count)) { last = true }
|
||||
}
|
||||
|
||||
// If skip is set, compute the cursor position to start parsing from
|
||||
let _cursor = 0
|
||||
if (node.skip > 0 && linecount < node.skip) {
|
||||
for (; _cursor < inputData.length; _cursor++) {
|
||||
if (firstLine && (linecount < node.skip)) {
|
||||
if (inputData[_cursor] === "\r" || inputData[_cursor] === "\n") {
|
||||
linecount += 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if (_cursor >= inputData.length) {
|
||||
return // skip this line
|
||||
}
|
||||
}
|
||||
|
||||
// count the number of line breaks in the string
|
||||
const noofCR = ((_cursor ? inputData.slice(_cursor) : inputData).match(/[\r\n]/g) || []).length
|
||||
|
||||
// if we have `parts` and we are outputting multiple objects and we have more than one line
|
||||
// then we need to set firstLine to true so that we process the header line
|
||||
if (has_parts && node.multi === "mult" && noofCR > 1) {
|
||||
firstLine = true
|
||||
}
|
||||
|
||||
// if we are processing the first line and the node has been set to extract the header line
|
||||
// update the template with the header line
|
||||
if (firstLine && node.hdrin === true) {
|
||||
/** @type {import('./lib/csv/index.js').CSVParseOptions} */
|
||||
const csvOptionsForHeaderRow = {
|
||||
cursor: _cursor,
|
||||
separator: node.sep,
|
||||
quote: node.quo,
|
||||
dataHasHeaderRow: true,
|
||||
headersOnly: true,
|
||||
outputStyle: 'array',
|
||||
strict: true // enforce strict parsing of the header row
|
||||
}
|
||||
try {
|
||||
const csvHeader = csv.parse(inputData, csvOptionsForHeaderRow)
|
||||
template = csvHeader.headers
|
||||
_cursor = csvHeader.cursor
|
||||
} catch (e) {
|
||||
// node.warn(RED._("csv.errors.bad_template")) // add warning?
|
||||
node.status({ fill: "red", shape: "dot", text: RED._("csv.errors.bad_template") })
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// now we process the data lines
|
||||
/** @type {import('./lib/csv/index.js').CSVParseOptions} */
|
||||
const csvOptions = {
|
||||
cursor: _cursor,
|
||||
separator: node.sep,
|
||||
quote: node.quo,
|
||||
dataHasHeaderRow: false,
|
||||
headers: hasTemplate(template) ? template : null,
|
||||
outputStyle: 'object',
|
||||
includeNullValues: node.include_null_values,
|
||||
includeEmptyStrings: node.include_empty_strings,
|
||||
parseNumeric: node.parsestrings,
|
||||
strict: false // relax the strictness of the parser for data rows
|
||||
}
|
||||
const csvParseResult = csv.parse(inputData, csvOptions)
|
||||
const data = csvParseResult.data
|
||||
|
||||
// output results
|
||||
if (node.multi !== "one") {
|
||||
if (has_parts && noofCR <= 1) {
|
||||
if (data.length > 0) {
|
||||
node.store.push(...data)
|
||||
}
|
||||
if (msg.parts.index + 1 === msg.parts.count) {
|
||||
msg.payload = node.store
|
||||
msg.columns = csvParseResult.header
|
||||
// msg._mode = 'RFC4180 mode'
|
||||
delete msg.parts
|
||||
send(msg)
|
||||
node.store = []
|
||||
}
|
||||
}
|
||||
else {
|
||||
msg.columns = csvParseResult.header
|
||||
// msg._mode = 'RFC4180 mode'
|
||||
msg.payload = data
|
||||
send(msg); // finally send the array
|
||||
}
|
||||
}
|
||||
else {
|
||||
const len = data.length
|
||||
for (let row = 0; row < len; row++) {
|
||||
const newMessage = RED.util.cloneMessage(msg)
|
||||
newMessage.columns = csvParseResult.header
|
||||
newMessage.payload = data[row]
|
||||
if (!has_parts) {
|
||||
newMessage.parts = {
|
||||
id: msg._msgid,
|
||||
index: row,
|
||||
count: len
|
||||
}
|
||||
}
|
||||
else {
|
||||
newMessage.parts.index -= node.skip
|
||||
newMessage.parts.count -= node.skip
|
||||
if (node.hdrin) { // if we removed the header line then shift the counts by 1
|
||||
newMessage.parts.index -= 1
|
||||
newMessage.parts.count -= 1
|
||||
}
|
||||
}
|
||||
if (last) { newMessage.complete = true }
|
||||
// newMessage._mode = 'RFC4180 mode'
|
||||
send(newMessage)
|
||||
}
|
||||
if (has_parts && last && len === 0) {
|
||||
// send({complete:true, _mode: 'RFC4180 mode'})
|
||||
send({ complete: true })
|
||||
}
|
||||
}
|
||||
|
||||
node.linecount = 0
|
||||
done()
|
||||
}
|
||||
catch (e) {
|
||||
done(e)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// RFC-vs-legacy mode difference: In RFC mode, we throw catchable errors and provide a status message
|
||||
const err = new Error(RED._("csv.errors.csv_js"))
|
||||
node.status({ fill: "red", shape: "dot", text: err.message })
|
||||
done(err)
|
||||
}
|
||||
}
|
||||
else { node.warn(RED._("csv.errors.csv_js")); done(); }
|
||||
}
|
||||
else {
|
||||
if (!msg.hasOwnProperty("reset")) {
|
||||
node.send(msg); // If no payload and not reset - just pass it on.
|
||||
else {
|
||||
if (!msg.hasOwnProperty("reset")) {
|
||||
node.send(msg); // If no payload and not reset - just pass it on.
|
||||
}
|
||||
done()
|
||||
}
|
||||
done();
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
}
|
||||
RED.nodes.registerType("csv",CSVNode);
|
||||
|
||||
RED.nodes.registerType("csv",CSVNode)
|
||||
}
|
||||
|
||||
@@ -41,8 +41,8 @@
|
||||
color:"#DEBD5C",
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
property: {value:"payload"},
|
||||
outproperty: {value:"payload"},
|
||||
property: {value:"payload", validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true }) },
|
||||
outproperty: {value:"payload", validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true }) },
|
||||
tag: {value:""},
|
||||
ret: {value:"html"},
|
||||
as: {value:"single"}
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
property: {value:"payload",required:true,
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true}),
|
||||
label:RED._("node-red:json.label.property")},
|
||||
action: {value:""},
|
||||
pretty: {value:false}
|
||||
|
||||
@@ -27,7 +27,8 @@
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
property: {value:"payload",required:true,
|
||||
label:RED._("node-red:common.label.property")},
|
||||
label:RED._("node-red:common.label.property"),
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true })},
|
||||
attr: {value:""},
|
||||
chr: {value:""}
|
||||
},
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
color:"#DEBD5C",
|
||||
defaults: {
|
||||
property: {value:"payload",required:true,
|
||||
validate: RED.validators.typedInput({ type: 'msg', allowUndefined: true }),
|
||||
label:RED._("node-red:common.label.property")},
|
||||
name: {value:""}
|
||||
},
|
||||
|
||||
324
packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js
vendored
Normal file
324
packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js
vendored
Normal file
@@ -0,0 +1,324 @@
|
||||
|
||||
/**
|
||||
* @typedef {Object} CSVParseOptions
|
||||
* @property {number} [cursor=0] - an index into the CSV to start parsing from
|
||||
* @property {string} [separator=','] - the separator character
|
||||
* @property {string} [quote='"'] - the quote character
|
||||
* @property {boolean} [headersOnly=false] - only parse the headers and return them
|
||||
* @property {string[]} [headers=[]] - an array of headers to use instead of the first row of the CSV data
|
||||
* @property {boolean} [dataHasHeaderRow=true] - whether the CSV data to parse has a header row
|
||||
* @property {boolean} [outputHeader=true] - whether the output data should include a header row (only applies to array output)
|
||||
* @property {boolean} [parseNumeric=false] - parse numeric values into numbers
|
||||
* @property {boolean} [includeNullValues=false] - include null values in the output
|
||||
* @property {boolean} [includeEmptyStrings=true] - include empty strings in the output
|
||||
* @property {string} [outputStyle='object'] - output an array of arrays or an array of objects
|
||||
* @property {boolean} [strict=false] - throw an error if the CSV is malformed
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parses a CSV string into an array of arrays or an array of objects.
|
||||
*
|
||||
* NOTES:
|
||||
* * Deviations from the RFC4180 spec (for the sake of user fiendliness, system implementations and flexibility), this parser will:
|
||||
* * accept any separator character, not just `,`
|
||||
* * accept any quote character, not just `"`
|
||||
* * parse `\r`, `\n` or `\r\n` as line endings (RRFC4180 2.1 states lines are separated by CRLF)
|
||||
* * Only single character `quote` is supported
|
||||
* * `quote` is `"` by default
|
||||
* * Any cell that contains a `quote` or `separator` will be quoted
|
||||
* * Any `quote` characters inside a cell will be escaped as per RFC 4180 2.6
|
||||
* * Only single character `separator` is supported
|
||||
* * Only `array` and `object` output styles are supported
|
||||
* * `array` output style is an array of arrays [[],[],[]]
|
||||
* * `object` output style is an array of objects [{},{},{}]
|
||||
* * Only `headers` or `dataHasHeaderRow` are supported, not both
|
||||
* @param {string} csvIn - the CSV string to parse
|
||||
* @param {CSVParseOptions} parseOptions - options
|
||||
* @throws {Error}
|
||||
*/
|
||||
function parse(csvIn, parseOptions) {
|
||||
/* Normalise options */
|
||||
parseOptions = parseOptions || {};
|
||||
const separator = parseOptions.separator ?? ',';
|
||||
const quote = parseOptions.quote ?? '"';
|
||||
const headersOnly = parseOptions.headersOnly ?? false;
|
||||
const headers = Array.isArray(parseOptions.headers) ? parseOptions.headers : []
|
||||
const dataHasHeaderRow = parseOptions.dataHasHeaderRow ?? true;
|
||||
const outputHeader = parseOptions.outputHeader ?? true;
|
||||
const parseNumeric = parseOptions.parseNumeric ?? false;
|
||||
const includeNullValues = parseOptions.includeNullValues ?? false;
|
||||
const includeEmptyStrings = parseOptions.includeEmptyStrings ?? true;
|
||||
const outputStyle = ['array', 'object'].includes(parseOptions.outputStyle) ? parseOptions.outputStyle : 'object'; // 'array [[],[],[]]' or 'object [{},{},{}]
|
||||
const strict = parseOptions.strict ?? false
|
||||
|
||||
/* Local variables */
|
||||
const cursorMax = csvIn.length;
|
||||
const ouputArrays = outputStyle === 'array';
|
||||
const headersSupplied = headers.length > 0
|
||||
// The original regex was an "is-a-number" positive logic test. /^ *[-]?(?!E)(?!0\d)\d*\.?\d*(E-?\+?)?\d+ *$/i;
|
||||
// Below, is less strict and inverted logic but coupled with +cast it is 13%+ faster than original regex+parsefloat
|
||||
// and has the benefit of understanding hexadecimals, binary and octal numbers.
|
||||
const skipNumberConversion = /^ *(\+|-0\d|0\d)/
|
||||
const cellBuilder = []
|
||||
let rowBuilder = []
|
||||
let cursor = typeof parseOptions.cursor === 'number' ? parseOptions.cursor : 0;
|
||||
let newCell = true, inQuote = false, closed = false, output = [];
|
||||
|
||||
/* inline helper functions */
|
||||
const finaliseCell = () => {
|
||||
let cell = cellBuilder.join('')
|
||||
cellBuilder.length = 0
|
||||
// push the cell:
|
||||
// NOTE: if cell is empty but newCell==true, then this cell had zero chars - push `null`
|
||||
// otherwise push empty string
|
||||
return rowBuilder.push(cell || (newCell ? null : ''))
|
||||
}
|
||||
const finaliseRow = () => {
|
||||
if (cellBuilder.length) {
|
||||
finaliseCell()
|
||||
}
|
||||
if (rowBuilder.length) {
|
||||
output.push(rowBuilder)
|
||||
rowBuilder = []
|
||||
}
|
||||
}
|
||||
|
||||
/* Main parsing loop */
|
||||
while (cursor < cursorMax) {
|
||||
const char = csvIn[cursor]
|
||||
if (inQuote) {
|
||||
if (char === quote && csvIn[cursor + 1] === quote) {
|
||||
cellBuilder.push(quote)
|
||||
cursor += 2;
|
||||
newCell = false;
|
||||
closed = false;
|
||||
} else if (char === quote) {
|
||||
inQuote = false;
|
||||
cursor += 1;
|
||||
newCell = false;
|
||||
closed = true;
|
||||
} else {
|
||||
cellBuilder.push(char)
|
||||
newCell = false;
|
||||
closed = false;
|
||||
cursor++;
|
||||
}
|
||||
} else {
|
||||
if (char === separator) {
|
||||
finaliseCell()
|
||||
cursor += 1;
|
||||
newCell = true;
|
||||
closed = false;
|
||||
} else if (char === quote) {
|
||||
if (newCell) {
|
||||
inQuote = true;
|
||||
cursor += 1;
|
||||
newCell = false;
|
||||
closed = false;
|
||||
}
|
||||
else if (strict) {
|
||||
throw new UnquotedQuoteError(cursor)
|
||||
} else {
|
||||
// not strict, keep 1 quote if the next char is not a cell/record separator
|
||||
cursor++
|
||||
if (csvIn[cursor] && csvIn[cursor] !== '\n' && csvIn[cursor] !== '\r' && csvIn[cursor] !== separator) {
|
||||
cellBuilder.push(char)
|
||||
if (csvIn[cursor] === quote) {
|
||||
cursor++ // skip the next quote
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (char === '\n' || char === '\r') {
|
||||
finaliseRow()
|
||||
if (csvIn[cursor + 1] === '\n') {
|
||||
cursor += 2;
|
||||
} else {
|
||||
cursor++
|
||||
}
|
||||
newCell = true;
|
||||
closed = false;
|
||||
if (headersOnly) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
if (closed) {
|
||||
if (strict) {
|
||||
throw new DataAfterCloseError(cursor)
|
||||
} else {
|
||||
cursor--; // move back to grab the previously discarded char
|
||||
closed = false
|
||||
}
|
||||
} else {
|
||||
cellBuilder.push(char)
|
||||
newCell = false;
|
||||
cursor++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (strict && inQuote) {
|
||||
throw new ParseError(`Missing quote, unclosed cell`, cursor)
|
||||
}
|
||||
// finalise the last cell/row
|
||||
finaliseRow()
|
||||
let firstRowIsHeader = false
|
||||
// if no headers supplied, generate them
|
||||
if (output.length >= 1) {
|
||||
if (headersSupplied) {
|
||||
// headers already supplied
|
||||
} else if (dataHasHeaderRow) {
|
||||
// take the first row as the headers
|
||||
headers.push(...output[0])
|
||||
firstRowIsHeader = true
|
||||
} else {
|
||||
// generate headers col1, col2, col3, etc
|
||||
for (let i = 0; i < output[0].length; i++) {
|
||||
headers.push("col" + (i + 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const finalResult = {
|
||||
/** @type {String[]} headers as an array of string */
|
||||
headers: headers,
|
||||
/** @type {String} headers as a comma-separated string */
|
||||
header: null,
|
||||
/** @type {Any[]} Result Data (may include header row: check `firstRowIsHeader` flag) */
|
||||
data: [],
|
||||
/** @type {Boolean|undefined} flag to indicate if the first row is a header row (only applies when `outputStyle` is 'array') */
|
||||
firstRowIsHeader: undefined,
|
||||
/** @type {'array'|'object'} flag to indicate the output style */
|
||||
outputStyle: outputStyle,
|
||||
/** @type {Number} The current cursor position */
|
||||
cursor: cursor,
|
||||
}
|
||||
|
||||
const quotedHeaders = []
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
if (!headers[i]) {
|
||||
continue
|
||||
}
|
||||
quotedHeaders.push(quoteCell(headers[i], { quote, separator: ',' }))
|
||||
}
|
||||
finalResult.header = quotedHeaders.join(',') // always quote headers and join with comma
|
||||
|
||||
// output is an array of arrays [[],[],[]]
|
||||
if (ouputArrays || headersOnly) {
|
||||
if (!firstRowIsHeader && !headersOnly && outputHeader && headers.length > 0) {
|
||||
if (output.length > 0) {
|
||||
output.unshift(headers)
|
||||
} else {
|
||||
output = [headers]
|
||||
}
|
||||
firstRowIsHeader = true
|
||||
}
|
||||
if (headersOnly) {
|
||||
delete finalResult.firstRowIsHeader
|
||||
return finalResult
|
||||
}
|
||||
finalResult.firstRowIsHeader = firstRowIsHeader
|
||||
finalResult.data = (firstRowIsHeader && !outputHeader) ? output.slice(1) : output
|
||||
return finalResult
|
||||
}
|
||||
|
||||
// output is an array of objects [{},{},{}]
|
||||
const outputObjects = []
|
||||
let i = firstRowIsHeader ? 1 : 0
|
||||
for (; i < output.length; i++) {
|
||||
const rowObject = {}
|
||||
let isEmpty = true
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
if (!headers[j]) {
|
||||
continue
|
||||
}
|
||||
let v = output[i][j] === undefined ? null : output[i][j]
|
||||
if (v === null && !includeNullValues) {
|
||||
continue
|
||||
} else if (v === "" && !includeEmptyStrings) {
|
||||
continue
|
||||
} else if (parseNumeric === true && v && !skipNumberConversion.test(v)) {
|
||||
const vTemp = +v
|
||||
const isNumber = !isNaN(vTemp)
|
||||
if(isNumber) {
|
||||
v = vTemp
|
||||
}
|
||||
}
|
||||
rowObject[headers[j]] = v
|
||||
isEmpty = false
|
||||
}
|
||||
// determine if this row is empty
|
||||
if (!isEmpty) {
|
||||
outputObjects.push(rowObject)
|
||||
}
|
||||
}
|
||||
finalResult.data = outputObjects
|
||||
delete finalResult.firstRowIsHeader
|
||||
return finalResult
|
||||
}
|
||||
|
||||
/**
|
||||
* Quotes a cell in a CSV string if necessary. Addiionally, any double quotes inside the cell will be escaped as per RFC 4180 2.6 (https://datatracker.ietf.org/doc/html/rfc4180#section-2).
|
||||
* @param {string} cell - the string to quote
|
||||
* @param {*} options - options
|
||||
* @param {string} [options.quote='"'] - the quote character
|
||||
* @param {string} [options.separator=','] - the separator character
|
||||
* @param {string[]} [options.quoteables] - an array of characters that, when encountered, will trigger the application of outer quotes
|
||||
* @returns
|
||||
*/
|
||||
function quoteCell(cell, { quote = '"', separator = ",", quoteables } = {
|
||||
quote: '"',
|
||||
separator: ",",
|
||||
quoteables: [quote, separator, '\r', '\n']
|
||||
}) {
|
||||
quoteables = quoteables || [quote, separator, '\r', '\n'];
|
||||
|
||||
let doubleUp = false;
|
||||
if (cell.indexOf(quote) !== -1) { // add double quotes if any quotes
|
||||
doubleUp = true;
|
||||
}
|
||||
const quoteChar = quoteables.some(q => cell.includes(q)) ? quote : '';
|
||||
return quoteChar + (doubleUp ? cell.replace(/"/g, '""') : cell) + quoteChar;
|
||||
}
|
||||
|
||||
// #region Custom Error Classes
|
||||
class ParseError extends Error {
|
||||
/**
|
||||
* @param {string} message - the error message
|
||||
* @param {number} cursor - the cursor index where the error occurred
|
||||
*/
|
||||
constructor(message, cursor) {
|
||||
super(message)
|
||||
this.name = 'ParseError'
|
||||
this.cursor = cursor
|
||||
}
|
||||
}
|
||||
|
||||
class UnquotedQuoteError extends ParseError {
|
||||
/**
|
||||
* @param {number} cursor - the cursor index where the error occurred
|
||||
*/
|
||||
constructor(cursor) {
|
||||
super('Quote found in the middle of an unquoted field', cursor)
|
||||
this.name = 'UnquotedQuoteError'
|
||||
}
|
||||
}
|
||||
|
||||
class DataAfterCloseError extends ParseError {
|
||||
/**
|
||||
* @param {number} cursor - the cursor index where the error occurred
|
||||
*/
|
||||
constructor(cursor) {
|
||||
super('Data found after closing quote', cursor)
|
||||
this.name = 'DataAfterCloseError'
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
exports.parse = parse
|
||||
exports.quoteCell = quoteCell
|
||||
exports.ParseError = ParseError
|
||||
exports.UnquotedQuoteError = UnquotedQuoteError
|
||||
exports.DataAfterCloseError = DataAfterCloseError
|
||||
@@ -60,7 +60,7 @@
|
||||
arraySplt: {value:1},
|
||||
arraySpltType: {value:"len"},
|
||||
stream: {value:false},
|
||||
addname: {value:""},
|
||||
addname: {value:"", validate: RED.validators.typedInput({ type: 'msg', allowBlank: true })},
|
||||
property: {value:"payload",required:true}
|
||||
},
|
||||
inputs:1,
|
||||
@@ -216,7 +216,22 @@
|
||||
validate:RED.validators.typedInput("propertyType", false)
|
||||
},
|
||||
propertyType: { value:"msg"},
|
||||
key: {value:"topic"},
|
||||
key: {value:"topic", validate: (function () {
|
||||
const typeValidator = RED.validators.typedInput({ type: 'msg' })
|
||||
return function(v, opt) {
|
||||
const joinMode = $("#node-input-mode").val() || this.mode
|
||||
if (joinMode !== 'custom') {
|
||||
return true
|
||||
}
|
||||
const buildType = $("#node-input-build").val() || this.build
|
||||
if (buildType !== 'object') {
|
||||
return true
|
||||
} else {
|
||||
return typeValidator(v, opt)
|
||||
}
|
||||
}
|
||||
})()
|
||||
},
|
||||
joiner: { value:"\\n"},
|
||||
joinerType: { value:"str"},
|
||||
accumulate: { value:"false" },
|
||||
|
||||
@@ -198,7 +198,7 @@
|
||||
category: 'storage',
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
filename: {value:""},
|
||||
filename: {value:"", validate: RED.validators.typedInput({ typeField: 'filenameType' })},
|
||||
filenameType: {value:"str"},
|
||||
appendNewline: {value:true},
|
||||
createDir: {value:false},
|
||||
@@ -297,7 +297,7 @@
|
||||
category: 'storage',
|
||||
defaults: {
|
||||
name: {value:""},
|
||||
filename: {value:""},
|
||||
filename: {value:"", validate: RED.validators.typedInput({ typeField: 'filenameType' }) },
|
||||
filenameType: {value:"str"},
|
||||
format: {value:"utf8"},
|
||||
chunk: {value:false},
|
||||
|
||||
Reference in New Issue
Block a user