Allow nested msg properties in msg/flow/global expressions (#2822)

* Allow nested msg properties in msg/flow/global expressions

* Remove typo in RED.utils

Co-authored-by: Nick O'Leary <knolleary@users.noreply.github.com>
This commit is contained in:
Nick O'Leary 2021-01-27 20:32:52 +00:00 committed by GitHub
parent 34ef055d7b
commit 438d51d26e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 342 additions and 31 deletions

View File

@ -615,18 +615,25 @@ RED.utils = (function() {
return element;
}
function normalisePropertyExpression(str) {
function createError(code, message) {
var e = new Error(message);
e.code = code;
return e;
}
function normalisePropertyExpression(str,msg) {
// This must be kept in sync with validatePropertyExpression
// in editor/js/ui/utils.js
var length = str.length;
if (length === 0) {
throw new Error("Invalid property expression: zero-length");
throw createError("INVALID_EXPR","Invalid property expression: zero-length");
}
var parts = [];
var start = 0;
var inString = false;
var inBox = false;
var boxExpression = false;
var quoteChar;
var v;
for (var i=0;i<length;i++) {
@ -634,14 +641,14 @@ RED.utils = (function() {
if (!inString) {
if (c === "'" || c === '"') {
if (i != start) {
throw new Error("Invalid property expression: unexpected "+c+" at position "+i);
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+c+" at position "+i);
}
inString = true;
quoteChar = c;
start = i+1;
} else if (c === '.') {
if (i===0) {
throw new Error("Invalid property expression: unexpected . at position 0");
throw createError("INVALID_EXPR","Invalid property expression: unexpected . at position 0");
}
if (start != i) {
v = str.substring(start,i);
@ -652,57 +659,99 @@ RED.utils = (function() {
}
}
if (i===length-1) {
throw new Error("Invalid property expression: unterminated expression");
throw createError("INVALID_EXPR","Invalid property expression: unterminated expression");
}
// Next char is first char of an identifier: a-z 0-9 $ _
if (!/[a-z0-9\$\_]/i.test(str[i+1])) {
throw new Error("Invalid property expression: unexpected "+str[i+1]+" at position "+(i+1));
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+str[i+1]+" at position "+(i+1));
}
start = i+1;
} else if (c === '[') {
if (i === 0) {
throw new Error("Invalid property expression: unexpected "+c+" at position "+i);
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+c+" at position "+i);
}
if (start != i) {
parts.push(str.substring(start,i));
}
if (i===length-1) {
throw new Error("Invalid property expression: unterminated expression");
throw createError("INVALID_EXPR","Invalid property expression: unterminated expression");
}
// Next char is either a quote or a number
if (!/["'\d]/.test(str[i+1])) {
throw new Error("Invalid property expression: unexpected "+str[i+1]+" at position "+(i+1));
// Start of a new expression. If it starts with msg it is a nested expression
// Need to scan ahead to find the closing bracket
if (/^msg[.\[]/.test(str.substring(i+1))) {
var depth = 1;
var inLocalString = false;
var localStringQuote;
for (var j=i+1;j<length;j++) {
if (/["']/.test(str[j])) {
if (inLocalString) {
if (str[j] === localStringQuote) {
inLocalString = false
}
} else {
inLocalString = true;
localStringQuote = str[j]
}
}
if (str[j] === '[') {
depth++;
} else if (str[j] === ']') {
depth--;
}
if (depth === 0) {
try {
if (msg) {
parts.push(getMessageProperty(msg, str.substring(i+1,j)))
} else {
parts.push(normalisePropertyExpression(str.substring(i+1,j), msg));
}
inBox = false;
i = j;
start = j+1;
break;
} catch(err) {
throw createError("INVALID_EXPR","Invalid expression started at position "+(i+1))
}
}
}
if (depth > 0) {
throw createError("INVALID_EXPR","Invalid property expression: unmatched '[' at position "+i);
}
continue;
} else if (!/["'\d]/.test(str[i+1])) {
// Next char is either a quote or a number
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+str[i+1]+" at position "+(i+1));
}
start = i+1;
inBox = true;
} else if (c === ']') {
if (!inBox) {
throw new Error("Invalid property expression: unexpected "+c+" at position "+i);
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+c+" at position "+i);
}
if (start != i) {
v = str.substring(start,i);
if (/^\d+$/.test(v)) {
parts.push(parseInt(v));
} else {
throw new Error("Invalid property expression: unexpected array expression at position "+start);
throw createError("INVALID_EXPR","Invalid property expression: unexpected array expression at position "+start);
}
}
start = i+1;
inBox = false;
} else if (c === ' ') {
throw new Error("Invalid property expression: unexpected ' ' at position "+i);
throw createError("INVALID_EXPR","Invalid property expression: unexpected ' ' at position "+i);
}
} else {
if (c === quoteChar) {
if (i-start === 0) {
throw new Error("Invalid property expression: zero-length string at position "+start);
throw createError("INVALID_EXPR","Invalid property expression: zero-length string at position "+start);
}
parts.push(str.substring(start,i));
// If inBox, next char must be a ]. Otherwise it may be [ or .
if (inBox && !/\]/.test(str[i+1])) {
throw new Error("Invalid property expression: unexpected array expression at position "+start);
throw createError("INVALID_EXPR","Invalid property expression: unexpected array expression at position "+start);
} else if (!inBox && i+1!==length && !/[\[\.]/.test(str[i+1])) {
throw new Error("Invalid property expression: unexpected "+str[i+1]+" expression at position "+(i+1));
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+str[i+1]+" expression at position "+(i+1));
}
start = i+1;
inString = false;
@ -711,7 +760,7 @@ RED.utils = (function() {
}
if (inBox || inString) {
throw new Error("Invalid property expression: unterminated expression");
throw new createError("INVALID_EXPR","Invalid property expression: unterminated expression");
}
if (start < length) {
parts.push(str.substring(start));

View File

@ -168,6 +168,10 @@ module.exports = function(RED) {
return getFromValueType(RED.util.getMessageProperty(msg,rule.from),done);
} else if (rule.fromt === 'flow' || rule.fromt === 'global') {
var contextKey = RED.util.parseContextStore(rule.from);
if (/\[msg\./.test(context.key)) {
// The key has a nest msg. reference to evaluate first
context.key = RED.util.normalisePropertyExpression(contextKey.key,msg,true);
}
node.context()[rule.fromt].get(contextKey.key, contextKey.store, (err,fromValue) => {
if (err) {
done(err)
@ -243,6 +247,10 @@ module.exports = function(RED) {
return done(undefined,msg);
} else if (rule.pt === 'flow' || rule.pt === 'global') {
var contextKey = RED.util.parseContextStore(property);
if (/\[msg/.test(contextKey.key)) {
// The key has a nest msg. reference to evaluate first
contextKey.key = RED.util.normalisePropertyExpression(contextKey.key, msg, true)
}
var target = node.context()[rule.pt];
var callback = err => {
if (err) {

View File

@ -189,11 +189,17 @@ function createError(code, message) {
*
* For example, `a["b"].c` returns `['a','b','c']`
*
* If `msg` is provided, any internal cross-references will be evaluated against that
* object. Otherwise, it will return a nested set of properties
*
* For example, without msg set, 'a[msg.foo]' returns `['a', [ 'msg', 'foo'] ]`
* But if msg is set to '{"foo": "bar"}', 'a[msg.foo]' returns `['a', 'bar' ]`
*
* @param {String} str - the property expression
* @return {Array} the normalised expression
* @memberof @node-red/util_util
*/
function normalisePropertyExpression(str) {
function normalisePropertyExpression(str, msg, toString) {
// This must be kept in sync with validatePropertyExpression
// in editor/js/ui/utils.js
@ -205,6 +211,7 @@ function normalisePropertyExpression(str) {
var start = 0;
var inString = false;
var inBox = false;
var boxExpression = false;
var quoteChar;
var v;
for (var i=0;i<length;i++) {
@ -247,8 +254,54 @@ function normalisePropertyExpression(str) {
if (i===length-1) {
throw createError("INVALID_EXPR","Invalid property expression: unterminated expression");
}
// Next char is either a quote or a number
if (!/["'\d]/.test(str[i+1])) {
// Start of a new expression. If it starts with msg it is a nested expression
// Need to scan ahead to find the closing bracket
if (/^msg[.\[]/.test(str.substring(i+1))) {
var depth = 1;
var inLocalString = false;
var localStringQuote;
for (var j=i+1;j<length;j++) {
if (/["']/.test(str[j])) {
if (inLocalString) {
if (str[j] === localStringQuote) {
inLocalString = false
}
} else {
inLocalString = true;
localStringQuote = str[j]
}
}
if (str[j] === '[') {
depth++;
} else if (str[j] === ']') {
depth--;
}
if (depth === 0) {
try {
if (msg) {
var crossRefProp = getMessageProperty(msg, str.substring(i+1,j));
if (crossRefProp === undefined) {
throw createError("INVALID_EXPR","Invalid expression: undefined reference at position "+(i+1)+" : "+str.substring(i+1,j))
}
parts.push(crossRefProp)
} else {
parts.push(normalisePropertyExpression(str.substring(i+1,j), msg));
}
inBox = false;
i = j;
start = j+1;
break;
} catch(err) {
throw createError("INVALID_EXPR","Invalid expression started at position "+(i+1))
}
}
}
if (depth > 0) {
throw createError("INVALID_EXPR","Invalid property expression: unmatched '[' at position "+i);
}
continue;
} else if (!/["'\d]/.test(str[i+1])) {
// Next char is either a quote or a number
throw createError("INVALID_EXPR","Invalid property expression: unexpected "+str[i+1]+" at position "+(i+1));
}
start = i+1;
@ -294,6 +347,23 @@ function normalisePropertyExpression(str) {
if (start < length) {
parts.push(str.substring(start));
}
if (toString) {
var result = parts.shift();
while(parts.length > 0) {
var p = parts.shift();
if (typeof p === 'string') {
if (/"/.test(p)) {
p = "'"+p+"'";
} else {
p = '"'+p+'"';
}
}
result = result+"["+p+"]";
}
return result;
}
return parts;
}
@ -340,8 +410,7 @@ function getMessageProperty(msg,expr) {
*/
function getObjectProperty(msg,expr) {
var result = null;
var msgPropParts = normalisePropertyExpression(expr);
var m;
var msgPropParts = normalisePropertyExpression(expr,msg);
msgPropParts.reduce(function(obj, key) {
result = (typeof obj[key] !== "undefined" ? obj[key] : undefined);
return result;
@ -381,7 +450,7 @@ function setObjectProperty(msg,prop,value,createMissing) {
if (typeof createMissing === 'undefined') {
createMissing = (typeof value !== 'undefined');
}
var msgPropParts = normalisePropertyExpression(prop);
var msgPropParts = normalisePropertyExpression(prop, msg);
var depth = 0;
var length = msgPropParts.length;
var obj = msg;
@ -553,6 +622,10 @@ function evaluateNodeProperty(value, type, node, msg, callback) {
}
} else if ((type === 'flow' || type === 'global') && node) {
var contextKey = parseContextStore(value);
if (/\[msg/.test(contextKey.key)) {
// The key has a nest msg. reference to evaluate first
contextKey.key = normalisePropertyExpression(contextKey.key, msg, true)
}
result = node.context()[type].get(contextKey.key,contextKey.store,callback);
if (callback) {
return;

View File

@ -119,13 +119,17 @@ describe('switch Node', function() {
* @param done - callback when done
*/
function customFlowSwitchTest(flow, shouldReceive, sendPayload, done) {
customFlowMessageSwitchTest(flow,shouldReceive,{payload: sendPayload}, done);
}
function customFlowMessageSwitchTest(flow, shouldReceive, message, done) {
helper.load(switchNode, flow, function() {
var switchNode1 = helper.getNode("switchNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
if (shouldReceive === true) {
should.equal(msg.payload,sendPayload);
should.equal(msg,message);
done();
} else {
should.fail(null, null, "We should never get an input!");
@ -134,7 +138,7 @@ describe('switch Node', function() {
done(err);
}
});
switchNode1.receive({payload:sendPayload});
switchNode1.receive(message);
if (shouldReceive === false) {
setTimeout(function() {
done();
@ -425,6 +429,29 @@ describe('switch Node', function() {
});
});
it('should use a nested message property to compare value - matches', function(done) {
var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload[msg.topic]",rules:[{"t":"eq","v":"bar"}],checkall:true,outputs:1,wires:[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
customFlowMessageSwitchTest(flow, true, {topic:"foo",payload:{"foo":"bar"}}, done);
})
it('should use a nested message property to compare value - no match', function(done) {
var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload[msg.topic]",rules:[{"t":"eq","v":"bar"}],checkall:true,outputs:1,wires:[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
customFlowMessageSwitchTest(flow, false, {topic:"foo",payload:{"foo":"none"}}, done);
})
it('should use a nested message property to compare nested message property - matches', function(done) {
var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload[msg.topic]",rules:[{"t":"eq","v":"payload[msg.topic2]",vt:"msg"}],checkall:true,outputs:1,wires:[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
customFlowMessageSwitchTest(flow, true, {topic:"foo",topic2:"foo2",payload:{"foo":"bar","foo2":"bar"}}, done);
})
it('should use a nested message property to compare nested message property - no match', function(done) {
var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload[msg.topic]",rules:[{"t":"eq","v":"payload[msg.topic2]",vt:"msg"}],checkall:true,outputs:1,wires:[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
customFlowMessageSwitchTest(flow, false, {topic:"foo",topic2:"foo2",payload:{"foo":"bar","foo2":"none"}}, done);
})
it('should match regex with ignore-case flag set true', function(done) {
var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload",rules:[{"t":"regex","v":"onetwothree","case":true}],checkall:true,outputs:1,wires:[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];

View File

@ -98,7 +98,7 @@ describe('change Node', function() {
});
describe('#set' , function() {
it('sets the value of the message property', function(done) {
var flow = [{"id":"changeNode1","type":"change","action":"replace","property":"payload","from":"","to":"changed","reg":false,"name":"changeNode","wires":[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
@ -672,6 +672,111 @@ describe('change Node', function() {
});
});
it('sets the value of a message property using a nested property', function(done) {
var flow = [{"id":"changeNode1","type":"change","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"lookup[msg.topic]","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"wires":[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
helper.load(changeNode, flow, function() {
var changeNode1 = helper.getNode("changeNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
msg.payload.should.equal(2);
done();
} catch(err) {
done(err);
}
});
changeNode1.receive({payload:"",lookup:{a:1,b:2},topic:"b"});
});
});
it('sets the value of a nested message property using a message property', function(done) {
var flow = [{"id":"changeNode1","type":"change","name":"","rules":[{"t":"set","p":"lookup[msg.topic]","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"wires":[["helperNode1"]]},
{id:"helperNode1", type:"helper", wires:[]}];
helper.load(changeNode, flow, function() {
var changeNode1 = helper.getNode("changeNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
msg.lookup.b.should.equal("newValue");
done();
} catch(err) {
done(err);
}
});
var msg = {
payload: "newValue",
lookup:{a:1,b:2},
topic:"b"
}
changeNode1.receive(msg);
});
});
it('sets the value of a message property using a nested property in flow context', function(done) {
var flow = [{"id":"changeNode1","type":"change","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"lookup[msg.topic]","tot":"flow"}],"action":"","property":"","from":"","to":"","reg":false,"wires":[["helperNode1"]],"z":"flow"},
{id:"helperNode1", type:"helper", wires:[]}];
helper.load(changeNode, flow, function() {
var changeNode1 = helper.getNode("changeNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
msg.payload.should.eql(2);
done();
} catch(err) {
done(err);
}
});
changeNode1.context().flow.set("lookup",{a:1, b:2});
changeNode1.receive({payload: "", topic: "b"});
});
})
it('sets the value of a message property using a nested property in flow context', function(done) {
var flow = [{"id":"changeNode1","type":"change","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"lookup[msg.topic]","tot":"flow"}],"action":"","property":"","from":"","to":"","reg":false,"wires":[["helperNode1"]],"z":"flow"},
{id:"helperNode1", type:"helper", wires:[]}];
helper.load(changeNode, flow, function() {
var changeNode1 = helper.getNode("changeNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
msg.payload.should.eql(2);
done();
} catch(err) {
done(err);
}
});
changeNode1.context().flow.set("lookup",{a:1, b:2});
changeNode1.receive({payload: "", topic: "b"});
});
})
it('sets the value of a nested flow context property using a message property', function(done) {
var flow = [{"id":"changeNode1","type":"change","name":"","rules":[{"t":"set","p":"lookup[msg.topic]","pt":"flow","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"wires":[["helperNode1"]],"z":"flow"},
{id:"helperNode1", type:"helper", wires:[]}];
helper.load(changeNode, flow, function() {
var changeNode1 = helper.getNode("changeNode1");
var helperNode1 = helper.getNode("helperNode1");
helperNode1.on("input", function(msg) {
try {
msg.payload.should.eql("newValue");
changeNode1.context().flow.get("lookup.b").should.eql("newValue");
done();
} catch(err) {
done(err);
}
});
changeNode1.context().flow.set("lookup",{a:1, b:2});
changeNode1.receive({payload: "newValue", topic: "b"});
});
})
});
describe('#change', function() {
it('changes the value of the message property', function(done) {

View File

@ -164,6 +164,13 @@ describe("@node-red/util/util", function() {
var v2 = util.getMessageProperty({a:"foo"},"a");
v2.should.eql("foo");
});
it('retrieves a nested property', function() {
var v = util.getMessageProperty({a:"foo",b:{foo:1,bar:2}},"msg.b[msg.a]");
v.should.eql(1);
var v2 = util.getMessageProperty({a:"bar",b:{foo:1,bar:2}},"b[msg.a]");
v2.should.eql(2);
});
it('should return undefined if property does not exist', function() {
var v = util.getMessageProperty({a:"foo"},"msg.b");
should.not.exist(v);
@ -331,7 +338,18 @@ describe("@node-red/util/util", function() {
msg.a[0].should.eql(1);
msg.a[1].should.eql(3);
})
it('handles nested message property references', function() {
var obj = {a:"foo",b:{}};
var result = util.setObjectProperty(obj,"b[msg.a]","bar");
result.should.be.true();
obj.b.should.have.property("foo","bar");
});
it('handles nested message property references', function() {
var obj = {a:"foo",b:{"foo":[0,0,0]}};
var result = util.setObjectProperty(obj,"b[msg.a][2]","bar");
result.should.be.true();
obj.b.foo.should.eql([0,0,"bar"])
});
});
describe('evaluateNodeProperty', function() {
@ -459,13 +477,24 @@ describe("@node-red/util/util", function() {
// console.log(result);
result.should.eql(expected);
}
function testInvalid(input) {
function testABCWithMessage(input,msg,expected) {
var result = util.normalisePropertyExpression(input,msg);
// console.log("+",input);
// console.log(result);
result.should.eql(expected);
}
function testInvalid(input,msg) {
/*jshint immed: false */
(function() {
util.normalisePropertyExpression(input);
util.normalisePropertyExpression(input,msg);
}).should.throw();
}
function testToString(input,msg,expected) {
var result = util.normalisePropertyExpression(input,msg,true);
console.log("+",input);
console.log(result);
result.should.eql(expected);
}
it('pass a.b.c',function() { testABC('a.b.c',['a','b','c']); })
it('pass a["b"]["c"]',function() { testABC('a["b"]["c"]',['a','b','c']); })
it('pass a["b"].c',function() { testABC('a["b"].c',['a','b','c']); })
@ -479,12 +508,25 @@ describe("@node-red/util/util", function() {
it("pass 'a.b'[1]",function() { testABC("'a.b'[1]",['a.b',1]); })
it("pass 'a.b'.c",function() { testABC("'a.b'.c",['a.b','c']); })
it("pass a[msg.b]",function() { testABC("a[msg.b]",["a",["msg","b"]]); })
it("pass a[msg[msg.b]]",function() { testABC("a[msg[msg.b]]",["a",["msg",["msg","b"]]]); })
it("pass a[msg.b]",function() { testABC("a[msg.b]",["a",["msg","b"]]); })
it("pass a[msg.b]",function() { testABC("a[msg.b]",["a",["msg","b"]]); })
it("pass a[msg['b]\"[']]",function() { testABC("a[msg['b]\"[']]",["a",["msg","b]\"["]]); })
it("pass a[msg['b][']]",function() { testABC("a[msg['b][']]",["a",["msg","b]["]]); })
it("pass b[msg.a][2]",function() { testABC("b[msg.a][2]",["b",["msg","a"],2])})
it("pass b[msg.a][2] (with message)",function() { testABCWithMessage("b[msg.a][2]",{a: "foo"},["b","foo",2])})
it('pass a.$b.c',function() { testABC('a.$b.c',['a','$b','c']); })
it('pass a["$b"].c',function() { testABC('a["$b"].c',['a','$b','c']); })
it('pass a._b.c',function() { testABC('a._b.c',['a','_b','c']); })
it('pass a["_b"].c',function() { testABC('a["_b"].c',['a','_b','c']); })
it("pass a['a.b[0]'].c",function() { testToString("a['a.b[0]'].c",null,'a["a.b[0]"]["c"]'); })
it("pass a.b.c",function() { testToString("a.b.c",null,'a["b"]["c"]'); })
it('pass a[msg.c][0]["fred"]',function() { testToString('a[msg.c][0]["fred"]',{c:"123"},'a["123"][0]["fred"]'); })
it("fail a'b'.c",function() { testInvalid("a'b'.c"); })
it("fail a['b'.c",function() { testInvalid("a['b'.c"); })
it("fail a[]",function() { testInvalid("a[]"); })
@ -505,6 +547,12 @@ describe("@node-red/util/util", function() {
it("fail a['']",function() { testInvalid("a['']"); })
it("fail 'a.b'c",function() { testInvalid("'a.b'c"); })
it("fail <blank>",function() { testInvalid("");})
it("fail a[b]",function() { testInvalid("a[b]"); })
it("fail a[msg.]",function() { testInvalid("a[msg.]"); })
it("fail a[msg[]",function() { testInvalid("a[msg[]"); })
it("fail a[msg.[]]",function() { testInvalid("a[msg.[]]"); })
it("fail a[msg['af]]",function() { testInvalid("a[msg['af]]"); })
it("fail b[msg.undefined][2] (with message)",function() { testInvalid("b[msg.undefined][2]",{})})
});
@ -983,4 +1031,5 @@ describe("@node-red/util/util", function() {
});
});
});