From c39e2ffd56b83164e73297ac942de342976f2280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathana=C3=ABl=20L=C3=A9caud=C3=A9?= Date: Thu, 28 Jun 2018 23:16:43 -0700 Subject: [PATCH 01/22] JSON node: add JSON schema validation via msg.schema --- nodes/core/parsers/70-JSON.js | 53 +++++++++++++++++++++++++++++++++-- package.json | 1 + 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/nodes/core/parsers/70-JSON.js b/nodes/core/parsers/70-JSON.js index aed328574..ce547fe10 100644 --- a/nodes/core/parsers/70-JSON.js +++ b/nodes/core/parsers/70-JSON.js @@ -16,21 +16,57 @@ module.exports = function(RED) { "use strict"; + const Ajv = require('ajv'); + const ajv = new Ajv({allErrors: true, schemaId: 'auto'}); + ajv.addMetaSchema(require('ajv/lib/refs/json-schema-draft-04.json')); function JSONNode(n) { RED.nodes.createNode(this,n); this.indent = n.pretty ? 4 : 0; this.action = n.action||""; this.property = n.property||"payload"; + this.schema = null; + this.compiledSchema = null; + var node = this; this.on("input", function(msg) { + var validate = false; + if (msg.schema) { + if (typeof msg.schema === "object") { + // If input schema is different, re-compile it + if (JSON.stringify(this.schema) != JSON.stringify(msg.schema)) { + node.warn('Schema different, compiling'); + try { + this.compiledSchema = ajv.compile(msg.schema); + this.schema = msg.schema; + } catch(e) { + this.schema = null; + this.compiledSchema = null; + node.error("JSON Schema error: failed to compile schema", msg); + return; + } + } + validate = true; + } else { + node.warn("Schema present but not an object, ignoring schema"); + } + } var value = RED.util.getMessageProperty(msg,node.property); if (value !== undefined) { if (typeof value === "string") { if (node.action === "" || node.action === "obj") { try { RED.util.setMessageProperty(msg,node.property,JSON.parse(value)); - node.send(msg); + if (validate) { + if (this.compiledSchema(msg[node.property])) { + node.send(msg); + } else { + msg.schemaError = this.compiledSchema.errors; + node.error(`JSON Schema error: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); + } + } else { + node.send(msg); + } } catch(e) { node.error(e.message,msg); } } else { @@ -41,8 +77,19 @@ module.exports = function(RED) { if (node.action === "" || node.action === "str") { if (!Buffer.isBuffer(value)) { try { - RED.util.setMessageProperty(msg,node.property,JSON.stringify(value,null,node.indent)); - node.send(msg); + if (validate) { + if (this.compiledSchema(value)) { + RED.util.setMessageProperty(msg,node.property,JSON.stringify(value,null,node.indent)); + node.send(msg); + } else { + msg.schemaError = this.compiledSchema.errors; + node.error(`JSON Schema error: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); + } + } else { + RED.util.setMessageProperty(msg,node.property,JSON.stringify(value,null,node.indent)); + node.send(msg); + } + } catch(e) { node.error(RED._("json.errors.dropped-error")); } } diff --git a/package.json b/package.json index ec0f63a34..866dfb11f 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "flow" ], "dependencies": { + "ajv": "6.5.1", "basic-auth": "2.0.0", "bcryptjs": "2.4.3", "body-parser": "1.18.3", From 905f89b0f55d0d059a2140dca19b1fe932185ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathana=C3=ABl=20L=C3=A9caud=C3=A9?= Date: Sat, 30 Jun 2018 16:19:39 -0700 Subject: [PATCH 02/22] JSON node: finalize JSON Schema validation --- nodes/core/locales/en-US/messages.json | 4 +- nodes/core/parsers/70-JSON.html | 5 ++ nodes/core/parsers/70-JSON.js | 31 ++++------ test/nodes/core/parsers/70-JSON_spec.js | 81 +++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/nodes/core/locales/en-US/messages.json b/nodes/core/locales/en-US/messages.json index 74ec0bcd4..c3586d412 100644 --- a/nodes/core/locales/en-US/messages.json +++ b/nodes/core/locales/en-US/messages.json @@ -697,7 +697,9 @@ "errors": { "dropped-object": "Ignored non-object payload", "dropped": "Ignored unsupported payload type", - "dropped-error": "Failed to convert payload" + "dropped-error": "Failed to convert payload", + "schema-error": "JSON Schema error", + "schema-error-compile": "JSON Schema error: failed to compile schema" }, "label": { "o2j": "Object to JSON options", diff --git a/nodes/core/parsers/70-JSON.html b/nodes/core/parsers/70-JSON.html index 565bae9a0..a440e5d45 100644 --- a/nodes/core/parsers/70-JSON.html +++ b/nodes/core/parsers/70-JSON.html @@ -31,6 +31,8 @@
payloadobject | string
A JavaScript object or JSON string.
+
schemaobject
+
An optional JSON Schema object to validate the payload against.

Outputs

@@ -41,6 +43,9 @@
  • If the input is a JavaScript object it creates a JSON string. The string can optionally be well-formatted.
    • +
    schemaErrorarray
    +
    If JSON schema validation fails, the catch node will have a schemaError property + containing an array of errors.

    Details

    By default, the node operates on msg.payload, but can be configured diff --git a/nodes/core/parsers/70-JSON.js b/nodes/core/parsers/70-JSON.js index ce547fe10..eb0bec63c 100644 --- a/nodes/core/parsers/70-JSON.js +++ b/nodes/core/parsers/70-JSON.js @@ -32,24 +32,19 @@ module.exports = function(RED) { this.on("input", function(msg) { var validate = false; if (msg.schema) { - if (typeof msg.schema === "object") { - // If input schema is different, re-compile it - if (JSON.stringify(this.schema) != JSON.stringify(msg.schema)) { - node.warn('Schema different, compiling'); - try { - this.compiledSchema = ajv.compile(msg.schema); - this.schema = msg.schema; - } catch(e) { - this.schema = null; - this.compiledSchema = null; - node.error("JSON Schema error: failed to compile schema", msg); - return; - } + // If input schema is different, re-compile it + if (JSON.stringify(this.schema) != JSON.stringify(msg.schema)) { + try { + this.compiledSchema = ajv.compile(msg.schema); + this.schema = msg.schema; + } catch(e) { + this.schema = null; + this.compiledSchema = null; + node.error(RED._("json.errors.schema-error-compile"), msg); + return; } - validate = true; - } else { - node.warn("Schema present but not an object, ignoring schema"); } + validate = true; } var value = RED.util.getMessageProperty(msg,node.property); if (value !== undefined) { @@ -62,7 +57,7 @@ module.exports = function(RED) { node.send(msg); } else { msg.schemaError = this.compiledSchema.errors; - node.error(`JSON Schema error: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); + node.error(`${RED._("json.errors.schema-error")}: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); } } else { node.send(msg); @@ -83,7 +78,7 @@ module.exports = function(RED) { node.send(msg); } else { msg.schemaError = this.compiledSchema.errors; - node.error(`JSON Schema error: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); + node.error(`${RED._("json.errors.schema-error")}: ${ajv.errorsText(this.compiledSchema.errors)}`, msg); } } else { RED.util.setMessageProperty(msg,node.property,JSON.stringify(value,null,node.indent)); diff --git a/test/nodes/core/parsers/70-JSON_spec.js b/test/nodes/core/parsers/70-JSON_spec.js index 28fffde85..ba913b1e7 100644 --- a/test/nodes/core/parsers/70-JSON_spec.js +++ b/test/nodes/core/parsers/70-JSON_spec.js @@ -247,4 +247,85 @@ describe('JSON node', function() { }); }); }); + + it('should pass an object if provided a valid JSON string and schema', function(done) { + var flow = [{id:"jn1",type:"json",wires:[["jn2"]]}, + {id:"jn2", type:"helper"}]; + helper.load(jsonNode, flow, function() { + var jn1 = helper.getNode("jn1"); + var jn2 = helper.getNode("jn2"); + jn2.on("input", function(msg) { + should.equal(msg.payload.number, 3); + should.equal(msg.payload.string, "allo"); + done(); + }); + var jsonString = '{"number": 3, "string": "allo"}'; + var schema = {title: "testSchema", type: "object", properties: {number: {type: "number"}, string: {type: "string" }}}; + jn1.receive({payload:jsonString, schema:schema}); + }); + }); + + it('should pass a string if provided a valid object and schema', function(done) { + var flow = [{id:"jn1",type:"json",wires:[["jn2"]]}, + {id:"jn2", type:"helper"}]; + helper.load(jsonNode, flow, function() { + var jn1 = helper.getNode("jn1"); + var jn2 = helper.getNode("jn2"); + jn2.on("input", function(msg) { + should.equal(msg.payload, '{"number":3,"string":"allo"}'); + done(); + }); + var obj = {"number": 3, "string": "allo"}; + var schema = {title: "testSchema", type: "object", properties: {number: {type: "number"}, string: {type: "string" }}}; + jn1.receive({payload:obj, schema:schema}); + }); + }); + + it('should log an error if passed an invalid object and valid schema', function(done) { + var flow = [{id:"jn1",type:"json",wires:[["jn2"]]}, + {id:"jn2", type:"helper"}]; + helper.load(jsonNode, flow, function() { + try { + var jn1 = helper.getNode("jn1"); + var jn2 = helper.getNode("jn2"); + var schema = {title: "testSchema", type: "object", properties: {number: {type: "number"}, string: {type: "string" }}}; + var obj = {"number": "foo", "string": 3}; + jn1.receive({payload:obj, schema:schema}); + var logEvents = helper.log().args.filter(function(evt) { + return evt[0].type == "json"; + }); + logEvents.should.have.length(1); + logEvents[0][0].should.have.a.property('msg'); + logEvents[0][0].msg.should.equal("json.errors.schema-error: data.number should be number, data.string should be string"); + logEvents[0][0].should.have.a.property('level',helper.log().ERROR); + done(); + } catch(err) { + done(err); + } + }); + }); + + it('should log an error if passed a valid object and invalid schema', function(done) { + var flow = [{id:"jn1",type:"json",wires:[["jn2"]]}, + {id:"jn2", type:"helper"}]; + helper.load(jsonNode, flow, function() { + try { + var jn1 = helper.getNode("jn1"); + var jn2 = helper.getNode("jn2"); + var schema = "garbage"; + var obj = {"number": "foo", "string": 3}; + jn1.receive({payload:obj, schema:schema}); + var logEvents = helper.log().args.filter(function(evt) { + return evt[0].type == "json"; + }); + logEvents.should.have.length(1); + logEvents[0][0].should.have.a.property('msg'); + logEvents[0][0].msg.should.equal("json.errors.schema-error-compile"); + logEvents[0][0].should.have.a.property('level',helper.log().ERROR); + done(); + } catch(err) { + done(err); + } + }); + }); }); From 4bcf13cb58869902e3d62294af91eeece5c93497 Mon Sep 17 00:00:00 2001 From: Dave Conway-Jones Date: Sat, 7 Jul 2018 19:01:14 +0100 Subject: [PATCH 03/22] Let nrgpio code work with python 3 (just in case that becomes default) --- nodes/core/hardware/nrgpio.py | 40 ++++++++++++++++------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/nodes/core/hardware/nrgpio.py b/nodes/core/hardware/nrgpio.py index 6bbcddbbe..0cde0e4df 100755 --- a/nodes/core/hardware/nrgpio.py +++ b/nodes/core/hardware/nrgpio.py @@ -23,10 +23,6 @@ from time import sleep bounce = 25; -if sys.version_info >= (3,0): - print("Sorry - currently only configured to work with python 2.x") - sys.exit(1) - if len(sys.argv) > 2: cmd = sys.argv[1].lower() pin = int(sys.argv[2]) @@ -34,7 +30,7 @@ if len(sys.argv) > 2: GPIO.setwarnings(False) if cmd == "pwm": - #print "Initialised pin "+str(pin)+" to PWM" + #print("Initialised pin "+str(pin)+" to PWM") try: freq = int(sys.argv[3]) except: @@ -54,10 +50,10 @@ if len(sys.argv) > 2: GPIO.cleanup(pin) sys.exit(0) except Exception as ex: - print "bad data: "+data + print("bad data: "+data) elif cmd == "buzz": - #print "Initialised pin "+str(pin)+" to Buzz" + #print("Initialised pin "+str(pin)+" to Buzz") GPIO.setup(pin,GPIO.OUT) p = GPIO.PWM(pin, 100) p.stop() @@ -76,10 +72,10 @@ if len(sys.argv) > 2: GPIO.cleanup(pin) sys.exit(0) except Exception as ex: - print "bad data: "+data + print("bad data: "+data) elif cmd == "out": - #print "Initialised pin "+str(pin)+" to OUT" + #print("Initialised pin "+str(pin)+" to OUT") GPIO.setup(pin,GPIO.OUT) if len(sys.argv) == 4: GPIO.output(pin,int(sys.argv[3])) @@ -103,11 +99,11 @@ if len(sys.argv) > 2: GPIO.output(pin,data) elif cmd == "in": - #print "Initialised pin "+str(pin)+" to IN" + #print("Initialised pin "+str(pin)+" to IN") bounce = float(sys.argv[4]) def handle_callback(chan): sleep(bounce/1000.0) - print GPIO.input(chan) + print(GPIO.input(chan)) if sys.argv[3].lower() == "up": GPIO.setup(pin,GPIO.IN,GPIO.PUD_UP) @@ -116,7 +112,7 @@ if len(sys.argv) > 2: else: GPIO.setup(pin,GPIO.IN) - print GPIO.input(pin) + print(GPIO.input(pin)) GPIO.add_event_detect(pin, GPIO.BOTH, callback=handle_callback, bouncetime=int(bounce)) while True: @@ -129,7 +125,7 @@ if len(sys.argv) > 2: sys.exit(0) elif cmd == "byte": - #print "Initialised BYTE mode - "+str(pin)+ + #print("Initialised BYTE mode - "+str(pin)+) list = [7,11,13,12,15,16,18,22] GPIO.setup(list,GPIO.OUT) @@ -152,7 +148,7 @@ if len(sys.argv) > 2: GPIO.output(list[bit], data & mask) elif cmd == "borg": - #print "Initialised BORG mode - "+str(pin)+ + #print("Initialised BORG mode - "+str(pin)+) GPIO.setup(11,GPIO.OUT) GPIO.setup(13,GPIO.OUT) GPIO.setup(15,GPIO.OUT) @@ -190,7 +186,7 @@ if len(sys.argv) > 2: button = ord( buf[0] ) & pin # mask out just the required button(s) if button != oldbutt: # only send if changed oldbutt = button - print button + print(button) while True: try: @@ -215,7 +211,7 @@ if len(sys.argv) > 2: # type,code,value print("%u,%u" % (code, value)) event = file.read(EVENT_SIZE) - print "0,0" + print("0,0") file.close() sys.exit(0) except: @@ -225,14 +221,14 @@ if len(sys.argv) > 2: elif len(sys.argv) > 1: cmd = sys.argv[1].lower() if cmd == "rev": - print GPIO.RPI_REVISION + print(GPIO.RPI_REVISION) elif cmd == "ver": - print GPIO.VERSION + print(GPIO.VERSION) elif cmd == "info": - print GPIO.RPI_INFO + print(GPIO.RPI_INFO) else: - print "Bad parameters - in|out|pwm|buzz|byte|borg|mouse|kbd|ver|info {pin} {value|up|down}" - print " only ver (gpio version) and info (board information) accept no pin parameter." + print("Bad parameters - in|out|pwm|buzz|byte|borg|mouse|kbd|ver|info {pin} {value|up|down}") + print(" only ver (gpio version) and info (board information) accept no pin parameter.") else: - print "Bad parameters - in|out|pwm|buzz|byte|borg|mouse|kbd|ver|info {pin} {value|up|down}" + print("Bad parameters - in|out|pwm|buzz|byte|borg|mouse|kbd|ver|info {pin} {value|up|down}") From f870e9ed3ef1db0c6bb4a1836cd4413bcc0048b6 Mon Sep 17 00:00:00 2001 From: Dave Conway-Jones Date: Sun, 8 Jul 2018 16:52:30 +0100 Subject: [PATCH 04/22] Let Join node accumulate top level properties Last in is still most significant --- nodes/core/logic/17-split.js | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/nodes/core/logic/17-split.js b/nodes/core/logic/17-split.js index 92b527d01..af16f5bbd 100644 --- a/nodes/core/logic/17-split.js +++ b/nodes/core/logic/17-split.js @@ -280,8 +280,8 @@ module.exports = function(RED) { msgs.sort(function(x,y) { var ix = x.parts.index; var iy = y.parts.index; - if (ix < iy) return -flag; - if (ix > iy) return flag; + if (ix < iy) { return -flag; } + if (ix > iy) { return flag; } return 0; }); for(var msg of msgs) { @@ -437,7 +437,8 @@ module.exports = function(RED) { newArray = newArray.concat(n); }) group.payload = newArray; - } else if (group.type === 'buffer') { + } + else if (group.type === 'buffer') { var buffers = []; var bufferLen = 0; if (group.joinChar !== undefined) { @@ -450,7 +451,8 @@ module.exports = function(RED) { buffers.push(group.payload[i]); bufferLen += group.payload[i].length; } - } else { + } + else { bufferLen = group.bufferLen; buffers = group.payload; } @@ -463,7 +465,8 @@ module.exports = function(RED) { groupJoinChar = group.joinChar.toString(); } RED.util.setMessageProperty(group.msg,node.property,group.payload.join(groupJoinChar)); - } else { + } + else { if (node.propertyType === 'full') { group.msg = RED.util.cloneMessage(group.msg); } @@ -471,7 +474,8 @@ module.exports = function(RED) { } if (group.msg.hasOwnProperty('parts') && group.msg.parts.hasOwnProperty('parts')) { group.msg.parts = group.msg.parts.parts; - } else { + } + else { delete group.msg.parts; } delete group.msg.complete; @@ -525,7 +529,7 @@ module.exports = function(RED) { payloadType = node.build; targetCount = node.count; joinChar = node.joiner; - if (targetCount === 0 && msg.hasOwnProperty('parts')) { + if (n.count === "" && msg.hasOwnProperty('parts')) { targetCount = msg.parts.count || 0; } if (node.build === 'object') { @@ -554,7 +558,7 @@ module.exports = function(RED) { payload:{}, targetCount:targetCount, type:"object", - msg:msg + msg:RED.util.cloneMessage(msg) }; } else if (node.accumulate === true) { @@ -564,7 +568,7 @@ module.exports = function(RED) { payload:{}, targetCount:targetCount, type:payloadType, - msg:msg + msg:RED.util.cloneMessage(msg) } if (payloadType === 'string' || payloadType === 'array' || payloadType === 'buffer') { inflight[partId].payload = []; @@ -576,7 +580,7 @@ module.exports = function(RED) { payload:[], targetCount:targetCount, type:payloadType, - msg:msg + msg:RED.util.cloneMessage(msg) }; if (payloadType === 'string') { inflight[partId].joinChar = joinChar; @@ -624,14 +628,14 @@ module.exports = function(RED) { } group.currentCount++; } - // TODO: currently reuse the last received - add option to pick first received - group.msg = msg; + group.msg = Object.assign(group.msg, msg); var tcnt = group.targetCount; if (msg.hasOwnProperty("parts")) { tcnt = group.targetCount || msg.parts.count; } if ((tcnt > 0 && group.currentCount >= tcnt) || msg.hasOwnProperty('complete')) { completeSend(partId); } - } catch(err) { + } + catch(err) { console.log(err.stack); } }); From afb566b6b4a4b2387ab5e300f35abd40fdcb1edf Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sat, 7 Jul 2018 22:09:55 +0100 Subject: [PATCH 05/22] Add async context support to Inject node --- nodes/core/core/20-inject.js | 40 +++++++++++++++++++++++------------- red/runtime/util.js | 2 +- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/nodes/core/core/20-inject.js b/nodes/core/core/20-inject.js index 9e6eb62a8..c0d9e0c2f 100644 --- a/nodes/core/core/20-inject.js +++ b/nodes/core/core/20-inject.js @@ -63,21 +63,33 @@ module.exports = function(RED) { } this.on("input",function(msg) { - try { - msg.topic = this.topic; - if ( (this.payloadType == null && this.payload === "") || this.payloadType === "date") { - msg.payload = Date.now(); - } else if (this.payloadType == null) { - msg.payload = this.payload; - } else if (this.payloadType === 'none') { - msg.payload = ""; - } else { - msg.payload = RED.util.evaluateNodeProperty(this.payload,this.payloadType,this,msg); + msg.topic = this.topic; + if (this.payloadType !== 'flow' && this.payloadType !== 'global') { + try { + if ( (this.payloadType == null && this.payload === "") || this.payloadType === "date") { + msg.payload = Date.now(); + } else if (this.payloadType == null) { + msg.payload = this.payload; + } else if (this.payloadType === 'none') { + msg.payload = ""; + } else { + msg.payload = RED.util.evaluateNodeProperty(this.payload,this.payloadType,this,msg); + } + this.send(msg); + msg = null; + } catch(err) { + this.error(err,msg); } - this.send(msg); - msg = null; - } catch(err) { - this.error(err,msg); + } else { + RED.util.evaluateNodeProperty(this.payload,this.payloadType,this,msg, function(err,res) { + if (err) { + node.error(err,msg); + } else { + msg.payload = res; + node.send(msg); + } + + }); } }); } diff --git a/red/runtime/util.js b/red/runtime/util.js index 7d6464174..6279c416c 100644 --- a/red/runtime/util.js +++ b/red/runtime/util.js @@ -368,7 +368,7 @@ function evaluateNodeProperty(value, type, node, msg, callback) { if (callback) { callback(result); } else { - return value; + return result; } } From 1b693eed372633b3fdb7dff3a666154acc085ffe Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sat, 7 Jul 2018 22:23:26 +0100 Subject: [PATCH 06/22] Add async context support to Change node --- nodes/core/logic/15-change.js | 263 +++++++++++++----------- test/nodes/core/logic/15-change_spec.js | 23 +++ 2 files changed, 163 insertions(+), 123 deletions(-) diff --git a/nodes/core/logic/15-change.js b/nodes/core/logic/15-change.js index ab4f59cc3..4094f68e0 100644 --- a/nodes/core/logic/15-change.js +++ b/nodes/core/logic/15-change.js @@ -99,158 +99,175 @@ module.exports = function(RED) { } function applyRule(msg,rule) { - try { - var property = rule.p; - var value = rule.to; - if (rule.tot === 'json') { - value = JSON.parse(rule.to); - } else if (rule.tot === 'bin') { - value = Buffer.from(JSON.parse(rule.to)) - } - var current; - var fromValue; - var fromType; - var fromRE; - if (rule.tot === "msg") { - value = RED.util.getMessageProperty(msg,rule.to); - } else if (rule.tot === 'flow') { - value = node.context().flow.get(rule.to); - } else if (rule.tot === 'global') { - value = node.context().global.get(rule.to); - } else if (rule.tot === 'date') { - value = Date.now(); - } else if (rule.tot === 'jsonata') { - try{ - value = RED.util.evaluateJSONataExpression(rule.to,msg); - } catch(err) { - node.error(RED._("change.errors.invalid-expr",{error:err.message}),msg); - return; + return new Promise(function(resolve, reject){ + try { + var property = rule.p; + var value = rule.to; + if (rule.tot === 'json') { + value = JSON.parse(rule.to); + } else if (rule.tot === 'bin') { + value = Buffer.from(JSON.parse(rule.to)) } - } - if (rule.t === 'change') { - if (rule.fromt === 'msg' || rule.fromt === 'flow' || rule.fromt === 'global') { - if (rule.fromt === "msg") { - fromValue = RED.util.getMessageProperty(msg,rule.from); - } else if (rule.fromt === 'flow') { - fromValue = node.context().flow.get(rule.from); - } else if (rule.fromt === 'global') { - fromValue = node.context().global.get(rule.from); + var current; + var fromValue; + var fromType; + var fromRE; + if (rule.tot === "msg") { + value = RED.util.getMessageProperty(msg,rule.to); + } else if (rule.tot === 'flow') { + value = node.context().flow.get(rule.to); + } else if (rule.tot === 'global') { + value = node.context().global.get(rule.to); + } else if (rule.tot === 'date') { + value = Date.now(); + } else if (rule.tot === 'jsonata') { + try{ + value = RED.util.evaluateJSONataExpression(rule.to,msg); + } catch(err) { + node.error(RED._("change.errors.invalid-expr",{error:err.message}),msg); + return; } - if (typeof fromValue === 'number' || fromValue instanceof Number) { - fromType = 'num'; - } else if (typeof fromValue === 'boolean') { - fromType = 'bool' - } else if (fromValue instanceof RegExp) { - fromType = 're'; - fromRE = fromValue; - } else if (typeof fromValue === 'string') { - fromType = 'str'; - fromRE = fromValue.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); - try { - fromRE = new RegExp(fromRE, "g"); - } catch (e) { - valid = false; - node.error(RED._("change.errors.invalid-from",{error:e.message}),msg); - return; + } + if (rule.t === 'change') { + if (rule.fromt === 'msg' || rule.fromt === 'flow' || rule.fromt === 'global') { + if (rule.fromt === "msg") { + fromValue = RED.util.getMessageProperty(msg,rule.from); + } else if (rule.fromt === 'flow') { + fromValue = node.context().flow.get(rule.from); + } else if (rule.fromt === 'global') { + fromValue = node.context().global.get(rule.from); + } + if (typeof fromValue === 'number' || fromValue instanceof Number) { + fromType = 'num'; + } else if (typeof fromValue === 'boolean') { + fromType = 'bool' + } else if (fromValue instanceof RegExp) { + fromType = 're'; + fromRE = fromValue; + } else if (typeof fromValue === 'string') { + fromType = 'str'; + fromRE = fromValue.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); + try { + fromRE = new RegExp(fromRE, "g"); + } catch (e) { + valid = false; + reject(RED._("change.errors.invalid-from",{error:e.message})); + return; + } + } else { + reject(RED._("change.errors.invalid-from",{error:"unsupported type: "+(typeof fromValue)})); + return } } else { - node.error(RED._("change.errors.invalid-from",{error:"unsupported type: "+(typeof fromValue)}),msg); - return - } - } else { - fromType = rule.fromt; - fromValue = rule.from; - fromRE = rule.fromRE; - } - } - if (rule.pt === 'msg') { - if (rule.t === 'delete') { - RED.util.setMessageProperty(msg,property,undefined); - } else if (rule.t === 'set') { - RED.util.setMessageProperty(msg,property,value); - } else if (rule.t === 'change') { - current = RED.util.getMessageProperty(msg,property); - if (typeof current === 'string') { - if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { - // str representation of exact from number/boolean - // only replace if they match exactly - RED.util.setMessageProperty(msg,property,value); - } else { - current = current.replace(fromRE,value); - RED.util.setMessageProperty(msg,property,current); - } - } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { - if (current == Number(fromValue)) { - RED.util.setMessageProperty(msg,property,value); - } - } else if (typeof current === 'boolean' && fromType === 'bool') { - if (current.toString() === fromValue) { - RED.util.setMessageProperty(msg,property,value); - } + fromType = rule.fromt; + fromValue = rule.from; + fromRE = rule.fromRE; } } - } - else { - var target; - if (rule.pt === 'flow') { - target = node.context().flow; - } else if (rule.pt === 'global') { - target = node.context().global; - } - if (target) { + if (rule.pt === 'msg') { if (rule.t === 'delete') { - target.set(property,undefined); + RED.util.setMessageProperty(msg,property,undefined); } else if (rule.t === 'set') { - target.set(property,value); + RED.util.setMessageProperty(msg,property,value); } else if (rule.t === 'change') { - current = target.get(property); + current = RED.util.getMessageProperty(msg,property); if (typeof current === 'string') { if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { // str representation of exact from number/boolean // only replace if they match exactly - target.set(property,value); + RED.util.setMessageProperty(msg,property,value); } else { current = current.replace(fromRE,value); - target.set(property,current); + RED.util.setMessageProperty(msg,property,current); } } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { if (current == Number(fromValue)) { - target.set(property,value); + RED.util.setMessageProperty(msg,property,value); } } else if (typeof current === 'boolean' && fromType === 'bool') { if (current.toString() === fromValue) { - target.set(property,value); + RED.util.setMessageProperty(msg,property,value); } } } } + else { + var target; + if (rule.pt === 'flow') { + target = node.context().flow; + } else if (rule.pt === 'global') { + target = node.context().global; + } + if (target) { + if (rule.t === 'delete') { + target.set(property,undefined); + } else if (rule.t === 'set') { + target.set(property,value); + } else if (rule.t === 'change') { + current = target.get(property); + if (typeof current === 'string') { + if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { + // str representation of exact from number/boolean + // only replace if they match exactly + target.set(property,value); + } else { + current = current.replace(fromRE,value); + target.set(property,current); + } + } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { + if (current == Number(fromValue)) { + target.set(property,value); + } + } else if (typeof current === 'boolean' && fromType === 'bool') { + if (current.toString() === fromValue) { + target.set(property,value); + } + } + } + } + } + } catch(err) {/*console.log(err.stack)*/} + resolve(msg); + }); + } + function applyRules(msg, currentRule) { + var r = node.rules[currentRule]; + var rulePromise; + if (r.t === "move") { + if ((r.tot !== r.pt) || (r.p.indexOf(r.to) !== -1)) { + rulePromise = applyRule(msg,{t:"set", p:r.to, pt:r.tot, to:r.p, tot:r.pt}).then( + msg => applyRule(msg,{t:"delete", p:r.p, pt:r.pt}) + ); } - } catch(err) {/*console.log(err.stack)*/} - return msg; + else { // 2 step move if we are moving from a child + rulePromise = applyRule(msg,{t:"set", p:"_temp_move", pt:r.tot, to:r.p, tot:r.pt}).then( + msg => applyRule(msg,{t:"delete", p:r.p, pt:r.pt}) + ).then( + msg => applyRule(msg,{t:"set", p:r.to, pt:r.tot, to:"_temp_move", tot:r.pt}) + ).then( + msg => applyRule(msg,{t:"delete", p:"_temp_move", pt:r.pt}) + ) + } + } else { + rulePromise = applyRule(msg,r); + } + return rulePromise.then( + msg => { + if (!msg) { + return + } else if (currentRule === node.rules.length - 1) { + return msg; + } else { + return applyRules(msg, currentRule+1); + } + } + ); } if (valid) { this.on('input', function(msg) { - for (var i=0; i { if (msg) { node.send(msg) }} ) + .catch( err => node.error(err, msg)) }); } } diff --git a/test/nodes/core/logic/15-change_spec.js b/test/nodes/core/logic/15-change_spec.js index e444b2663..f819c5d0d 100644 --- a/test/nodes/core/logic/15-change_spec.js +++ b/test/nodes/core/logic/15-change_spec.js @@ -15,6 +15,7 @@ **/ var should = require("should"); +var sinon = require("sinon"); var changeNode = require("../../../../nodes/core/logic/15-change.js"); var helper = require("node-red-node-test-helper"); @@ -454,6 +455,28 @@ describe('change Node', function() { }); }); + it('reports invalid jsonata expression', function(done) { + var flow = [{"id":"changeNode1","type":"change",rules:[{"t":"set","p":"payload","to":"$invalid(payload)","tot":"jsonata"}],"name":"changeNode","wires":[["helperNode1"]]}, + {id:"helperNode1", type:"helper", wires:[]}]; + helper.load(changeNode, flow, function() { + var changeNode1 = helper.getNode("changeNode1"); + var helperNode1 = helper.getNode("helperNode1"); + sinon.spy(changeNode1,"error"); + helperNode1.on("input", function(msg) { + done("Invalid jsonata expression passed message through"); + }); + changeNode1.receive({payload:"Hello World!"}); + setTimeout(function() { + try { + changeNode1.error.called.should.be.true(); + done(); + } catch(err) { + done(err); + } + },50); + }); + }); + }); describe('#change', function() { it('changes the value of the message property', function(done) { From 1a6babd199db2a3c35594ac64f1f54e63f848e52 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sun, 8 Jul 2018 22:13:36 +0100 Subject: [PATCH 07/22] Lint Switch code --- nodes/core/logic/10-switch.js | 113 ++++++++++++++++------------------ 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/nodes/core/logic/10-switch.js b/nodes/core/logic/10-switch.js index 919fe13ac..ee3b4ef12 100644 --- a/nodes/core/logic/10-switch.js +++ b/nodes/core/logic/10-switch.js @@ -59,19 +59,19 @@ module.exports = function(RED) { 'else': function(a) { return a === true; } }; - var _max_kept_msgs_count = undefined; + var _maxKeptCount; - function max_kept_msgs_count(node) { - if (_max_kept_msgs_count === undefined) { + function getMaxKeptCount() { + if (_maxKeptCount === undefined) { var name = "nodeMessageBufferMaxLength"; if (RED.settings.hasOwnProperty(name)) { - _max_kept_msgs_count = RED.settings[name]; + _maxKeptCount = RED.settings[name]; } else { - _max_kept_msgs_count = 0; + _maxKeptCount = 0; } } - return _max_kept_msgs_count; + return _maxKeptCount; } function SwitchNode(n) { @@ -94,10 +94,10 @@ module.exports = function(RED) { var node = this; var valid = true; var repair = n.repair; - var needs_count = repair; + var needsCount = repair; for (var i=0; i 0) && (pending_count > max_msgs)) { - clear_pending(); + pendingCount++; + var max_msgs = getMaxKeptCount(); + if ((max_msgs > 0) && (pendingCount > max_msgs)) { + clearPending(); node.error(RED._("switch.errors.too-many"), msg); } if (parts.hasOwnProperty("count")) { @@ -170,32 +170,29 @@ module.exports = function(RED) { return group; } - function del_group_in(id, group) { - pending_count -= group.msgs.length; - delete pending_in[id]; - } - function add2pending_in(msg) { + function addMessageToPending(msg) { var parts = msg.parts; if (parts.hasOwnProperty("id") && parts.hasOwnProperty("index")) { - var group = add2group_in(parts.id, msg, parts); + var group = addMessageToGroup(parts.id, msg, parts); var msgs = group.msgs; var count = group.count; if (count === msgs.length) { for (var i = 0; i < msgs.length; i++) { var msg = msgs[i]; msg.parts.count = count; - process_msg(msg, false); + processMessage(msg, false); } - del_group_in(parts.id, group); + pendingCount -= group.msgs.length; + delete pendingIn[parts.id]; } return true; } return false; } - function send_group(onwards, port_count) { + function sendGroup(onwards, port_count) { var counts = new Array(port_count).fill(0); for (var i = 0; i < onwards.length; i++) { var onward = onwards[i]; @@ -230,47 +227,41 @@ module.exports = function(RED) { } } - function send2ports(onward, msg) { + function sendMessages(onward, msg) { var parts = msg.parts; var gid = parts.id; received[gid] = ((gid in received) ? received[gid] : 0) +1; var send_ok = (received[gid] === parts.count); - if (!(gid in pending_out)) { - pending_out[gid] = { + if (!(gid in pendingOut)) { + pendingOut[gid] = { onwards: [] }; } - var group = pending_out[gid]; + var group = pendingOut[gid]; var onwards = group.onwards; onwards.push(onward); - pending_count++; + pendingCount++; if (send_ok) { - send_group(onwards, onward.length, msg); - pending_count -= onward.length; - delete pending_out[gid]; + sendGroup(onwards, onward.length, msg); + pendingCount -= onward.length; + delete pendingOut[gid]; delete received[gid]; } - var max_msgs = max_kept_msgs_count(node); - if ((max_msgs > 0) && (pending_count > max_msgs)) { - clear_pending(); + var max_msgs = getMaxKeptCount(); + if ((max_msgs > 0) && (pendingCount > max_msgs)) { + clearPending(); node.error(RED._("switch.errors.too-many"), msg); } } - function msg_has_parts(msg) { - if (msg.hasOwnProperty("parts")) { - var parts = msg.parts; - return (parts.hasOwnProperty("id") && - parts.hasOwnProperty("index")); - } - return false; - } + function processMessage(msg, check_parts) { + var has_parts = msg.hasOwnProperty("parts") && + msg.parts.hasOwnProperty("id") && + msg.parts.hasOwnProperty("index"); - function process_msg(msg, check_parts) { - var has_parts = msg_has_parts(msg); - if (needs_count && check_parts && has_parts && - add2pending_in(msg)) { + if (needsCount && check_parts && has_parts && + addMessageToPending(msg)) { return; } var onward = []; @@ -344,27 +335,27 @@ module.exports = function(RED) { node.send(onward); } else { - send2ports(onward, msg); + sendMessages(onward, msg); } } catch(err) { node.warn(err); } } - function clear_pending() { - pending_count = 0; - pending_id = 0; - pending_in = {}; - pending_out = {}; + function clearPending() { + pendingCount = 0; + pendingId = 0; + pendingIn = {}; + pendingOut = {}; received = {}; } this.on('input', function(msg) { - process_msg(msg, true); + processMessage(msg, true); }); this.on('close', function() { - clear_pending(); + clearPending(); }); } From 9c00492dc25922d846c1a82fee76045e594a2a4f Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Sun, 8 Jul 2018 23:06:52 +0100 Subject: [PATCH 08/22] WIP: create async Switch node helper functions --- nodes/core/logic/10-switch.js | 103 +++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 7 deletions(-) diff --git a/nodes/core/logic/10-switch.js b/nodes/core/logic/10-switch.js index ee3b4ef12..a58ac91ca 100644 --- a/nodes/core/logic/10-switch.js +++ b/nodes/core/logic/10-switch.js @@ -74,6 +74,81 @@ module.exports = function(RED) { return _maxKeptCount; } + function getProperty(node,msg) { + return new Promise((resolve,reject) => { + if (node.propertyType === 'jsonata') { + try { + resolve(RED.util.evaluateJSONataExpression(node.property,msg)); + } catch(err) { + // TODO: proper invalid expr message + reject(err); + } + } else { + resolve(RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg)); + } + }); + } + + function getV1(node,msg,rule,hasParts) { + return new Promise( (resolve,reject) => { + if (rule.vt === 'prev') { + resolve(node.previousValue); + } else if (rule.vt === 'jsonata') { + try { + var exp = rule.v; + if (rule.t === 'jsonata_exp') { + if (hasParts) { + exp.assign("I", msg.parts.index); + exp.assign("N", msg.parts.count); + } + } + resolve(RED.util.evaluateJSONataExpression(exp,msg)); + } catch(err) { + reject(RED._("switch.errors.invalid-expr",{error:err.message})); + } + } else if (rule.vt === 'json') { + resolve("json"); + } else if (rule.vt === 'null') { + resolve("null"); + } else { + RED.util.evaluateNodeProperty(rule.v,rule.vt,node,msg, function(err,value) { + if (err) { + resolve(undefined); + } else { + resolve(value); + } + }); + } + }); + } + + function getV2(node,msg,rule) { + return new Promise((resolve,reject) => { + var v2 = rule.v2; + if (rule.v2t === 'prev') { + resolve(node.previousValue); + } else if (rule.v2t === 'jsonata') { + try { + resolve(RED.util.evaluateJSONataExpression(rule.v2,msg)); + } catch(err) { + reject(RED._("switch.errors.invalid-expr",{error:err.message})); + } + } else if (typeof v2 !== 'undefined') { + RED.util.evaluateNodeProperty(rule.v2,rule.v2t,node,msg, function(err,value) { + if (err) { + resolve(undefined); + } else { + resolve(value); + } + }); + } else { + resolve(v2); + } + }) + } + + + function SwitchNode(n) { RED.nodes.createNode(this, n); this.rules = n.rules || []; @@ -227,7 +302,7 @@ module.exports = function(RED) { } } - function sendMessages(onward, msg) { + function sendGroupMessages(onward, msg) { var parts = msg.parts; var gid = parts.id; received[gid] = ((gid in received) ? received[gid] : 0) +1; @@ -255,35 +330,43 @@ module.exports = function(RED) { } } - function processMessage(msg, check_parts) { - var has_parts = msg.hasOwnProperty("parts") && + + + function processMessage(msg, checkParts) { + var hasParts = msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("id") && msg.parts.hasOwnProperty("index"); - if (needsCount && check_parts && has_parts && + if (needsCount && checkParts && hasParts && addMessageToPending(msg)) { return; } var onward = []; try { var prop; + + // getProperty if (node.propertyType === 'jsonata') { prop = RED.util.evaluateJSONataExpression(node.property,msg); } else { prop = RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg); } + // end getProperty + var elseflag = true; for (var i=0; i Date: Mon, 9 Jul 2018 11:30:53 +0100 Subject: [PATCH 09/22] Add async property handling to Switch node --- nodes/core/logic/10-switch.js | 219 +++++++++++++----------- red/runtime/util.js | 13 +- test/nodes/core/logic/10-switch_spec.js | 26 ++- 3 files changed, 149 insertions(+), 109 deletions(-) diff --git a/nodes/core/logic/10-switch.js b/nodes/core/logic/10-switch.js index a58ac91ca..d15be39de 100644 --- a/nodes/core/logic/10-switch.js +++ b/nodes/core/logic/10-switch.js @@ -84,7 +84,13 @@ module.exports = function(RED) { reject(err); } } else { - resolve(RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg)); + RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg,(err,value) => { + if (err) { + resolve(undefined); + } else { + resolve(value); + } + }); } }); } @@ -147,6 +153,56 @@ module.exports = function(RED) { }) } + function applyRule(node, msg, property, state) { + return new Promise((resolve,reject) => { + + var rule = node.rules[state.currentRule]; + var v1,v2; + + getV1(node,msg,rule,state.hasParts).then(value => { + v1 = value; + }).then(()=>getV2(node,msg,rule)).then(value => { + v2 = value; + }).then(() => { + if (rule.t == "else") { + property = state.elseflag; + state.elseflag = true; + } + if (operators[rule.t](property,v1,v2,rule.case,msg.parts)) { + state.onward.push(msg); + state.elseflag = false; + if (node.checkall == "false") { + return resolve(false); + } + } else { + state.onward.push(null); + } + resolve(state.currentRule < node.rules.length - 1); + }); + }) + } + + function applyRules(node, msg, property,state) { + if (!state) { + state = { + currentRule: 0, + elseflag: true, + onward: [], + hasParts: msg.hasOwnProperty("parts") && + msg.parts.hasOwnProperty("id") && + msg.parts.hasOwnProperty("index") + } + } + return applyRule(node,msg,property,state).then(hasMore => { + if (hasMore) { + state.currentRule++; + return applyRules(node,msg,property,state); + } else { + node.previousValue = property; + return state.onward; + } + }); + } function SwitchNode(n) { @@ -248,23 +304,23 @@ module.exports = function(RED) { function addMessageToPending(msg) { var parts = msg.parts; - if (parts.hasOwnProperty("id") && - parts.hasOwnProperty("index")) { - var group = addMessageToGroup(parts.id, msg, parts); - var msgs = group.msgs; - var count = group.count; - if (count === msgs.length) { - for (var i = 0; i < msgs.length; i++) { - var msg = msgs[i]; + // We've already checked the msg.parts has the require bits + var group = addMessageToGroup(parts.id, msg, parts); + var msgs = group.msgs; + var count = group.count; + if (count === msgs.length) { + // We have a complete group - send the individual parts + return msgs.reduce((promise, msg) => { + return promise.then((result) => { msg.parts.count = count; - processMessage(msg, false); - } + return processMessage(msg, false); + }) + }, Promise.resolve()).then( () => { pendingCount -= group.msgs.length; delete pendingIn[parts.id]; - } - return true; + }); } - return false; + return Promise.resolve(); } function sendGroup(onwards, port_count) { @@ -332,103 +388,28 @@ module.exports = function(RED) { + + function processMessage(msg, checkParts) { var hasParts = msg.hasOwnProperty("parts") && msg.parts.hasOwnProperty("id") && msg.parts.hasOwnProperty("index"); - if (needsCount && checkParts && hasParts && - addMessageToPending(msg)) { - return; + if (needsCount && checkParts && hasParts) { + return addMessageToPending(msg); } - var onward = []; - try { - var prop; - - // getProperty - if (node.propertyType === 'jsonata') { - prop = RED.util.evaluateJSONataExpression(node.property,msg); - } else { - prop = RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg); - } - // end getProperty - - var elseflag = true; - for (var i=0; i applyRules(node,msg,property)) + .then(onward => { + if (!repair || !hasParts) { + node.send(onward); } - } else if (rule.vt === 'json') { - v1 = "json"; - } else if (rule.vt === 'null') { - v1 = "null"; - } else { - try { - v1 = RED.util.evaluateNodeProperty(rule.v,rule.vt,node,msg); - } catch(err) { - v1 = undefined; + else { + sendGroupMessages(onward, msg); } - } - //// end getV1 - - //// getV2 - v2 = rule.v2; - if (rule.v2t === 'prev') { - v2 = node.previousValue; - } else if (rule.v2t === 'jsonata') { - try { - v2 = RED.util.evaluateJSONataExpression(rule.v2,msg); - } catch(err) { - node.error(RED._("switch.errors.invalid-expr",{error:err.message})); - return; - } - } else if (typeof v2 !== 'undefined') { - try { - v2 = RED.util.evaluateNodeProperty(rule.v2,rule.v2t,node,msg); - } catch(err) { - v2 = undefined; - } - } - //// end getV2 - - - if (rule.t == "else") { test = elseflag; elseflag = true; } - if (operators[rule.t](test,v1,v2,rule.case,msg.parts)) { - onward.push(msg); - elseflag = false; - if (node.checkall == "false") { break; } - } else { - onward.push(null); - } - } - node.previousValue = prop; - if (!repair || !hasParts) { - node.send(onward); - } - else { - sendGroupMessages(onward, msg); - } - } catch(err) { - node.warn(err); - } + }).catch(err => { + node.warn(err); + }); } function clearPending() { @@ -439,8 +420,38 @@ module.exports = function(RED) { received = {}; } + var pendingMessages = []; + var activeMessagePromise = null; + var processMessageQueue = function(msg) { + if (msg) { + // A new message has arrived - add it to the message queue + pendingMessages.push(msg); + if (activeMessagePromise !== null) { + // The node is currently processing a message, so do nothing + // more with this message + return; + } + } + if (pendingMessages.length === 0) { + // There are no more messages to process, clear the active flag + // and return + activeMessagePromise = null; + return; + } + + // There are more messages to process. Get the next message and + // start processing it. Recurse back in to check for any more + var nextMsg = pendingMessages.shift(); + activeMessagePromise = processMessage(nextMsg,true) + .then(processMessageQueue) + .catch((err) => { + node.error(err,nextMsg); + return processMessageQueue(); + }); + } + this.on('input', function(msg) { - processMessage(msg, true); + processMessageQueue(msg, true); }); this.on('close', function() { diff --git a/red/runtime/util.js b/red/runtime/util.js index 6279c416c..7ffe737b3 100644 --- a/red/runtime/util.js +++ b/red/runtime/util.js @@ -350,7 +350,16 @@ function evaluateNodeProperty(value, type, node, msg, callback) { var data = JSON.parse(value); result = Buffer.from(data); } else if (type === 'msg' && msg) { - result = getMessageProperty(msg,value); + try { + result = getMessageProperty(msg,value); + } catch(err) { + if (callback) { + callback(err); + } else { + throw err; + } + return; + } } else if ((type === 'flow' || type === 'global') && node) { var contextKey = parseContextStore(value); result = node.context()[type].get(contextKey.key,contextKey.store,callback); @@ -366,7 +375,7 @@ function evaluateNodeProperty(value, type, node, msg, callback) { result = evaluteEnvProperty(value); } if (callback) { - callback(result); + callback(null,result); } else { return result; } diff --git a/test/nodes/core/logic/10-switch_spec.js b/test/nodes/core/logic/10-switch_spec.js index e655a3fba..5159bf42e 100644 --- a/test/nodes/core/logic/10-switch_spec.js +++ b/test/nodes/core/logic/10-switch_spec.js @@ -460,7 +460,7 @@ describe('switch Node', function() { } catch(err) { done(err); } - },100) + },500) }); }); @@ -599,7 +599,7 @@ describe('switch Node', function() { it('should take head of message sequence (w. context)', function(done) { var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload",rules:[{"t":"head","v":"count",vt:"global"}],checkall:false,repair:true,outputs:1,wires:[["helperNode1"]]}, {id:"helperNode1", type:"helper", wires:[]}]; - customFlowSequenceSwitchTest(flow, [0, 1, 2, 3, 4], [0, 1, 2], true, + customFlowSequenceSwitchTest(flow, [0, 1, 2, 3, 4], [0, 1, 2], true, function(node) { node.context().global.set("count", 3); }, done); @@ -642,7 +642,7 @@ describe('switch Node', function() { {id:"helperNode1", type:"helper", wires:[]}]; customFlowSwitchTest(flow, true, 9, done); }); - + it('should be able to use $I in JSONata expression', function(done) { var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"payload",rules:[{"t":"jsonata_exp","v":"$I % 2 = 1",vt:"jsonata"}],checkall:true,repair:true,outputs:1,wires:[["helperNode1"]]}, {id:"helperNode1", type:"helper", wires:[]}]; @@ -821,4 +821,24 @@ describe('switch Node', function() { n1.receive({payload:1, parts:{index:0, count:4, id:222}}); }); }); + + it('should handle invalid jsonata expression', function(done) { + + var flow = [{id:"switchNode1",type:"switch",name:"switchNode",property:"$invalidExpression(payload)",propertyType:"jsonata",rules:[{"t":"btwn","v":"$sqrt(16)","vt":"jsonata","v2":"$sqrt(36)","v2t":"jsonata"}],checkall:true,outputs:1,wires:[["helperNode1"]]}, + {id:"helperNode1", type:"helper", wires:[]}]; + helper.load(switchNode, flow, function() { + var n1 = helper.getNode("switchNode1"); + setTimeout(function() { + var logEvents = helper.log().args.filter(function (evt) { + return evt[0].type == "switch"; + }); + var evt = logEvents[0][0]; + evt.should.have.property('id', "switchNode1"); + evt.should.have.property('type', "switch"); + done(); + }, 150); + n1.receive({payload:1}); + }); + }); + }); From b0d7e11d482378440600cba48557ed688f9cb988 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 9 Jul 2018 12:40:25 +0100 Subject: [PATCH 10/22] Fix evaluateNodeProperty handling of unknown types --- red/runtime/util.js | 2 +- test/red/runtime/util_spec.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/red/runtime/util.js b/red/runtime/util.js index 7ffe737b3..f4c0e1fa7 100644 --- a/red/runtime/util.js +++ b/red/runtime/util.js @@ -335,7 +335,7 @@ var parseContextStore = function(key) { } function evaluateNodeProperty(value, type, node, msg, callback) { - var result; + var result = value; if (type === 'str') { result = ""+value; } else if (type === 'num') { diff --git a/test/red/runtime/util_spec.js b/test/red/runtime/util_spec.js index 1b7438efd..f7bd3e950 100644 --- a/test/red/runtime/util_spec.js +++ b/test/red/runtime/util_spec.js @@ -307,6 +307,10 @@ describe("red/util", function() { },{}); result.should.eql("123"); }); + it('returns null', function() { + var result = util.evaluateNodeProperty(null,'null'); + (result === null).should.be.true(); + }) describe('environment variable', function() { before(function() { process.env.NR_TEST_A = "foo"; From d7adff9a654f1a74b2dd0c845cf3ea155620f76c Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 9 Jul 2018 14:12:44 +0100 Subject: [PATCH 11/22] Add async message handling to Trigger node --- nodes/core/core/89-trigger.js | 187 ++++++++++++++++++------ nodes/core/logic/10-switch.js | 2 +- test/nodes/core/core/89-trigger_spec.js | 6 +- 3 files changed, 143 insertions(+), 52 deletions(-) diff --git a/nodes/core/core/89-trigger.js b/nodes/core/core/89-trigger.js index f5fd28b7f..48a595ccc 100644 --- a/nodes/core/core/89-trigger.js +++ b/nodes/core/core/89-trigger.js @@ -76,8 +76,43 @@ module.exports = function(RED) { var node = this; node.topics = {}; - this.on("input", function(msg) { + var pendingMessages = []; + var activeMessagePromise = null; + var processMessageQueue = function(msg) { + if (msg) { + // A new message has arrived - add it to the message queue + pendingMessages.push(msg); + if (activeMessagePromise !== null) { + // The node is currently processing a message, so do nothing + // more with this message + return; + } + } + if (pendingMessages.length === 0) { + // There are no more messages to process, clear the active flag + // and return + activeMessagePromise = null; + return; + } + + // There are more messages to process. Get the next message and + // start processing it. Recurse back in to check for any more + var nextMsg = pendingMessages.shift(); + activeMessagePromise = processMessage(nextMsg) + .then(processMessageQueue) + .catch((err) => { + node.error(err,nextMsg); + return processMessageQueue(); + }); + } + + this.on('input', function(msg) { + processMessageQueue(msg); + }); + + var processMessage = function(msg) { var topic = msg.topic || "_none"; + var promise; if (node.bytopic === "all") { topic = "_none"; } node.topics[topic] = node.topics[topic] || {}; if (msg.hasOwnProperty("reset") || ((node.reset !== '') && msg.hasOwnProperty("payload") && (msg.payload !== null) && msg.payload.toString && (msg.payload.toString() == node.reset)) ) { @@ -88,48 +123,88 @@ module.exports = function(RED) { } else { if (((!node.topics[topic].tout) && (node.topics[topic].tout !== 0)) || (node.loop === true)) { + promise = Promise.resolve(); if (node.op2type === "pay" || node.op2type === "payl") { node.topics[topic].m2 = RED.util.cloneMessage(msg.payload); } else if (node.op2Templated) { node.topics[topic].m2 = mustache.render(node.op2,msg); } else if (node.op2type !== "nul") { - node.topics[topic].m2 = RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg); - } - - if (node.op1type === "pay") { } - else if (node.op1Templated) { msg.payload = mustache.render(node.op1,msg); } - else if (node.op1type !== "nul") { - msg.payload = RED.util.evaluateNodeProperty(node.op1,node.op1type,node,msg); - } - - if (node.duration === 0) { node.topics[topic].tout = 0; } - else if (node.loop === true) { - /* istanbul ignore else */ - if (node.topics[topic].tout) { clearInterval(node.topics[topic].tout); } - /* istanbul ignore else */ - if (node.op1type !== "nul") { - var msg2 = RED.util.cloneMessage(msg); - node.topics[topic].tout = setInterval(function() { node.send(RED.util.cloneMessage(msg2)); }, node.duration); - } - } - else { - if (!node.topics[topic].tout) { - node.topics[topic].tout = setTimeout(function() { - var msg2 = null; - if (node.op2type !== "nul") { - msg2 = RED.util.cloneMessage(msg); - if (node.op2type === "flow" || node.op2type === "global") { - node.topics[topic].m2 = RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg); - } - msg2.payload = node.topics[topic].m2; - delete node.topics[topic]; - node.send(msg2); + promise = new Promise((resolve,reject) => { + RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg,(err,value) => { + if (err) { + reject(err); + } else { + node.topics[topic].m2 = value; + resolve(); } - else { delete node.topics[topic]; } - node.status({}); - }, node.duration); - } + }); + }); } - node.status({fill:"blue",shape:"dot",text:" "}); - if (node.op1type !== "nul") { node.send(RED.util.cloneMessage(msg)); } + + return promise.then(() => { + promise = Promise.resolve(); + if (node.op1type === "pay") { } + else if (node.op1Templated) { msg.payload = mustache.render(node.op1,msg); } + else if (node.op1type !== "nul") { + promise = new Promise((resolve,reject) => { + RED.util.evaluateNodeProperty(node.op1,node.op1type,node,msg,(err,value) => { + if (err) { + reject(err); + } else { + msg.payload = value; + resolve(); + } + }); + }); + } + return promise.then(() => { + if (node.duration === 0) { node.topics[topic].tout = 0; } + else if (node.loop === true) { + /* istanbul ignore else */ + if (node.topics[topic].tout) { clearInterval(node.topics[topic].tout); } + /* istanbul ignore else */ + if (node.op1type !== "nul") { + var msg2 = RED.util.cloneMessage(msg); + node.topics[topic].tout = setInterval(function() { node.send(RED.util.cloneMessage(msg2)); }, node.duration); + } + } + else { + if (!node.topics[topic].tout) { + node.topics[topic].tout = setTimeout(function() { + var msg2 = null; + if (node.op2type !== "nul") { + var promise = Promise.resolve(); + msg2 = RED.util.cloneMessage(msg); + if (node.op2type === "flow" || node.op2type === "global") { + promise = new Promise((resolve,reject) => { + RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg,(err,value) => { + if (err) { + reject(err); + } else { + node.topics[topic].m2 = value; + resolve(); + } + }); + }); + } + promise.then(() => { + msg2.payload = node.topics[topic].m2; + delete node.topics[topic]; + node.send(msg2); + node.status({}); + }).catch(err => { + node.error(err); + }); + } else { + delete node.topics[topic]; + node.status({}); + } + + }, node.duration); + } + } + node.status({fill:"blue",shape:"dot",text:" "}); + if (node.op1type !== "nul") { node.send(RED.util.cloneMessage(msg)); } + }); + }); } else if ((node.extend === "true" || node.extend === true) && (node.duration > 0)) { /* istanbul ignore else */ @@ -138,25 +213,43 @@ module.exports = function(RED) { if (node.topics[topic].tout) { clearTimeout(node.topics[topic].tout); } node.topics[topic].tout = setTimeout(function() { var msg2 = null; + var promise = Promise.resolve(); + if (node.op2type !== "nul") { if (node.op2type === "flow" || node.op2type === "global") { - node.topics[topic].m2 = RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg); - } - if (node.topics[topic] !== undefined) { - msg2 = RED.util.cloneMessage(msg); - msg2.payload = node.topics[topic].m2; + promise = new Promise((resolve,reject) => { + RED.util.evaluateNodeProperty(node.op2,node.op2type,node,msg,(err,value) => { + if (err) { + reject(err); + } else { + node.topics[topic].m2 = value; + resolve(); + } + }); + }); } } - delete node.topics[topic]; - node.status({}); - node.send(msg2); + promise.then(() => { + if (node.op2type !== "nul") { + if (node.topics[topic] !== undefined) { + msg2 = RED.util.cloneMessage(msg); + msg2.payload = node.topics[topic].m2; + } + } + delete node.topics[topic]; + node.status({}); + node.send(msg2); + }).catch(err => { + node.error(err); + }); }, node.duration); } else { if (node.op2type === "payl") { node.topics[topic].m2 = RED.util.cloneMessage(msg.payload); } } } - }); + return Promise.resolve(); + } this.on("close", function() { for (var t in node.topics) { /* istanbul ignore else */ diff --git a/nodes/core/logic/10-switch.js b/nodes/core/logic/10-switch.js index d15be39de..79b6e20b3 100644 --- a/nodes/core/logic/10-switch.js +++ b/nodes/core/logic/10-switch.js @@ -451,7 +451,7 @@ module.exports = function(RED) { } this.on('input', function(msg) { - processMessageQueue(msg, true); + processMessageQueue(msg); }); this.on('close', function() { diff --git a/test/nodes/core/core/89-trigger_spec.js b/test/nodes/core/core/89-trigger_spec.js index d07ab2a9b..f19bc42fe 100644 --- a/test/nodes/core/core/89-trigger_spec.js +++ b/test/nodes/core/core/89-trigger_spec.js @@ -288,7 +288,7 @@ describe('trigger node', function() { it('should be able to return things from flow and global context variables', function(done) { var spy = sinon.stub(RED.util, 'evaluateNodeProperty', - function(arg1, arg2, arg3, arg4) { return arg1; } + function(arg1, arg2, arg3, arg4, arg5) { if (arg5) { arg5(null, arg1) } else { return arg1; } } ); var flow = [{"id":"n1", "type":"trigger", "name":"triggerNode", op1:"foo", op1type:"flow", op2:"bar", op2type:"global", duration:"20", wires:[["n2"]] }, {id:"n2", type:"helper"} ]; @@ -386,7 +386,7 @@ describe('trigger node', function() { it('should be able to extend the delay', function(done) { this.timeout(5000); // add extra time for flake var spy = sinon.stub(RED.util, 'evaluateNodeProperty', - function(arg1, arg2, arg3, arg4) { return arg1; } + function(arg1, arg2, arg3, arg4, arg5) { if (arg5) { arg5(null, arg1) } else { return arg1; } } ); var flow = [{"id":"n1", "type":"trigger", "name":"triggerNode", extend:"true", op1type:"flow", op1:"foo", op2:"bar", op2type:"global", duration:"100", wires:[["n2"]] }, {id:"n2", type:"helper"} ]; @@ -428,12 +428,10 @@ describe('trigger node', function() { n2.on("input", function(msg) { try { if (c === 0) { - console.log(c,Date.now() - ss,msg); msg.should.have.a.property("payload", "Hello"); c += 1; } else { - console.log(c,Date.now() - ss,msg); msg.should.have.a.property("payload", "World"); (Date.now() - ss).should.be.greaterThan(150); done(); From b2f06b6777acba3de5e70efe969c355450dfcbc1 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 9 Jul 2018 15:12:09 +0100 Subject: [PATCH 12/22] Add async mode to evaluateJSONataExpression --- red/runtime/util.js | 33 +++++++++++++++++++++++++++++++-- test/red/runtime/util_spec.js | 24 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/red/runtime/util.js b/red/runtime/util.js index f4c0e1fa7..b6a5a7e7b 100644 --- a/red/runtime/util.js +++ b/red/runtime/util.js @@ -394,15 +394,44 @@ function prepareJSONataExpression(value,node) { }) expr.registerFunction('clone', cloneMessage, '<(oa)-:o>'); expr._legacyMode = /(^|[^a-zA-Z0-9_'"])msg([^a-zA-Z0-9_'"]|$)/.test(value); + expr._node = node; return expr; } -function evaluateJSONataExpression(expr,msg) { +function evaluateJSONataExpression(expr,msg,callback) { var context = msg; if (expr._legacyMode) { context = {msg:msg}; } - return expr.evaluate(context); + var bindings = {}; + + if (callback) { + // If callback provided, need to override the pre-assigned sync + // context functions to be their async variants + bindings.flowContext = function(val) { + return new Promise((resolve,reject) => { + expr._node.context().flow.get(val, function(err,value) { + if (err) { + reject(err); + } else { + resolve(value); + } + }) + }); + } + bindings.globalContext = function(val) { + return new Promise((resolve,reject) => { + expr._node.context().global.get(val, function(err,value) { + if (err) { + reject(err); + } else { + resolve(value); + } + }) + }); + } + } + return expr.evaluate(context, bindings, callback); } diff --git a/test/red/runtime/util_spec.js b/test/red/runtime/util_spec.js index f7bd3e950..1be194f42 100644 --- a/test/red/runtime/util_spec.js +++ b/test/red/runtime/util_spec.js @@ -458,6 +458,30 @@ describe("red/util", function() { var result = util.evaluateJSONataExpression(expr,{payload:"hello"}); should.not.exist(result); }); + it('handles async flow context access', function(done) { + var expr = util.prepareJSONataExpression('$flowContext("foo")',{context:function() { return {flow:{get: function(key,callback) { setTimeout(()=>{callback(null,{'foo':'bar'}[key])},10)}}}}}); + util.evaluateJSONataExpression(expr,{payload:"hello"},function(err,value) { + try { + should.not.exist(err); + value.should.eql("bar"); + done(); + } catch(err2) { + done(err2); + } + }); + }) + it('handles async global context access', function(done) { + var expr = util.prepareJSONataExpression('$globalContext("foo")',{context:function() { return {global:{get: function(key,callback) { setTimeout(()=>{callback(null,{'foo':'bar'}[key])},10)}}}}}); + util.evaluateJSONataExpression(expr,{payload:"hello"},function(err,value) { + try { + should.not.exist(err); + value.should.eql("bar"); + done(); + } catch(err2) { + done(err2); + } + }); + }) }); From 807b512ef7feab6e2bef10de47a535f20005a9ac Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 9 Jul 2018 21:56:39 +0100 Subject: [PATCH 13/22] Add JSONata async support to Switch and Change nodes --- nodes/core/logic/10-switch.js | 47 +++--- nodes/core/logic/15-change.js | 293 ++++++++++++++++++++-------------- 2 files changed, 195 insertions(+), 145 deletions(-) diff --git a/nodes/core/logic/10-switch.js b/nodes/core/logic/10-switch.js index 79b6e20b3..df5a52023 100644 --- a/nodes/core/logic/10-switch.js +++ b/nodes/core/logic/10-switch.js @@ -77,12 +77,13 @@ module.exports = function(RED) { function getProperty(node,msg) { return new Promise((resolve,reject) => { if (node.propertyType === 'jsonata') { - try { - resolve(RED.util.evaluateJSONataExpression(node.property,msg)); - } catch(err) { - // TODO: proper invalid expr message - reject(err); - } + RED.util.evaluateJSONataExpression(node.property,msg,(err,value) => { + if (err) { + reject(RED._("switch.errors.invalid-expr",{error:err.message})); + } else { + resolve(value); + } + }); } else { RED.util.evaluateNodeProperty(node.property,node.propertyType,node,msg,(err,value) => { if (err) { @@ -100,18 +101,20 @@ module.exports = function(RED) { if (rule.vt === 'prev') { resolve(node.previousValue); } else if (rule.vt === 'jsonata') { - try { - var exp = rule.v; - if (rule.t === 'jsonata_exp') { - if (hasParts) { - exp.assign("I", msg.parts.index); - exp.assign("N", msg.parts.count); - } + var exp = rule.v; + if (rule.t === 'jsonata_exp') { + if (hasParts) { + exp.assign("I", msg.parts.index); + exp.assign("N", msg.parts.count); } - resolve(RED.util.evaluateJSONataExpression(exp,msg)); - } catch(err) { - reject(RED._("switch.errors.invalid-expr",{error:err.message})); } + RED.util.evaluateJSONataExpression(exp,msg,(err,value) => { + if (err) { + reject(RED._("switch.errors.invalid-expr",{error:err.message})); + } else { + resolve(value); + } + }); } else if (rule.vt === 'json') { resolve("json"); } else if (rule.vt === 'null') { @@ -134,11 +137,13 @@ module.exports = function(RED) { if (rule.v2t === 'prev') { resolve(node.previousValue); } else if (rule.v2t === 'jsonata') { - try { - resolve(RED.util.evaluateJSONataExpression(rule.v2,msg)); - } catch(err) { - reject(RED._("switch.errors.invalid-expr",{error:err.message})); - } + RED.util.evaluateJSONataExpression(rule.v2,msg,(err,value) => { + if (err) { + reject(RED._("switch.errors.invalid-expr",{error:err.message})); + } else { + resolve(value); + } + }); } else if (typeof v2 !== 'undefined') { RED.util.evaluateNodeProperty(rule.v2,rule.v2t,node,msg, function(err,value) { if (err) { diff --git a/nodes/core/logic/15-change.js b/nodes/core/logic/15-change.js index 4094f68e0..d98bf6bde 100644 --- a/nodes/core/logic/15-change.js +++ b/nodes/core/logic/15-change.js @@ -98,138 +98,183 @@ module.exports = function(RED) { } } - function applyRule(msg,rule) { - return new Promise(function(resolve, reject){ - try { - var property = rule.p; - var value = rule.to; - if (rule.tot === 'json') { - value = JSON.parse(rule.to); - } else if (rule.tot === 'bin') { - value = Buffer.from(JSON.parse(rule.to)) - } - var current; - var fromValue; - var fromType; - var fromRE; - if (rule.tot === "msg") { - value = RED.util.getMessageProperty(msg,rule.to); - } else if (rule.tot === 'flow') { - value = node.context().flow.get(rule.to); - } else if (rule.tot === 'global') { - value = node.context().global.get(rule.to); - } else if (rule.tot === 'date') { - value = Date.now(); - } else if (rule.tot === 'jsonata') { - try{ - value = RED.util.evaluateJSONataExpression(rule.to,msg); - } catch(err) { - node.error(RED._("change.errors.invalid-expr",{error:err.message}),msg); - return; + function getToValue(msg,rule) { + var value = rule.to; + if (rule.tot === 'json') { + value = JSON.parse(rule.to); + } else if (rule.tot === 'bin') { + value = Buffer.from(JSON.parse(rule.to)) + } + if (rule.tot === "msg") { + value = RED.util.getMessageProperty(msg,rule.to); + } else if (rule.tot === 'flow') { + value = node.context().flow.get(rule.to); + } else if (rule.tot === 'global') { + value = node.context().global.get(rule.to); + } else if (rule.tot === 'date') { + value = Date.now(); + } else if (rule.tot === 'jsonata') { + return new Promise((resolve,reject) => { + RED.util.evaluateJSONataExpression(rule.to,msg, (err, value) => { + if (err) { + reject(RED._("change.errors.invalid-expr",{error:err.message})) + } else { + resolve(value); } - } - if (rule.t === 'change') { - if (rule.fromt === 'msg' || rule.fromt === 'flow' || rule.fromt === 'global') { - if (rule.fromt === "msg") { - fromValue = RED.util.getMessageProperty(msg,rule.from); - } else if (rule.fromt === 'flow') { - fromValue = node.context().flow.get(rule.from); - } else if (rule.fromt === 'global') { - fromValue = node.context().global.get(rule.from); - } - if (typeof fromValue === 'number' || fromValue instanceof Number) { - fromType = 'num'; - } else if (typeof fromValue === 'boolean') { - fromType = 'bool' - } else if (fromValue instanceof RegExp) { - fromType = 're'; - fromRE = fromValue; - } else if (typeof fromValue === 'string') { - fromType = 'str'; - fromRE = fromValue.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); - try { - fromRE = new RegExp(fromRE, "g"); - } catch (e) { - valid = false; - reject(RED._("change.errors.invalid-from",{error:e.message})); - return; + }); + }); + } + return Promise.resolve(value); + } + function getFromValue(msg,rule) { + var fromValue; + var fromType; + var fromRE; + if (rule.t === 'change') { + if (rule.fromt === 'msg' || rule.fromt === 'flow' || rule.fromt === 'global') { + return new Promise((resolve,reject) => { + if (rule.fromt === "msg") { + resolve(RED.util.getMessageProperty(msg,rule.from)); + } else if (rule.fromt === 'flow' || rule.fromt === 'global') { + node.context()[rule.fromt].get(rule.from,(err,fromValue) => { + if (err) { + reject(err); + } else { + resolve(fromValue); } - } else { - reject(RED._("change.errors.invalid-from",{error:"unsupported type: "+(typeof fromValue)})); - return + }); + } + }).then(fromValue => { + if (typeof fromValue === 'number' || fromValue instanceof Number) { + fromType = 'num'; + } else if (typeof fromValue === 'boolean') { + fromType = 'bool' + } else if (fromValue instanceof RegExp) { + fromType = 're'; + fromRE = fromValue; + } else if (typeof fromValue === 'string') { + fromType = 'str'; + fromRE = fromValue.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); + try { + fromRE = new RegExp(fromRE, "g"); + } catch (e) { + reject(new Error(RED._("change.errors.invalid-from",{error:e.message}))); + return; } } else { - fromType = rule.fromt; - fromValue = rule.from; - fromRE = rule.fromRE; + reject(new Error(RED._("change.errors.invalid-from",{error:"unsupported type: "+(typeof fromValue)}))); + return; } - } - if (rule.pt === 'msg') { - if (rule.t === 'delete') { - RED.util.setMessageProperty(msg,property,undefined); - } else if (rule.t === 'set') { - RED.util.setMessageProperty(msg,property,value); - } else if (rule.t === 'change') { - current = RED.util.getMessageProperty(msg,property); - if (typeof current === 'string') { - if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { - // str representation of exact from number/boolean - // only replace if they match exactly - RED.util.setMessageProperty(msg,property,value); - } else { - current = current.replace(fromRE,value); - RED.util.setMessageProperty(msg,property,current); - } - } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { - if (current == Number(fromValue)) { - RED.util.setMessageProperty(msg,property,value); - } - } else if (typeof current === 'boolean' && fromType === 'bool') { - if (current.toString() === fromValue) { - RED.util.setMessageProperty(msg,property,value); - } - } + return { + fromType, + fromValue, + fromRE } - } - else { - var target; - if (rule.pt === 'flow') { - target = node.context().flow; - } else if (rule.pt === 'global') { - target = node.context().global; - } - if (target) { - if (rule.t === 'delete') { - target.set(property,undefined); - } else if (rule.t === 'set') { - target.set(property,value); - } else if (rule.t === 'change') { - current = target.get(property); - if (typeof current === 'string') { - if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { - // str representation of exact from number/boolean - // only replace if they match exactly - target.set(property,value); - } else { - current = current.replace(fromRE,value); - target.set(property,current); - } - } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { - if (current == Number(fromValue)) { - target.set(property,value); - } - } else if (typeof current === 'boolean' && fromType === 'bool') { - if (current.toString() === fromValue) { - target.set(property,value); - } - } - } - } - } - } catch(err) {/*console.log(err.stack)*/} - resolve(msg); + }); + } else { + fromType = rule.fromt; + fromValue = rule.from; + fromRE = rule.fromRE; + } + } + return Promise.resolve({ + fromType, + fromValue, + fromRE }); } + function applyRule(msg,rule) { + var property = rule.p; + var current; + var fromValue; + var fromType; + var fromRE; + try { + return getToValue(msg,rule).then(value => { + return getFromValue(msg,rule).then(fromParts => { + fromValue = fromParts.fromValue; + fromType = fromParts.fromType; + fromRE = fromParts.fromRE; + if (rule.pt === 'msg') { + try { + if (rule.t === 'delete') { + RED.util.setMessageProperty(msg,property,undefined); + } else if (rule.t === 'set') { + RED.util.setMessageProperty(msg,property,value); + } else if (rule.t === 'change') { + current = RED.util.getMessageProperty(msg,property); + if (typeof current === 'string') { + if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { + // str representation of exact from number/boolean + // only replace if they match exactly + RED.util.setMessageProperty(msg,property,value); + } else { + current = current.replace(fromRE,value); + RED.util.setMessageProperty(msg,property,current); + } + } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { + if (current == Number(fromValue)) { + RED.util.setMessageProperty(msg,property,value); + } + } else if (typeof current === 'boolean' && fromType === 'bool') { + if (current.toString() === fromValue) { + RED.util.setMessageProperty(msg,property,value); + } + } + } + } catch(err) {} + return msg; + } else if (rule.pt === 'flow' || rule.pt === 'global') { + return new Promise((resolve,reject) => { + var target = node.context()[rule.pt]; + var callback = err => { + if (err) { + reject(err); + } else { + resolve(msg); + } + } + if (rule.t === 'delete') { + target.set(property,undefined,callback); + } else if (rule.t === 'set') { + target.set(property,value,callback); + } else if (rule.t === 'change') { + target.get(property,(err,current) => { + if (err) { + reject(err); + return; + } + if (typeof current === 'string') { + if ((fromType === 'num' || fromType === 'bool' || fromType === 'str') && current === fromValue) { + // str representation of exact from number/boolean + // only replace if they match exactly + target.set(property,value,callback); + } else { + current = current.replace(fromRE,value); + target.set(property,current,callback); + } + } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') { + if (current == Number(fromValue)) { + target.set(property,value,callback); + } + } else if (typeof current === 'boolean' && fromType === 'bool') { + if (current.toString() === fromValue) { + target.set(property,value,callback); + } + } + }); + } + }); + } + }); + }).catch(err => { + node.error(err, msg); + return null; + }); + } catch(err) { + return Promise.resolve(msg); + } + } function applyRules(msg, currentRule) { var r = node.rules[currentRule]; var rulePromise; From d8d82e2ba3f617545fcc8d1fa38945d5e0575538 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 9 Jul 2018 23:06:51 +0100 Subject: [PATCH 14/22] Update sort node for async use of jsonata --- nodes/core/logic/18-sort.js | 216 ++++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 98 deletions(-) diff --git a/nodes/core/logic/18-sort.js b/nodes/core/logic/18-sort.js index e30535dc1..a3023e5f1 100644 --- a/nodes/core/logic/18-sort.js +++ b/nodes/core/logic/18-sort.js @@ -17,7 +17,7 @@ module.exports = function(RED) { "use strict"; - var _max_kept_msgs_count = undefined; + var _max_kept_msgs_count; function max_kept_msgs_count(node) { if (_max_kept_msgs_count === undefined) { @@ -32,30 +32,20 @@ module.exports = function(RED) { return _max_kept_msgs_count; } - function eval_jsonata(node, code, val) { - try { - return RED.util.evaluateJSONataExpression(code, val); - } - catch (e) { - node.error(RED._("sort.invalid-exp")); - throw e; - } - } - - function get_context_val(node, name, dval) { - var context = node.context(); - var val = context.get(name); - if (val === undefined) { - context.set(name, dval); - return dval; - } - return val; - } + // function get_context_val(node, name, dval) { + // var context = node.context(); + // var val = context.get(name); + // if (val === undefined) { + // context.set(name, dval); + // return dval; + // } + // return val; + // } function SortNode(n) { RED.nodes.createNode(this, n); var node = this; - var pending = get_context_val(node, 'pending', {}) + var pending = {};//get_context_val(node, 'pending', {}) var pending_count = 0; var pending_id = 0; var order = n.order || "ascending"; @@ -76,11 +66,10 @@ module.exports = function(RED) { } } var dir = (order === "ascending") ? 1 : -1; - var conv = as_num - ? function(x) { return Number(x); } - : function(x) { return x; }; + var conv = as_num ? function(x) { return Number(x); } + : function(x) { return x; }; - function gen_comp(key) { + function generateComparisonFunction(key) { return function(x, y) { var xp = conv(key(x)); var yp = conv(key(y)); @@ -90,74 +79,97 @@ module.exports = function(RED) { }; } - function send_group(group) { - var key = key_is_exp - ? function(msg) { - return eval_jsonata(node, key_exp, msg); - } - : function(msg) { - return RED.util.getMessageProperty(msg, key_prop); - }; - var comp = gen_comp(key); + function sortMessageGroup(group) { + var promise; var msgs = group.msgs; - try { - msgs.sort(comp); - } - catch (e) { - return; // not send when error - } - for (var i = 0; i < msgs.length; i++) { - var msg = msgs[i]; - msg.parts.index = i; - node.send(msg); - } - } - - function sort_payload(msg) { - var data = RED.util.getMessageProperty(msg, target_prop); - if (Array.isArray(data)) { - var key = key_is_exp - ? function(elem) { - return eval_jsonata(node, key_exp, elem); - } - : function(elem) { return elem; }; - var comp = gen_comp(key); + if (key_is_exp) { + var evaluatedDataPromises = msgs.map(msg => { + return new Promise((resolve,reject) => { + RED.util.evaluateJSONataExpression(key_exp, msg, (err, result) => { + resolve({ + item: msg, + sortValue: result + }) + }); + }) + }); + promise = Promise.all(evaluatedDataPromises).then(evaluatedElements => { + // Once all of the sort keys are evaluated, sort by them + var comp = generateComparisonFunction(elem=>elem.sortValue); + return evaluatedElements.sort(comp).map(elem=>elem.item); + }); + } else { + var key = function(msg) { + return ; + } + var comp = generateComparisonFunction(msg => RED.util.getMessageProperty(msg, key_prop)); try { - data.sort(comp); + msgs.sort(comp); } catch (e) { - return false; + return; // not send when error } - return true; + promise = Promise.resolve(msgs); } - return false; + return promise.then(msgs => { + for (var i = 0; i < msgs.length; i++) { + var msg = msgs[i]; + msg.parts.index = i; + node.send(msg); + } + }); } - function check_parts(parts) { - if (parts.hasOwnProperty("id") && - parts.hasOwnProperty("index")) { - return true; + function sortMessageProperty(msg) { + var data = RED.util.getMessageProperty(msg, target_prop); + if (Array.isArray(data)) { + if (key_is_exp) { + // key is an expression. Evaluated the expression for each item + // to get its sort value. As this could be async, need to do + // it first. + var evaluatedDataPromises = data.map(elem => { + return new Promise((resolve,reject) => { + RED.util.evaluateJSONataExpression(key_exp, elem, (err, result) => { + resolve({ + item: elem, + sortValue: result + }) + }); + }) + }) + return Promise.all(evaluatedDataPromises).then(evaluatedElements => { + // Once all of the sort keys are evaluated, sort by them + // and reconstruct the original message item with the newly + // sorted values. + var comp = generateComparisonFunction(elem=>elem.sortValue); + data = evaluatedElements.sort(comp).map(elem=>elem.item); + RED.util.setMessageProperty(msg, target_prop,data); + return true; + }) + } else { + var comp = generateComparisonFunction(elem=>elem); + try { + data.sort(comp); + } catch (e) { + return Promise.resolve(false); + } + return Promise.resolve(true); + } } - return false; + return Promise.resolve(false); } - function clear_pending() { + function removeOldestPending() { + var oldest; + var oldest_key; for(var key in pending) { - node.log(RED._("sort.clear"), pending[key].msgs[0]); - delete pending[key]; - } - pending_count = 0; - } - - function remove_oldest_pending() { - var oldest = undefined; - var oldest_key = undefined; - for(var key in pending) { - var item = pending[key]; - if((oldest === undefined) || - (oldest.seq_no > item.seq_no)) { - oldest = item; - oldest_key = key; + if (pending.hasOwnProperty(key)) { + var item = pending[key]; + if((oldest === undefined) || + (oldest.seq_no > item.seq_no)) { + oldest = item; + oldest_key = key; + } } } if(oldest !== undefined) { @@ -166,16 +178,18 @@ module.exports = function(RED) { } return 0; } - - function process_msg(msg) { + + function processMessage(msg) { if (target_is_prop) { - if (sort_payload(msg)) { - node.send(msg); - } - return; + sortMessageProperty(msg).then(send => { + if (send) { + node.send(msg); + } + }).catch(err => { + }); } var parts = msg.parts; - if (!check_parts(parts)) { + if (!parts.hasOwnProperty("id") || !parts.hasOwnProperty("index")) { return; } var gid = parts.id; @@ -195,23 +209,29 @@ module.exports = function(RED) { pending_count++; if (group.count === msgs.length) { delete pending[gid] - send_group(group); + sortMessageGroup(group); pending_count -= msgs.length; - } - var max_msgs = max_kept_msgs_count(node); - if ((max_msgs > 0) && (pending_count > max_msgs)) { - pending_count -= remove_oldest_pending(); - node.error(RED._("sort.too-many"), msg); + } else { + var max_msgs = max_kept_msgs_count(node); + if ((max_msgs > 0) && (pending_count > max_msgs)) { + pending_count -= removeOldestPending(); + node.error(RED._("sort.too-many"), msg); + } } } - + this.on("input", function(msg) { - process_msg(msg); + processMessage(msg); }); this.on("close", function() { - clear_pending(); - }) + for(var key in pending) { + if (pending.hasOwnProperty(key)) { + node.log(RED._("sort.clear"), pending[key].msgs[0]); + delete pending[key]; + } + } + pending_count = 0; }) } RED.nodes.registerType("sort", SortNode); From 57c1524a9a5de597c5d94e8c67d4390229c4c61f Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Tue, 10 Jul 2018 11:24:57 +0100 Subject: [PATCH 15/22] Add async jsonata support to join node --- nodes/core/logic/17-split.js | 201 +++++++++++++++++++++++------------ 1 file changed, 132 insertions(+), 69 deletions(-) diff --git a/nodes/core/logic/17-split.js b/nodes/core/logic/17-split.js index 92b527d01..73231335f 100644 --- a/nodes/core/logic/17-split.js +++ b/nodes/core/logic/17-split.js @@ -233,7 +233,7 @@ module.exports = function(RED) { RED.nodes.registerType("split",SplitNode); - var _max_kept_msgs_count = undefined; + var _max_kept_msgs_count; function max_kept_msgs_count(node) { if (_max_kept_msgs_count === undefined) { @@ -252,7 +252,15 @@ module.exports = function(RED) { exp.assign("I", index); exp.assign("N", count); exp.assign("A", accum); - return RED.util.evaluateJSONataExpression(exp, msg); + return new Promise((resolve,reject) => { + RED.util.evaluateJSONataExpression(exp, msg, (err, result) => { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + }); } function apply_f(exp, accum, count) { @@ -269,32 +277,37 @@ module.exports = function(RED) { return exp } - function reduce_and_send_group(node, group) { + function reduceAndSendGroup(node, group) { var is_right = node.reduce_right; var flag = is_right ? -1 : 1; var msgs = group.msgs; - var accum = eval_exp(node, node.exp_init, node.exp_init_type); - var reduce_exp = node.reduce_exp; - var reduce_fixup = node.reduce_fixup; - var count = group.count; - msgs.sort(function(x,y) { - var ix = x.parts.index; - var iy = y.parts.index; - if (ix < iy) return -flag; - if (ix > iy) return flag; - return 0; + return getInitialReduceValue(node, node.exp_init, node.exp_init_type).then(accum => { + var reduce_exp = node.reduce_exp; + var reduce_fixup = node.reduce_fixup; + var count = group.count; + msgs.sort(function(x,y) { + var ix = x.parts.index; + var iy = y.parts.index; + if (ix < iy) {return -flag;} + if (ix > iy) {return flag;} + return 0; + }); + + return msgs.reduce((promise, msg) => promise.then(accum => apply_r(reduce_exp, accum, msg, msg.parts.index, count)), Promise.resolve(accum)) + .then(accum => { + if(reduce_fixup !== undefined) { + accum = apply_f(reduce_fixup, accum, count); + } + node.send({payload: accum}); + }); + }).catch(err => { + throw new Error(RED._("join.errors.invalid-expr",{error:e.message})); }); - for(var msg of msgs) { - accum = apply_r(reduce_exp, accum, msg, msg.parts.index, count); - } - if(reduce_fixup !== undefined) { - accum = apply_f(reduce_fixup, accum, count); - } - node.send({payload: accum}); } function reduce_msg(node, msg) { - if(msg.hasOwnProperty('parts')) { + var promise; + if (msg.hasOwnProperty('parts')) { var parts = msg.parts; var pending = node.pending; var pending_count = node.pending_count; @@ -312,65 +325,82 @@ module.exports = function(RED) { var group = pending[gid]; var msgs = group.msgs; if(parts.hasOwnProperty('count') && - (group.count === undefined)) { + (group.count === undefined)) { group.count = count; } msgs.push(msg); pending_count++; + var completeProcess = function() { + node.pending_count = pending_count; + var max_msgs = max_kept_msgs_count(node); + if ((max_msgs > 0) && (pending_count > max_msgs)) { + node.pending = {}; + node.pending_count = 0; + var promise = Promise.reject(RED._("join.too-many")); + promise.catch(()=>{}); + return promise; + } + return Promise.resolve(); + } if(msgs.length === group.count) { delete pending[gid]; - try { - pending_count -= msgs.length; - reduce_and_send_group(node, group); - } catch(e) { - node.error(RED._("join.errors.invalid-expr",{error:e.message})); } + pending_count -= msgs.length; + promise = reduceAndSendGroup(node, group).then(completeProcess); + } else { + promise = completeProcess(); } - node.pending_count = pending_count; - var max_msgs = max_kept_msgs_count(node); - if ((max_msgs > 0) && (pending_count > max_msgs)) { - node.pending = {}; - node.pending_count = 0; - node.error(RED._("join.too-many"), msg); - } - } - else { + } else { node.send(msg); } + if (!promise) { + promise = Promise.resolve(); + } + return promise; } - function eval_exp(node, exp, exp_type) { - if(exp_type === "flow") { - return node.context().flow.get(exp); - } - else if(exp_type === "global") { - return node.context().global.get(exp); - } - else if(exp_type === "str") { - return exp; - } - else if(exp_type === "num") { - return Number(exp); - } - else if(exp_type === "bool") { - if (exp === 'true') { - return true; + function getInitialReduceValue(node, exp, exp_type) { + return new Promise((resolve,reject) => { + if(exp_type === "flow" || exp_type === "global") { + node.context()[exp_type].get(exp,(err,value) => { + if (err) { + reject(err); + } else { + resolve(value); + } + }); + return; + } else if(exp_type === "jsonata") { + var jexp = RED.util.prepareJSONataExpression(exp, node); + RED.util.evaluateJSONataExpression(jexp, {},(err,value) => { + if (err) { + reject(err); + } else { + resolve(value); + } + }); + return; } - else if (exp === 'false') { - return false; + var result; + if(exp_type === "str") { + result = exp; + } else if(exp_type === "num") { + result = Number(exp); + } else if(exp_type === "bool") { + if (exp === 'true') { + result = true; + } else if (exp === 'false') { + result = false; + } + } else if ((exp_type === "bin") || (exp_type === "json")) { + result = JSON.parse(exp); + } else if(exp_type === "date") { + result = Date.now(); + } else { + reject(new Error("unexpected initial value type")); + return; } - } - else if ((exp_type === "bin") || - (exp_type === "json")) { - return JSON.parse(exp); - } - else if(exp_type === "date") { - return Date.now(); - } - else if(exp_type === "jsonata") { - var jexp = RED.util.prepareJSONataExpression(exp, node); - return RED.util.evaluateJSONataExpression(jexp, {}); - } - throw new Error("unexpected initial value type"); + resolve(result); + }); } function JoinNode(n) { @@ -478,6 +508,40 @@ module.exports = function(RED) { node.send(group.msg); } + var pendingMessages = []; + var activeMessagePromise = null; + // In reduce mode, we must process messages fully in order otherwise + // groups may overlap and cause unexpected results. The use of JSONata + // means some async processing *might* occur if flow/global context is + // accessed. + var processReduceMessageQueue = function(msg) { + if (msg) { + // A new message has arrived - add it to the message queue + pendingMessages.push(msg); + if (activeMessagePromise !== null) { + // The node is currently processing a message, so do nothing + // more with this message + return; + } + } + if (pendingMessages.length === 0) { + // There are no more messages to process, clear the active flag + // and return + activeMessagePromise = null; + return; + } + + // There are more messages to process. Get the next message and + // start processing it. Recurse back in to check for any more + var nextMsg = pendingMessages.shift(); + activeMessagePromise = reduce_msg(node, nextMsg) + .then(processReduceMessageQueue) + .catch((err) => { + node.error(err,nextMsg); + return processReduceMessageQueue(); + }); + } + this.on("input", function(msg) { try { var property; @@ -516,8 +580,7 @@ module.exports = function(RED) { propertyIndex = msg.parts.index; } else if (node.mode === 'reduce') { - reduce_msg(node, msg); - return; + return processReduceMessageQueue(msg); } else { // Use the node configuration to identify all of the group information From d8cf86fd6f200805e588c0730d66c008218786d5 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Tue, 10 Jul 2018 11:41:46 +0100 Subject: [PATCH 16/22] Add RED.utils.parseContextKey --- editor/js/ui/common/typedInput.js | 13 ++++--------- editor/js/ui/utils.js | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/editor/js/ui/common/typedInput.js b/editor/js/ui/common/typedInput.js index 74bbeb9ac..59db274f7 100644 --- a/editor/js/ui/common/typedInput.js +++ b/editor/js/ui/common/typedInput.js @@ -15,16 +15,11 @@ **/ (function($) { var contextParse = function(v) { - var parts = {}; - var m = /^#:\((\S+?)\)::(.*)$/.exec(v); - if (m) { - parts.option = m[1]; - parts.value = m[2]; - } else { - parts.value = v; - parts.option = RED.settings.context.default; + var parts = RED.utils.parseContextKey(v); + return { + option: parts.store, + value: parts.key } - return parts; } var contextExport = function(v,opt) { if (!opt) { diff --git a/editor/js/ui/utils.js b/editor/js/ui/utils.js index e64a33996..7da56ab78 100644 --- a/editor/js/ui/utils.js +++ b/editor/js/ui/utils.js @@ -828,6 +828,20 @@ RED.utils = (function() { return payload; } + function parseContextKey(key) { + var parts = {}; + var m = /^#:\((\S+?)\)::(.*)$/.exec(key); + if (m) { + parts.store = m[1]; + parts.key = m[2]; + } else { + parts.key = key; + if (RED.settings.context) { + parts.store = RED.settings.context.default; + } + } + return parts; + } return { createObjectElement: buildMessageElement, @@ -839,6 +853,7 @@ RED.utils = (function() { getNodeIcon: getNodeIcon, getNodeLabel: getNodeLabel, addSpinnerOverlay: addSpinnerOverlay, - decodeObject: decodeObject + decodeObject: decodeObject, + parseContextKey: parseContextKey } })(); From 6e9fe3248a63ccda3fcb1e45f4380314f2f3e18c Mon Sep 17 00:00:00 2001 From: Hiroyasu Nishiyama Date: Tue, 10 Jul 2018 20:40:31 +0900 Subject: [PATCH 17/22] Fix appearrence of change node label for flow/global ref (#1792) * fix appearence of change node label for flow/global ref * use RED.utils.parseContextKey --- nodes/core/logic/15-change.html | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nodes/core/logic/15-change.html b/nodes/core/logic/15-change.html index 83bdecf9e..359b43fdd 100644 --- a/nodes/core/logic/15-change.html +++ b/nodes/core/logic/15-change.html @@ -54,6 +54,10 @@ outputs: 1, icon: "swap.png", label: function() { + function prop2name(type, key) { + var result = RED.utils.parseContextKey(key); + return type +"." +result.key; + } if (this.name) { return this.name; } @@ -70,13 +74,13 @@ } else { if (this.rules.length == 1) { if (this.rules[0].t === "set") { - return this._("change.label.set",{property:(this.rules[0].pt||"msg")+"."+this.rules[0].p}); + return this._("change.label.set",{property:prop2name((this.rules[0].pt||"msg"), this.rules[0].p)}); } else if (this.rules[0].t === "change") { - return this._("change.label.change",{property:(this.rules[0].pt||"msg")+"."+this.rules[0].p}); + return this._("change.label.change",{property:prop2name((this.rules[0].pt||"msg"), this.rules[0].p)}); } else if (this.rules[0].t === "move") { - return this._("change.label.move",{property:(this.rules[0].pt||"msg")+"."+this.rules[0].p}); + return this._("change.label.move",{property:prop2name((this.rules[0].pt||"msg"), this.rules[0].p)}); } else { - return this._("change.label.delete",{property:(this.rules[0].pt||"msg")+"."+this.rules[0].p}); + return this._("change.label.delete",{property:prop2name((this.rules[0].pt||"msg"), this.rules[0].p)}); } } else { return this._("change.label.changeCount",{count:this.rules.length}); From 407e16e9009db75f901e4b03f1cc1056cbaef9ed Mon Sep 17 00:00:00 2001 From: Hiroyasu Nishiyama Date: Tue, 10 Jul 2018 20:40:52 +0900 Subject: [PATCH 18/22] Fix appearrence of switch node port label for flow/global ref. (#1793) * fix appearrence of switch node port label for flow/global ref * use RED.utils.parseContextKey --- nodes/core/logic/10-switch.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nodes/core/logic/10-switch.html b/nodes/core/logic/10-switch.html index f4ffc7dea..a57ce2187 100644 --- a/nodes/core/logic/10-switch.html +++ b/nodes/core/logic/10-switch.html @@ -99,11 +99,17 @@ } return v; } + function prop2name(key) { + var result = RED.utils.parseContextKey(key); + return result.key; + } function getValueLabel(t,v) { if (t === 'str') { return '"'+clipValueLength(v)+'"'; - } else if (t === 'msg' || t==='flow' || t==='global') { + } else if (t === 'msg') { return t+"."+clipValueLength(v); + } else if (t === 'flow' || t === 'global') { + return t+"."+clipValueLength(prop2name(v)); } return clipValueLength(v); } From 1bf4addf638091c6b650667627e24a0b01d148c2 Mon Sep 17 00:00:00 2001 From: Hiroki Uchikawa <31908137+HirokiUchikawa@users.noreply.github.com> Date: Tue, 10 Jul 2018 20:41:16 +0900 Subject: [PATCH 19/22] Fix an error when initializing the cache (#1788) * Fix a error when initializing the cache * Make context directory if it is not there in initialization --- red/runtime/nodes/context/localfilesystem.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/red/runtime/nodes/context/localfilesystem.js b/red/runtime/nodes/context/localfilesystem.js index 28e1c4cce..89c7e6760 100644 --- a/red/runtime/nodes/context/localfilesystem.js +++ b/red/runtime/nodes/context/localfilesystem.js @@ -143,7 +143,13 @@ LocalFileSystem.prototype.open = function(){ self.cache.set(scope,key,data[key]); }) }); - }) + }).catch(function(err){ + if(err.code == 'ENOENT') { + return fs.mkdir(self.storageBaseDir); + }else{ + return Promise.reject(err); + } + }); } else { return Promise.resolve(); } From 1a544b3b824079ac426f83efe81147737c08aecd Mon Sep 17 00:00:00 2001 From: YumaMatsuura <38545050+YumaMatsuura@users.noreply.github.com> Date: Tue, 10 Jul 2018 20:42:56 +0900 Subject: [PATCH 20/22] Headless option for ui test (#1784) --- Gruntfile.js | 4 ++++ test/editor/wdio.conf.js | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index faf68f100..508fadc58 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -24,6 +24,10 @@ module.exports = function(grunt) { nodemonArgs.push(flowFile); } + var nonHeadless = grunt.option('non-headless'); + if (nonHeadless) { + process.env.NODE_RED_NON_HEADLESS = 'true'; + } grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), paths: { diff --git a/test/editor/wdio.conf.js b/test/editor/wdio.conf.js index 08cc1e60f..eb23a9a2c 100644 --- a/test/editor/wdio.conf.js +++ b/test/editor/wdio.conf.js @@ -62,10 +62,11 @@ exports.config = { // browserName: 'chrome', chromeOptions: { - // Runs tests without opening a broser. - args: ['--headless', '--disable-gpu', 'window-size=1920,1080'], - // Runs tests with opening a broser. - // args: ['--disable-gpu'], + args: process.env.NODE_RED_NON_HEADLESS + // Runs tests with opening a browser. + ? ['--disable-gpu'] + // Runs tests without opening a browser. + : ['--headless', '--disable-gpu', 'window-size=1920,1080'] }, }], // From f368f5a9c42fe014f326f80ccc674078ac8b6277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathana=C3=ABl=20L=C3=A9caud=C3=A9?= Date: Tue, 10 Jul 2018 11:29:01 -0400 Subject: [PATCH 21/22] JSON node: Add link to JSON schema spec in node help --- nodes/core/parsers/70-JSON.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodes/core/parsers/70-JSON.html b/nodes/core/parsers/70-JSON.html index a440e5d45..32b5a332a 100644 --- a/nodes/core/parsers/70-JSON.html +++ b/nodes/core/parsers/70-JSON.html @@ -58,6 +58,8 @@ receives a String, no further checks will be made of the property. It will not check the String is valid JSON nor will it reformat it if the format option is selected.

    +

    For more details about JSON Schema you can consult the specification + here.