From f051fbd1e1ca74896d06f6493b4a91644279ab00 Mon Sep 17 00:00:00 2001 From: Dave C-J Date: Sun, 24 Nov 2013 13:10:48 +0000 Subject: [PATCH 01/15] Make imap node check for email right away on start/restart. Add some more console logging for re-assurance of things happening - or not. --- nodes/core/social/61-imap.js | 139 +++++++++++++++++------------------ 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/nodes/core/social/61-imap.js b/nodes/core/social/61-imap.js index 999ae7a93..f767ca3d7 100644 --- a/nodes/core/social/61-imap.js +++ b/nodes/core/social/61-imap.js @@ -20,89 +20,86 @@ var util = require('util'); var oldmail = {}; try { - var emailkey = RED.settings.email || require(process.env.NODE_RED_HOME+"/../emailkeys.js"); -} catch(err) { - throw new Error("Failed to load Email credentials"); + var emailkey = RED.settings.email || require(process.env.NODE_RED_HOME+"/../emailkeys.js"); +} catch (err) { + util.log("[imap] : Failed to load Email credentials"); + return; } var imap = new Imap({ - user: emailkey.user, - password: emailkey.pass, - host: emailkey.server||"imap.gmail.com", - port: emailkey.port||"993", - secure: true + user: emailkey.user, + password: emailkey.pass, + host: emailkey.server||"imap.gmail.com", + port: emailkey.port||"993", + secure: true }); -function fail(err) { - util.log('[imap] : ' + err); -} - function openInbox(cb) { - imap.connect(function(err) { - if (err) fail(err); - imap.openBox('INBOX', true, cb); - }); + imap.connect(function(err) { + if (err) util.log("[imap] : error : "+err); + imap.openBox('INBOX', true, cb); + }); } function ImapNode(n) { - RED.nodes.createNode(this,n); - this.name = n.name; - this.repeat = n.repeat * 1000; - var node = this; - this.interval_id = null; + RED.nodes.createNode(this,n); + this.name = n.name; + this.repeat = n.repeat * 1000 || 300000; + var node = this; + this.interval_id = null; - if (this.repeat && !isNaN(this.repeat) && this.repeat > 0) { - this.log("repeat = "+this.repeat); - this.interval_id = setInterval( function() { - node.emit("input",{}); - }, this.repeat ); - } + if (!isNaN(this.repeat) && this.repeat > 0) { + node.log("repeat = "+this.repeat); + this.interval_id = setInterval( function() { + node.emit("input",{}); + }, this.repeat ); + } - this.on("input", function(msg) { - openInbox(function(err, mailbox) { - if (err) fail(err); - imap.seq.fetch(mailbox.messages.total + ':*', { struct: false }, - { headers: ['from', 'subject'], - body: true, - cb: function(fetch) { - fetch.on('message', function(msg) { - //node.log('Saw message no. ' + msg.seqno); - var pay = {}; - var body = ''; - msg.on('headers', function(hdrs) { - pay.from = hdrs.from[0]; - pay.topic = hdrs.subject[0]; - }); - msg.on('data', function(chunk) { - body += chunk.toString('utf8'); - }); - msg.on('end', function() { - pay.payload = body; - if ((pay.topic !== oldmail.topic)|(pay.payload !== oldmail.payload)) { - oldmail = pay; - //node.log("From: "+pay.from); - node.log("Subj: "+pay.topic); - //node.log("Body: "+pay.payload); - node.send(pay); - } - }); - }); - } - }, function(err) { - if (err) node.log("Err : "+err); - //node.log("Done fetching messages."); - imap.logout(); - } - ); - }); + this.on("input", function(msg) { + openInbox(function(err, mailbox) { + if (err) node.log("error : "+err); + imap.seq.fetch(mailbox.messages.total + ':*', { struct: false }, + { headers: ['from', 'subject'], + body: true, + cb: function(fetch) { + fetch.on('message', function(msg) { + node.log('Read message no. ' + msg.seqno); + var pay = {}; + var body = ''; + msg.on('headers', function(hdrs) { + pay.from = hdrs.from[0]; + pay.topic = hdrs.subject[0]; + }); + msg.on('data', function(chunk) { + body += chunk.toString('utf8'); + }); + msg.on('end', function() { + pay.payload = body; + if ((pay.topic !== oldmail.topic)|(pay.payload !== oldmail.payload)) { + oldmail = pay; + //node.log("From: "+pay.from); + node.log("Subj: "+pay.topic); + //node.log("Body: "+pay.payload); + node.send(pay); + } + }); + }); + } + }, function(err) { + if (err) node.log("error : "+err); + node.log("Done fetching messages."); + imap.logout(); + } + ); + }); + }); - }); + this.on("close", function() { + if (this.interval_id != null) { + clearInterval(this.interval_id); + } + }); - this.on("close", function() { - if (this.interval_id != null) { - clearInterval(this.interval_id); - } - }); + node.emit("input",{}); } - RED.nodes.registerType("imap",ImapNode); From 9104b4200a01f1f1345b323f2b490198e116efbd Mon Sep 17 00:00:00 2001 From: Nicholas O'Leary Date: Sun, 24 Nov 2013 16:01:52 +0000 Subject: [PATCH 02/15] Update feedparser node for underlying module api changes --- nodes/core/social/32-feedparse.js | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/nodes/core/social/32-feedparse.js b/nodes/core/social/32-feedparse.js index f7029206d..77ae4a8d2 100644 --- a/nodes/core/social/32-feedparse.js +++ b/nodes/core/social/32-feedparse.js @@ -35,22 +35,18 @@ function FeedParseNode(n) { node.error(error); }) .on('meta', function (meta) {}) - .on('article', function (article) { - if (!(article.guid in node.seen) || ( node.seen[article.guid] != 0 && node.seen[article.guid] != article.date.getTime())) { - node.seen[article.guid] = article.date?article.date.getTime():0; - var msg = { - topic:article.origlink||article.link, - payload: article.description, - article: { - summary:article.summary, - link:article.link, - date: article.date, - pubdate: article.pubdate, - author: article.author, - guid: article.guid, - } - }; - node.send(msg); + .on('readable', function () { + var stream = this, article; + while (article = stream.read()) { + if (!(article.guid in node.seen) || ( node.seen[article.guid] != 0 && node.seen[article.guid] != article.date.getTime())) { + node.seen[article.guid] = article.date?article.date.getTime():0; + var msg = { + topic:article.origlink||article.link, + payload: article.description, + article: article + }; + node.send(msg); + } } }) .on('end', function () { From cb8a3f064efa675063133f309b2ff62b5330139e Mon Sep 17 00:00:00 2001 From: Nicholas O'Leary Date: Sun, 24 Nov 2013 16:48:24 +0000 Subject: [PATCH 03/15] Twitter doc updates and rate limit fixes --- nodes/core/social/27-twitter.html | 17 +++++++++++++++-- nodes/core/social/27-twitter.js | 10 +++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nodes/core/social/27-twitter.html b/nodes/core/social/27-twitter.html index 2515a3541..b81af1d46 100644 --- a/nodes/core/social/27-twitter.html +++ b/nodes/core/social/27-twitter.html @@ -129,7 +129,11 @@ + + + + diff --git a/nodes/core/logic/15-change.js b/nodes/core/logic/15-change.js new file mode 100644 index 000000000..4866239c0 --- /dev/null +++ b/nodes/core/logic/15-change.js @@ -0,0 +1,66 @@ +/** + * Copyright 2013 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +var RED = require(process.env.NODE_RED_HOME + "/red/red"); + +function ChangeNode(n) { + RED.nodes.createNode(this, n); + this.action = n.action; + this.property = n.property || ""; + this.from = n.from || " "; + this.to = n.to || " "; + var node = this; + + var makeNew = function( stem, path, value ) { + var lastPart = (arguments.length === 3) ? path.pop() : false; + for (var i = 0; i < path.length; i++) { + stem = stem[path[i]] = stem[path[i]] || {}; + } + if (lastPart) { stem = stem[lastPart] = value; } + return stem; + }; + + this.on('input', function (msg) { + if (node.action == "change") { + node.re = new RegExp(this.from, "g"); + if (typeof msg[node.property] === "string") { + msg[node.property] = (msg[node.property]).replace(node.re, node.to); + } + } + //else if (node.action == "replace") { + //if (node.to.indexOf("msg.") == 0) { + //msg[node.property] = eval(node.to); + //} + //else { + //msg[node.property] = node.to; + //} + //} + else if (node.action == "replace") { + if (node.to.indexOf("msg.") == 0) { + makeNew( msg, node.property.split("."), eval(node.to) ); + } + else { + makeNew( msg, node.property.split("."), node.to ); + } + //makeNew( msg, node.property.split("."), node.to ); + } + else if (node.action == "delete") { + delete(msg[node.property]); + } + node.send(msg); + }); +} +RED.nodes.registerType("change", ChangeNode); diff --git a/public/icons/swap.png b/public/icons/swap.png new file mode 100644 index 0000000000000000000000000000000000000000..f93e6490d5f0801c326cd2e2404f950b91efd818 GIT binary patch literal 580 zcmV-K0=xZ*P)P001Zm1^@s63(rw&00001b5ch_0Itp) z=>Px#24YJ`L;wH)0002_L%V+f000SaNLh0L01FcU01FcV0GgZ_0005kNklzXx$a;vx=$qmu{;9>C2*AmT+Njvm09aChQngGk6mY`S+pmzhax z-MUwdh^VA8>6)tgx~jUXyM-jmrV4(5{z3jWh*;lJKxnX(ovs|H+?uq}p0hzDGS?wl zL3Pxx9v?B3>s72=+I;BBF#tmAO^1F#BO`dqZ6)hn-UjxH*XaftH&zr?04jFX1#8kk zNfqM{>Zk>^}f~y5qa(9+=}Qs^>+c z1-4^by+4ZQ=TWQJYa|m>&}4JKvrlI2bOm?3CQn11!2Wl~yEOM5?7|sM0%!t&Qdmq9 zT=a16OD#h22*7PVND^H3coa)aB$6>pIZqU@Vgck6($IP%HeEteB5L9?rYAiBfL03= zX^IB`tbG9>xKNbBa+>14n9(=CPiWkN3dgk)u3e>GQ^nv?@50V{{l8CsAIKYT_2+{K S=*kEH0000 Date: Mon, 25 Nov 2013 10:30:19 +0000 Subject: [PATCH 08/15] Twitter node: tags field not required if DM's selected Fixes #91 --- nodes/core/social/27-twitter.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/core/social/27-twitter.html b/nodes/core/social/27-twitter.html index b81af1d46..bc1316449 100644 --- a/nodes/core/social/27-twitter.html +++ b/nodes/core/social/27-twitter.html @@ -146,7 +146,7 @@ color:"#C0DEED", defaults: { twitter: {type:"twitter-credentials",required:true}, - tags: {value:"",required:true}, + tags: {value:"",validate:function(v) { return this.user == "dm" || v.length > 0;}}, user: {value:"false",required:true}, name: {value:""}, topic: {value:"tweets"} From 796080471df46a4e93d3a558216516daca43b14a Mon Sep 17 00:00:00 2001 From: Nicholas O'Leary Date: Mon, 25 Nov 2013 21:46:15 +0000 Subject: [PATCH 09/15] Twitter: add help text about rate limits --- nodes/core/social/27-twitter.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nodes/core/social/27-twitter.html b/nodes/core/social/27-twitter.html index bc1316449..66619c5ea 100644 --- a/nodes/core/social/27-twitter.html +++ b/nodes/core/social/27-twitter.html @@ -138,6 +138,10 @@

Sets the msg.topic to tweets/ and then appends the senders screen name.

Sets msg.location to the tweeters location if known.

Sets msg.tweet to the full tweet object as documented by Twitter. +

Note: when set to a specific user's tweets, or your direct messages, the node is subject to + Twitter's API rate limiting. If you deploy the flows multiple times within a 15 minute window, you may + exceed the limit and will see errors from the node. These errors will clear when the current 15 minute window + passes.

diff --git a/nodes/core/io/10-mqtt.js b/nodes/core/io/10-mqtt.js index af3d77e94..f729d286b 100644 --- a/nodes/core/io/10-mqtt.js +++ b/nodes/core/io/10-mqtt.js @@ -22,9 +22,54 @@ function MQTTBrokerNode(n) { RED.nodes.createNode(this,n); this.broker = n.broker; this.port = n.port; + this.clientid = n.clientid; + var credentials = RED.nodes.getCredentials(n.id); + if (credentials) { + this.username = credentials.user; + this.password = credentials.password; + } } RED.nodes.registerType("mqtt-broker",MQTTBrokerNode); +var querystring = require('querystring'); + +RED.app.get('/mqtt-broker/:id',function(req,res) { + var credentials = RED.nodes.getCredentials(req.params.id); + if (credentials) { + res.send(JSON.stringify({user:credentials.user,hasPassword:(credentials.password&&credentials.password!="")})); + } else { + res.send(JSON.stringify({})); + } +}); + +RED.app.delete('/mqtt-broker/:id',function(req,res) { + RED.nodes.deleteCredentials(req.params.id); + res.send(200); +}); + +RED.app.post('/mqtt-broker/:id',function(req,res) { + var body = ""; + req.on('data', function(chunk) { + body+=chunk; + }); + req.on('end', function(){ + var newCreds = querystring.parse(body); + var credentials = RED.nodes.getCredentials(req.params.id)||{}; + if (newCreds.user == null || newCreds.user == "") { + delete credentials.user; + } else { + credentials.user = newCreds.user; + } + if (newCreds.password == "") { + delete credentials.password; + } else { + credentials.password = newCreds.password||credentials.password; + } + RED.nodes.addCredentials(req.params.id,credentials); + res.send(200); + }); +}); + function MQTTInNode(n) { RED.nodes.createNode(this,n); @@ -32,7 +77,7 @@ function MQTTInNode(n) { this.broker = n.broker; this.brokerConfig = RED.nodes.getNode(this.broker); if (this.brokerConfig) { - this.client = connectionPool.get(this.brokerConfig.broker,this.brokerConfig.port); + this.client = connectionPool.get(this.brokerConfig.broker,this.brokerConfig.port,this.brokerConfig.clientid,this.brokerConfig.username,this.brokerConfig.password); var node = this; this.client.subscribe(this.topic,2,function(topic,payload,qos,retain) { var msg = {topic:topic,payload:payload,qos:qos,retain:retain}; @@ -65,7 +110,7 @@ function MQTTOutNode(n) { this.brokerConfig = RED.nodes.getNode(this.broker); if (this.brokerConfig) { - this.client = connectionPool.get(this.brokerConfig.broker,this.brokerConfig.port); + this.client = connectionPool.get(this.brokerConfig.broker,this.brokerConfig.port,this.brokerConfig.clientid,this.brokerConfig.username,this.brokerConfig.password); this.on("input",function(msg) { if (msg != null) { if (this.topic) { diff --git a/nodes/core/io/lib/mqttConnectionPool.js b/nodes/core/io/lib/mqttConnectionPool.js index 1e4b681d4..6b31dea65 100644 --- a/nodes/core/io/lib/mqttConnectionPool.js +++ b/nodes/core/io/lib/mqttConnectionPool.js @@ -25,13 +25,16 @@ function matchTopic(ts,t) { } module.exports = { - get: function(broker,port) { - var id = broker+":"+port; + get: function(broker,port,clientid,username,password) { + var id = "["+(username||"")+":"+(password||"")+"]["+(clientid||"")+"]@"+broker+":"+port; if (!connections[id]) { connections[id] = function() { var client = mqtt.createClient(port,broker); client.setMaxListeners(0); - var options = {keepalive:15,clientId:'mqtt_' + (1+Math.random()*4294967295).toString(16)}; + var options = {keepalive:15}; + options.clientId = clientid || 'mqtt_' + (1+Math.random()*4294967295).toString(16); + options.username = username; + options.password = password; var queue = []; var subscriptions = []; var connecting = false; From ab04fcf7c06e802b04f379edd7d444340ea576da Mon Sep 17 00:00:00 2001 From: Dave C-J Date: Tue, 26 Nov 2013 19:55:40 +0000 Subject: [PATCH 11/15] Update IMAP node to use new 0.8.x API Fixes Issue #96 this necessitates an update to the underlying npm npm install --force imap --- nodes/core/social/61-imap.html | 29 +++++----- nodes/core/social/61-imap.js | 102 ++++++++++++++++++++------------- 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/nodes/core/social/61-imap.html b/nodes/core/social/61-imap.html index 6d98c5f62..002b6f7a8 100644 --- a/nodes/core/social/61-imap.html +++ b/nodes/core/social/61-imap.html @@ -15,23 +15,24 @@ -->