From 42cd6131b27f963c9d39c66a695e098651cb9e2f Mon Sep 17 00:00:00 2001
From: Nick O'Leary <nick.oleary@gmail.com>
Date: Wed, 15 Aug 2018 15:23:08 +0100
Subject: [PATCH] big twitter node update for api changes

---
 social/twitter/27-twitter.html |  67 +--
 social/twitter/27-twitter.js   | 723 +++++++++++++++++++++------------
 social/twitter/package.json    |  18 +-
 3 files changed, 510 insertions(+), 298 deletions(-)

diff --git a/social/twitter/27-twitter.html b/social/twitter/27-twitter.html
index b501f407..90b32c6c 100644
--- a/social/twitter/27-twitter.html
+++ b/social/twitter/27-twitter.html
@@ -36,7 +36,7 @@
     <p>Authentication for the Twitter API</p>
     <p>Earlier versions of this node provided one-click authentication. Twitter removed the ability
        to do that in June 2018. You must now register your own application with
-       <a href="https://apps.twitter.com">Twitter</a> and generate your own access tokens.</p>
+       <a href="https://developer.twitter.com/">Twitter</a> and generate your own access tokens.</p>
 
 </script>
 
@@ -96,12 +96,8 @@
             <option value="true" data-i18n="twitter.search.follow"></option>
             <option value="user" data-i18n="twitter.search.user"></option>
             <option value="dm" data-i18n="twitter.search.direct"></option>
-            <option value="event" data-i18n="twitter.search.events"></option>
         </select>
     </div>
-    <div id="tweet-events-deprecated" class="hide form-tips" style="background: #edd; padding: 20px; margin-bottom: 20px">
-        <i class="fa fa-warning"></i> Twitter are withdrawing the API used to access a user&apos;s activity stream in August 2018 so this feature will be removed from the node in the near future. See <a href="https://bit.ly/2kr7InE">here</a> for details.
-    </div>
     <div class="form-row" id="node-input-tags-row">
         <label for="node-input-tags"><i class="fa fa-tags"></i> <span id="node-input-tags-label" data-i18n="twitter.label.for"></span></label>
         <input type="text" id="node-input-tags" data-i18n="[placeholder]twitter.placeholder.for">
@@ -116,23 +112,28 @@
 <script type="text/x-red" data-help-name="twitter in">
     <p>Twitter input node. Can be used to search either:
     <ul><li>the public stream for tweets containing the configured search term</li>
-        <li>all the tweets from accounts that the authenticated user follows</li>
-        <li>all tweets by specified users</li>
+        <li>tweets from accounts that the authenticated user follows</li>
+        <li>tweets by specified users</li>
         <li>direct messages received by the authenticated user</li>
     </ul></p>
-    <p>Use space for <i>and</i> and comma , for <i>or</i> when searching for multiple terms.
-    If you want to pass in the search term(s) via the <code>msg.payload</code>, leave the <b>for</b> field blank.</p>
-    <p>Sets the <code>msg.topic</code> to <i>tweets/</i> and then appends the senders screen name.</p>
-    <p>Sets <code>msg.location</code> to the tweeters location if known.</p>
-    <p>When returning events it sets the <code>msg.payload</code> to the twitter event, a full list is documented by
-    <a href="https://dev.twitter.com/streaming/overview/messages-types#Events_event" target="_new">Twitter</a>.</p>
-    <p>Sets <code>msg.tweet</code> to the full tweet object as documented by <a href="https://dev.twitter.com/overview/api/tweets" target="_new">Twitter</a>.
-
-    <p><b>Note</b>: This node is not connected to the FireHose, so will not return 100% of all tweets to a busy @id or #hashtag.</p>
-    <p><b>Note:</b> when set to follow specific users, or your direct messages, the node is subject to
-      the rate limiting of the Twitter API. If you deploy the flows multiple times within a 15 minute window, you may
-      exceed the limit and will see errors from the node. These errors will clear automatically when the current 15
-      minute window passes.</p>
+    <h3>Outputs</h3>
+    <dl class="message-properties">
+        <dt>payload <span class="property-type">string</span></dt>
+        <dd>the text of the tweet</dd>
+        <dt>topic <span class="property-type">string</span></dt>
+        <dd>set to <code>tweets/<i>screen_name</i></code></dd>
+        <dt>tweet <span class="property-type">object</span></dt>
+        <dd>the full tweet object returned by the Twitter API</dd>
+        <dt>location <span class="property-type">object</span></dt>
+        <dd>location information associated with the tweet, if known</dd>
+    </dl>
+    <h3>Details</h3>
+    <p>When searching for multiple terms, use a space for <i>and</i> and comma `,` for <i>or</i>.
+    <p>The full tweet object is documented <a href="https://developer.twitter.com/en/docs/tweets/data-dictionary/overview/tweet-object" target="_new">here</a>.</p>
+    <p><b>Note:</b> this node is subject to the rate limiting restrictions of the Twitter
+    API. It polls the API once a minute for updates. If you deploy frequently you may
+      exceed the limit. The node will automatically delay polling until the end of the current
+      rate limiting window.</p>
 </script>
 
 <script type="text/javascript">
@@ -144,7 +145,6 @@
             tags: {value:""},
             user: {value:"false",required:true},
             name: {value:""},
-            topic: {value:"tweets"},
             inputs: {value:0}
         },
         inputs: 0,
@@ -189,7 +189,6 @@
                     $("#node-input-tags-label").html(forlabel);
                     $("#node-input-tags").attr("placeholder",forph);
                 }
-                $("#tweet-events-deprecated").toggle((type === 'event'));
             });
             $("#node-input-user").change();
         },
@@ -218,12 +217,24 @@
 </script>
 
 <script type="text/x-red" data-help-name="twitter out">
-    <p>Twitter out node. Tweets the <code>msg.payload</code>.</p>
-    <p>To send a Direct Message (DM) - use a payload like "D {username} {message}"</p>
-    <p>If <code>msg.media</code> exists and is a Buffer object, this node will treat it
-       as an image and attach it to the tweet.</p>
-    <p>If <code>msg.params</code> exists and is an object of name:value pairs,
-    this node will treat it as parameters for the update request.</p>
+    <p>Send tweets and direct messages.</p>
+    <h3>Inputs</h3>
+    <dl class="message-properties">
+        <dt>payload <span class="property-type">string</span></dt>
+        <dd>Sent as the body of the tweet. See below for how to send a Direct Message</dd>
+        <dt class="optional">media <span class="property-type">buffer</span></dt>
+        <dd>A Buffer of an image to attach to the tweet</dd>
+        <dt class="optional">params <span class="property-type">object</span></dt>
+        <dd>Additional parameters to pass to the Twitter status update API.</dd>
+    </dl>
+    <h3>Details</h3>
+    <p>This nodes sends a tweet for the authenticated user. If <code>msg.media</code>
+    is set and contains a Buffer, it is attached as an image.</p>
+    <p>To send a Direct Message, the payload should be formatted as: <code>D {username} {message}</code>.</p>
+    <p>Note that you cannot attach an image to a direct message.</p>
+    <p>If <code>msg.params</code> exists and is an object of name:value pairs, they
+       will be included in the request to the Twitter api. The available values are documented
+       <a href="https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update#parameters" target="_new">here</a>.</p>
 </script>
 
 <script type="text/javascript">
diff --git a/social/twitter/27-twitter.js b/social/twitter/27-twitter.js
index 531011af..16418a53 100644
--- a/social/twitter/27-twitter.js
+++ b/social/twitter/27-twitter.js
@@ -2,18 +2,53 @@
 module.exports = function(RED) {
     "use strict";
     var Ntwitter = require('twitter-ng');
-    var OAuth= require('oauth').OAuth;
     var request = require('request');
+    var crypto = require('crypto');
+    // var fileType = require('file-type');
     var twitterRateTimeout;
     var retry = 60000; // 60 secs backoff for now
 
+    var localUserCache = {};
+    var userObjectCache = {};
+    var userSreenNameToIdCache = {};
+
     function TwitterCredentialsNode(n) {
         RED.nodes.createNode(this,n);
         this.screen_name = n.screen_name;
-        if (this.screen_name && this.screen_name[0] !== "@") {
-            this.screen_name = "@"+this.screen_name;
+        if (this.screen_name && this.screen_name[0] === "@") {
+            this.screen_name = this.screen_name.substring(1);
+        }
+        if (this.credentials.consumer_key &&
+            this.credentials.consumer_secret &&
+            this.credentials.access_token &&
+            this.credentials.access_token_secret) {
+
+            this.oauth = {
+                consumer_key: this.credentials.consumer_key,
+                consumer_secret: this.credentials.consumer_secret,
+                token: this.credentials.access_token,
+                token_secret: this.credentials.access_token_secret
+            }
+            this.credHash = crypto.createHash('sha1').update(
+                this.credentials.consumer_key+this.credentials.consumer_secret+
+                this.credentials.access_token+this.credentials.access_token_secret
+            ).digest('base64');
+            var self = this;
+            if (localUserCache.hasOwnProperty(self.credHash)) {
+                this.localIdentityPromise = Promise.resolve(localUserCache[self.credHash]);
+            } else {
+                this.localIdentityPromise = this.get("https://api.twitter.com/1.1/account/settings.json").then(function(body) {
+                    if (body.status === 200) {
+                        localUserCache[self.credHash] = body.body.screen_name;
+                        self.screen_name = body.body.screen_name;
+                    } else {
+                        self.warn("Failed to get user profile");
+                    }
+                });
+            }
         }
     }
+
     RED.nodes.registerType("twitter-credentials",TwitterCredentialsNode,{
         credentials: {
             consumer_key: { type: "password"},
@@ -22,8 +57,81 @@ module.exports = function(RED) {
             access_token_secret: {type:"password"}
         }
     });
+    TwitterCredentialsNode.prototype.get = function(url,opts) {
+        var node = this;
+        opts = opts || {};
+        opts.tweet_mode = 'extended';
+        return new Promise(function(resolve,reject) {
+            request.get({
+                url: url,
+                oauth: node.oauth,
+                json: true,
+                qs: opts
+            }, function(err, response,body) {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve({
+                        status: response.statusCode,
+                        rateLimitRemaining: response.headers['x-rate-limit-remaining'],
+                        rateLimitTimeout: 5000+parseInt(response.headers['x-rate-limit-reset'])*1000 - Date.now(),
+                        body: body
+                    });
+                }
+            });
+        })
+    }
+    TwitterCredentialsNode.prototype.post = function(url,data,opts,formData) {
+        var node = this;
+        opts = opts || {};
+        var options = {
+            url: url,
+            oauth: node.oauth,
+            json: true,
+            qs: opts,
+        };
+        if (data) {
+            options.body = data;
+        }
+        if (formData) {
+            options.formData = formData;
+        }
+        return new Promise(function(resolve,reject) {
+            request.post(options, function(err, response,body) {
+                if (err) {
+                    reject(err);
+                } else {
+                    resolve({
+                        status: response.statusCode,
+                        rateLimitRemaining: response.headers['x-rate-limit-remaining'],
+                        rateLimitTimeout: 5000+parseInt(response.headers['x-rate-limit-reset'])*1000 - Date.now(),
+                        body: body
+                    });
+                }
+            });
+        })
+    }
 
 
+    TwitterCredentialsNode.prototype.getUsers = function(users,getBy) {
+        if (users.length === 0) {
+            return Promise.resolve();
+        }
+        var params = {};
+        params[getBy||"user_id"] = users;
+
+        return this.get("https://api.twitter.com/1.1/users/lookup.json",params).then(function(result) {
+            var res = result.body;
+            if (res.errors) {
+                throw new Error(res.errors[0].message);
+            }
+            res.forEach(user => {
+                userObjectCache[user.id_str] = user
+                userSreenNameToIdCache[user.screen_name] = user.id_str;
+            });
+        })
+    }
+
     /**
      * Populate msg.location based on data found in msg.tweet.
      */
@@ -55,192 +163,45 @@ module.exports = function(RED) {
         this.active = true;
         this.user = n.user;
         //this.tags = n.tags.replace(/ /g,'');
-        this.tags = n.tags;
+        this.tags = n.tags||"";
         this.twitter = n.twitter;
-        this.topic = n.topic||"tweets";
+        this.topic = "tweets";
         this.twitterConfig = RED.nodes.getNode(this.twitter);
+        this.poll_ids = [];
+        this.timeout_ids = [];
+
         var credentials = RED.nodes.getCredentials(this.twitter);
+        this.status({});
 
-        if (credentials && credentials.consumer_key && credentials.consumer_secret && credentials.access_token && credentials.access_token_secret) {
-            var twit = new Ntwitter({
-                consumer_key: credentials.consumer_key,
-                consumer_secret: credentials.consumer_secret,
-                access_token_key: credentials.access_token,
-                access_token_secret: credentials.access_token_secret
-            });
-
-            //setInterval(function() {
-            //        twit.get("/application/rate_limit_status.json",null,function(err,cb) {
-            //                console.log("direct_messages:",cb["resources"]["direct_messages"]);
-            //        });
-            //
-            //},10000);
-
+        if (this.twitterConfig.oauth) {
             var node = this;
-            if (this.user === "user") {
-                node.poll_ids = [];
-                node.since_ids = {};
-                node.status({});
-                var users = node.tags.split(",");
-                if (users === '') { node.warn(RED._("twitter.warn.nousers")); }
-                //if (users.length === 0) { node.warn(RED._("twitter.warn.nousers")); }
-                else {
-                    for (var i=0; i<users.length; i++) {
-                        var user = users[i].replace(" ","");
-                        twit.getUserTimeline({
-                            screen_name:user,
-                            trim_user:0,
-                            count:1
-                        },(function() {
-                            var u = user+"";
-                            return function(err,cb) {
-                                if (err) {
-                                    node.error(err);
-                                    return;
-                                }
-                                if (cb[0]) {
-                                    node.since_ids[u] = cb[0].id_str;
-                                }
-                                else {
-                                    node.since_ids[u] = '0';
-                                }
-                                node.poll_ids.push(setInterval(function() {
-                                    twit.getUserTimeline({
-                                        screen_name:u,
-                                        trim_user:0,
-                                        since_id:node.since_ids[u]
-                                    }, function(err,cb) {
-                                        if (cb) {
-                                            for (var t=cb.length-1; t>=0; t-=1) {
-                                                var tweet = cb[t];
-                                                var where = tweet.user.location;
-                                                var la = tweet.lang || tweet.user.lang;
-                                                var msg = { topic:node.topic+"/"+tweet.user.screen_name, payload:tweet.text, lang:la, tweet:tweet };
-                                                if (where) {
-                                                    msg.location = {place:where};
-                                                    addLocationToTweet(msg);
-                                                }
-                                                node.send(msg);
-                                                if (t === 0) {
-                                                    node.since_ids[u] = tweet.id_str;
-                                                }
-                                            }
-                                        }
-                                        if (err) {
-                                            node.error(err);
-                                        }
-                                    });
-                                },60000));
-                            }
-                        }()));
-                    }
+            if (this.user === "true") {
+                // Poll User Home Timeline 1/min
+                this.poll(60000,"https://api.twitter.com/1.1/statuses/home_timeline.json");
+            } else if (this.user === "user") {
+                var users = node.tags.split(/\s*,\s*/).filter(v=>!!v);
+                if (users.length === 0) {
+                    node.error(RED._("twitter.warn.nousers"));
+                    return;
                 }
-            }
-            else if (this.user === "dm") {
-                node.poll_ids = [];
-                node.status({});
-                twit.getDirectMessages({
-                    screen_name:node.twitterConfig.screen_name,
-                    trim_user:0,
-                    count:1
-                },function(err,cb) {
-                    if (err) {
-                        node.error(err);
-                        return;
-                    }
-                    if (cb[0]) {
-                        node.since_id = cb[0].id_str;
-                    }
-                    else {
-                        node.since_id = '0';
-                    }
-                    node.poll_ids.push(setInterval(function() {
-                        twit.getDirectMessages({
-                            screen_name:node.twitterConfig.screen_name,
-                            trim_user:0,
-                            since_id:node.since_id
-                        },function(err,cb) {
-                                if (cb) {
-                                    for (var t=cb.length-1; t>=0; t-=1) {
-                                        var tweet = cb[t];
-                                        var where = tweet.sender.location;
-                                        var la = tweet.lang || tweet.sender.lang;
-                                        var msg = { topic:node.topic+"/"+tweet.sender.screen_name, payload:tweet.text, lang:la, tweet:tweet };
-                                        if (where) {
-                                            msg.location = {place:where};
-                                            addLocationToTweet(msg);
-                                        }
-                                        node.send(msg);
-                                        if (t === 0) {
-                                            node.since_id = tweet.id_str;
-                                        }
-                                    }
-                                }
-                                if (err) {
-                                    node.error(err);
-                                }
-                            });
-                    },120000));
+                // Poll User timeline
+                users.forEach(function(user) {
+                    node.poll(60000,"https://api.twitter.com/1.1/statuses/user_timeline.json",{screen_name: user});
+                })
+            } else if (this.user === "dm") {
+                node.pollDirectMessages();
+            } else if (this.user === "event") {
+                this.error("This Twitter node is configured to access a user's activity stream. Twitter removed this API in August 2018 and is no longer available.");
+                return;
+            } else if (this.user === "false") {
+                var twit = new Ntwitter({
+                    consumer_key: credentials.consumer_key,
+                    consumer_secret: credentials.consumer_secret,
+                    access_token_key: credentials.access_token,
+                    access_token_secret: credentials.access_token_secret
                 });
-            }
-            else if (this.user === "event") {
-                this.error("This Twitter node is configured to access a user's activity stream. Twitter are withdrawing this API in August 2018 so this feature will be removed from the node in the near future. See https://bit.ly/2kr7InE for details.")
-                try {
-                    var thingu = 'user';
-                    var setupEvStream = function() {
-                        if (node.active) {
-                            twit.stream(thingu, st, function(stream) {
-                                node.status({fill:"green", shape:"dot", text:" "});
-                                node.stream = stream;
-                                stream.on('data', function(tweet) {
-                                    if (tweet.event !== undefined) {
-                                        var where = tweet.source.location;
-                                        var la = tweet.source.lang;
-                                        var msg = { topic:node.topic+"/"+tweet.source.screen_name, payload:tweet.event, lang:la, tweet:tweet };
-                                        if (where) {
-                                            msg.location = {place:where};
-                                            addLocationToTweet(msg);
-                                        }
-                                        node.send(msg);
-                                    }
-                                });
-                                stream.on('limit', function(tweet) {
-                                    node.status({fill:"grey", shape:"dot", text:" "});
-                                    node.tout2 = setTimeout(function() { node.status({fill:"green", shape:"dot", text:" "}); },10000);
-                                });
-                                stream.on('error', function(tweet,rc) {
-                                    //console.log("ERRO",rc,tweet);
-                                    if (rc == 420) {
-                                        node.status({fill:"red", shape:"ring", text:RED._("twitter.errors.ratelimit")});
-                                    }
-                                    else {
-                                        node.status({fill:"red", shape:"ring", text:" "});
-                                        node.warn(RED._("twitter.errors.streamerror",{error:tweet.toString(),rc:rc}));
-                                    }
-                                    twitterRateTimeout = Date.now() + retry;
-                                    if (node.restart) {
-                                        node.tout = setTimeout(function() { setupEvStream() },retry);
-                                    }
-                                });
-                                stream.on('destroy', function (response) {
-                                    //console.log("DEST",response)
-                                    twitterRateTimeout = Date.now() + 15000;
-                                    if (node.restart) {
-                                        node.status({fill:"red", shape:"dot", text:" "});
-                                        node.warn(RED._("twitter.errors.unexpectedend"));
-                                        node.tout = setTimeout(function() { setupEvStream() },15000);
-                                    }
-                                });
-                            });
-                        }
-                    }
-                    setupEvStream();
-                }
-                catch (err) {
-                    node.error(err);
-                }
-            }
-            else {
+
+                // Stream public tweets
                 try {
                     var thing = 'statuses/filter';
                     var tags = node.tags;
@@ -297,15 +258,6 @@ module.exports = function(RED) {
                         }
                     }
 
-                    // ask for users stream instead of public
-                    if (this.user === "true") {
-                        thing = 'user';
-                        // twit.getFriendsIds(node.twitterConfig.screen_name.substr(1), function(err,list) {
-                        //     friends = list;
-                        // });
-                        st = null;
-                    }
-
                     // if 4 numeric tags that look like a geo area then set geo area
                     var bits = node.tags.split(",");
                     if (bits.length == 4) {
@@ -369,6 +321,11 @@ module.exports = function(RED) {
                     this.stream.removeAllListeners();
                     this.stream.destroy();
                 }
+                if (this.timeout_ids) {
+                    for (var i=0; i<this.timeout_ids.length; i++) {
+                        clearTimeout(this.timeout_ids[i]);
+                    }
+                }
                 if (this.poll_ids) {
                     for (var i=0; i<this.poll_ids.length; i++) {
                         clearInterval(this.poll_ids[i]);
@@ -382,6 +339,197 @@ module.exports = function(RED) {
     }
     RED.nodes.registerType("twitter in",TwitterInNode);
 
+    TwitterInNode.prototype.poll = function(interval, url, opts) {
+        var node = this;
+        var opts = opts || {};
+        var pollId;
+        opts.count = 1;
+        this.twitterConfig.get(url,opts).then(function(result) {
+            if (result.status === 429) {
+                node.warn("Rate limit hit. Waiting "+Math.floor(result.rateLimitTimeout/1000)+" seconds to try again");
+                node.timeout_ids.push(setTimeout(function() {
+                    node.poll(interval,url,opts);
+                },result.rateLimitTimeout))
+                return;
+            }
+            node.debug("Twitter Poll, rateLimitRemaining="+result.rateLimitRemaining+" rateLimitTimeout="+Math.floor(result.rateLimitTimeout/1000)+"s");
+            var res = result.body;
+            opts.count = 200;
+            var since = "0";
+            if (res.length > 0) {
+                since = res[0].id_str;
+            }
+            pollId = setInterval(function() {
+                opts.since_id = since;
+                node.twitterConfig.get(url,opts).then(function(result) {
+                    if (result.status === 429) {
+                        node.warn("Rate limit hit. Waiting "+Math.floor(result.rateLimitTimeout/1000)+" seconds to try again");
+                        clearInterval(pollId);
+                        node.timeout_ids.push(setTimeout(function() {
+                            node.poll(interval,url,opts);
+                        },result.rateLimitTimeout))
+                        return;
+                    }
+                    node.debug("Twitter Poll, rateLimitRemaining="+result.rateLimitRemaining+" rateLimitTimeout="+Math.floor(result.rateLimitTimeout/1000)+"s");
+                    var res = result.body;
+                    if (res.errors) {
+                        node.error(res.errors[0].message);
+                        return;
+                    }
+                    if (res.length > 0) {
+                        since = res[0].id_str;
+                        var len = res.length;
+                        for (var i = len-1; i >= 0; i--) {
+                            var tweet = res[i];
+                            if (tweet.user !== undefined) {
+                                var where = tweet.user.location;
+                                var la = tweet.lang || tweet.user.lang;
+                                tweet.text = tweet.text || tweet.full_text;
+                                var msg = {
+                                    topic:"tweets/"+tweet.user.screen_name,
+                                    payload:tweet.text,
+                                    lang:la,
+                                    tweet:tweet
+                                };
+                                if (where) {
+                                    msg.location = {place:where};
+                                    addLocationToTweet(msg);
+                                }
+                                node.send(msg);
+                            }
+                        }
+                    }
+                }).catch(function(err) {
+                    node.error(err);
+                    clearInterval(pollId);
+                    node.timeout_ids.push(setTimeout(function() {
+                        delete opts.since_id;
+                        delete opts.count;
+                        node.poll(interval,url,opts);
+                    },interval))
+                })
+            },interval)
+            node.poll_ids.push(pollId);
+        }).catch(function(err) {
+            node.error(err);
+            node.timeout_ids.push(setTimeout(function() {
+                delete opts.since_id;
+                delete opts.count;
+                node.poll(interval,url,opts);
+            },interval))
+        })
+    }
+
+    TwitterInNode.prototype.pollDirectMessages = function() {
+        var interval = 70000;
+        var node = this;
+        var opts = {};
+        var url = "https://api.twitter.com/1.1/direct_messages/events/list.json";
+        var pollId;
+        opts.count = 50;
+        this.twitterConfig.get(url,opts).then(function(result) {
+            if (result.status === 429) {
+                node.warn("Rate limit hit. Waiting "+Math.floor(result.rateLimitTimeout/1000)+" seconds to try again");
+                node.timeout_ids.push(setTimeout(function() {
+                    node.pollDirectMessages();
+                },result.rateLimitTimeout))
+                return;
+            }
+            node.debug("Twitter DM Poll, rateLimitRemaining="+result.rateLimitRemaining+" rateLimitTimeout="+Math.floor(result.rateLimitTimeout/1000)+"s");
+            var res = result.body;
+            if (res.errors) {
+                throw new Error(res.errors[0].message);
+            }
+            var since = "0";
+            var messages = res.events.filter(tweet => tweet.type === 'message_create' && tweet.id > since);
+            if (messages.length > 0) {
+                since = messages[0].id;
+            }
+            pollId = setInterval(function() {
+                node.twitterConfig.get(url,opts).then(function(result) {
+                    if (result.status === 429) {
+                        node.warn("Rate limit hit. Waiting "+Math.floor(result.rateLimitTimeout/1000)+" seconds to try again");
+                        clearInterval(pollId);
+                        node.timeout_ids.push(setTimeout(function() {
+                            node.pollDirectMessages();
+                        },result.rateLimitTimeout))
+                        return;
+                    }
+                    node.debug("Twitter DM Poll, rateLimitRemaining="+result.rateLimitRemaining+" rateLimitTimeout="+Math.floor(result.rateLimitTimeout/1000)+"s");
+                    var res = result.body;
+                    if (res.errors) {
+                        node.error(res.errors[0].message);
+                        return;
+                    }
+                    var messages = res.events.filter(tweet => tweet.type === 'message_create' && tweet.id > since);
+                    if (messages.length > 0) {
+                        since = messages[0].id;
+                        var len = messages.length;
+                        var missingUsers = {};
+                        var tweets = [];
+                        for (var i = len-1; i >= 0; i--) {
+                            var tweet = messages[i];
+                            // node.log(JSON.stringify(tweet," ",4));
+                            var output = {
+                                id: tweet.id,
+                                id_str: tweet.id,
+                                text: tweet.message_create.message_data.text,
+                                created_timestamp: tweet.created_timestamp,
+                                entities: tweet.message_create.message_data.entities
+                            }
+                            if (!userObjectCache.hasOwnProperty(tweet.message_create.sender_id)) {
+                                missingUsers[tweet.message_create.sender_id] = true;
+                            }
+                            if (!userObjectCache.hasOwnProperty(tweet.message_create.target.recipient_id)) {
+                                missingUsers[tweet.message_create.target.recipient_id] = true;
+                            }
+                            tweets.push(output);
+                        }
+
+                        var missingUsernames = Object.keys(missingUsers).join(",");
+                        return node.twitterConfig.getUsers(missingUsernames).then(function() {
+                            var len = tweets.length;
+                            for (var i = 0;i < len; i++) {
+                                var tweet = messages[i];
+                                var output = tweets[i];
+                                output.sender = userObjectCache[tweet.message_create.sender_id];
+                                output.sender_id = output.sender.id;
+                                output.sender_id_str = output.sender.id_str;
+                                output.sender_screen_name = output.sender.screen_name;
+
+                                output.recipient = userObjectCache[tweet.message_create.target.recipient_id];
+                                output.recipient_id = output.recipient.id;
+                                output.recipient_id_str = output.recipient.id_str;
+                                output.recipient_screen_name = output.recipient.screen_name;
+
+                                if (output.sender_screen_name !== node.twitterConfig.screen_name) {
+                                    var msg = {
+                                        topic:"tweets/"+output.sender_screen_name,
+                                        payload:output.text,
+                                        tweet:output
+                                    };
+                                    node.send(msg);
+                                }
+                            }
+
+                        })
+                    }
+                }).catch(function(err) {
+                    node.error(err);
+                    clearInterval(pollId);
+                    node.timeout_ids.push(setTimeout(function() {
+                        node.pollDirectMessages();
+                    },interval))
+                })
+            },interval)
+            node.poll_ids.push(pollId);
+        }).catch(function(err) {
+            node.error(err);
+            node.timeout_ids.push(setTimeout(function() {
+                node.pollDirectMessages();
+            },interval))
+        })
+    }
 
     function TwitterOutNode(n) {
         RED.nodes.createNode(this,n);
@@ -390,9 +538,8 @@ module.exports = function(RED) {
         this.twitterConfig = RED.nodes.getNode(this.twitter);
         var credentials = RED.nodes.getCredentials(this.twitter);
         var node = this;
-        var dm_user;
-
-        if (credentials && credentials.consumer_key && credentials.consumer_secret && credentials.access_token && credentials.access_token_secret) {
+        node.status({});
+        if (this.twitterConfig.oauth) {
             var twit = new Ntwitter({
                 consumer_key: credentials.consumer_key,
                 consumer_secret: credentials.consumer_secret,
@@ -400,83 +547,135 @@ module.exports = function(RED) {
                 access_token_secret: credentials.access_token_secret
             });
 
-            var oa = new OAuth(
-                "https://api.twitter.com/oauth/request_token",
-                "https://api.twitter.com/oauth/access_token",
-                credentials.consumer_key,
-                credentials.consumer_secret,
-                "1.0",
-                null,
-                "HMAC-SHA1"
-            );
-
-
             node.on("input", function(msg) {
                 if (msg.hasOwnProperty("payload")) {
                     node.status({fill:"blue",shape:"dot",text:"twitter.status.tweeting"});
-
                     if (msg.payload.slice(0,2) == "D ") {
-                            // direct message syntax: "D user message"
-                            var t = msg.payload.match(/D\s+(\S+)\s+(.*)/).slice(1);
-                            dm_user = t[0];
-                            msg.payload = t[1];
-                    }
-                    if (msg.payload.length > 280) {
-                        msg.payload = msg.payload.slice(0,279);
-                        node.warn(RED._("twitter.errors.truncated"));
-                    }
-
-                    if (msg.media && Buffer.isBuffer(msg.media)) {
-                        var apiUrl = "https://api.twitter.com/1.1/statuses/update_with_media.json";
-                        var signedUrl = oa.signUrl(apiUrl,
-                            credentials.access_token,
-                            credentials.access_token_secret,
-                            "POST");
-
-                        var r = request.post(signedUrl,function(err,httpResponse,body) {
-                            if (err) {
-                                node.error(err,msg);
+                        var dm_user;
+                        // direct message syntax: "D user message"
+                        var t = msg.payload.match(/D\s+(\S+)\s+(.*)/).slice(1);
+                        dm_user = t[0];
+                        msg.payload = t[1];
+                        var lookupPromise;
+                        if (userSreenNameToIdCache.hasOwnProperty(dm_user)) {
+                            lookupPromise = Promise.resolve();
+                        } else {
+                            lookupPromise = node.twitterConfig.getUsers(dm_user,"screen_name")
+                        }
+                        lookupPromise.then(function() {
+                            if (userSreenNameToIdCache.hasOwnProperty(dm_user)) {
+                                // Send a direct message
+                                node.twitterConfig.post("https://api.twitter.com/1.1/direct_messages/events/new.json",{
+                                    event: {
+                                        type: "message_create",
+                                        "message_create": {
+                                            "target": {
+                                                "recipient_id": userSreenNameToIdCache[dm_user]
+                                            },
+                                            "message_data": {"text": msg.payload}
+                                        }
+                                    }
+                                }).then(function() {
+                                    node.status({});
+                                }).catch(function(err) {
+                                    node.error(err,msg);
+                                    node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                                });
+                            } else {
+                                node.error("Unknown user",msg);
                                 node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
                             }
-                            else {
-                                var response = JSON.parse(body);
-                                if (response.errors) {
-                                    var errorList = response.errors.map(function(er) { return er.code+": "+er.message }).join(", ");
-                                    node.error(RED._("twitter.errors.sendfail",{error:errorList}),msg);
-                                    node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
-                                }
-                                else {
-                                    node.status({});
-                                }
-                            }
-                        });
-                        var form = r.form();
-                        form.append("status",msg.payload);
-                        form.append("media[]",msg.media,{filename:"image"});
-
-                    }
-                    else {
-                        if (typeof msg.params === 'undefined') { msg.params = {}; }
-                        if (dm_user) {
-                            twit.newDirectMessage(dm_user,msg.payload, msg.params, function (err, data) {
-                                if (err) {
-                                    node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
-                                    node.error(err,msg);
-                                }
-                                node.status({});
-                            });
-                        } else {
-                            twit.updateStatus(msg.payload, msg.params, function (err, data) {
-                                if (err) {
-                                    node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
-                                    node.error(err,msg);
-                                }
-                                node.status({});
-                            });
+                        }).catch(function(err) {
+                            node.error(err,msg);
+                            node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                        })
+                    } else {
+                        if (msg.payload.length > 280) {
+                            msg.payload = msg.payload.slice(0,279);
+                            node.warn(RED._("twitter.errors.truncated"));
                         }
+                        var mediaPromise;
+                        if (msg.media && Buffer.isBuffer(msg.media)) {
+                            // var mediaType = fileType(msg.media);
+                            // if (mediaType === null) {
+                            //     node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                            //     node.error("msg.media is not a valid media object",msg);
+                            //     return;
+                            // }
+                            mediaPromise = node.twitterConfig.post("https://upload.twitter.com/1.1/media/upload.json",null,null,{
+                                media: msg.media
+                            }).then(function(result) {
+                                if (result.status === 200) {
+                                    return result.body.media_id_string;
+                                } else {
+                                    throw new Error(result.body.errors[0]);
+                                }
+                            });
+
+                        } else {
+                            mediaPromise = Promise.resolve();
+                        }
+                        mediaPromise.then(function(mediaId) {
+                            var params = msg.params || {};
+                            params.status = msg.payload;
+                            if (mediaId) {
+                                params.media_ids = mediaId;
+                            }
+                            node.twitterConfig.post("https://api.twitter.com/1.1/statuses/update.json",{},params).then(function(result) {
+                                if (result.status === 200) {
+                                    node.status({});
+                                } else {
+                                    node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                                    node.error(result.body.errors[0].message,msg);
+                                }
+                            }).catch(function(err) {
+                                node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                                node.error(err,msg);
+                            })
+                        }).catch(function(err) {
+                            node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                            node.error(err,msg);
+                        });
+                        // if (msg.payload.length > 280) {
+                        //     msg.payload = msg.payload.slice(0,279);
+                        //     node.warn(RED._("twitter.errors.truncated"));
+                        // }
+                        // if (msg.media && Buffer.isBuffer(msg.media)) {
+                        //     var apiUrl = "https://api.twitter.com/1.1/statuses/update_with_media.json";
+                        //     var signedUrl = oa.signUrl(apiUrl,credentials.access_token,credentials.access_token_secret,"POST");
+                        //     var r = request.post(signedUrl,function(err,httpResponse,body) {
+                        //         if (err) {
+                        //             node.error(err,msg);
+                        //             node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                        //         }
+                        //         else {
+                        //             var response = JSON.parse(body);
+                        //             if (response.errors) {
+                        //                 var errorList = response.errors.map(function(er) { return er.code+": "+er.message }).join(", ");
+                        //                 node.error(RED._("twitter.errors.sendfail",{error:errorList}),msg);
+                        //                 node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                        //             }
+                        //             else {
+                        //                 node.status({});
+                        //             }
+                        //         }
+                        //     });
+                        //     var form = r.form();
+                        //     form.append("status",msg.payload);
+                        //     form.append("media[]",msg.media,{filename:"image"});
+                        //
+                        // } else {
+                        //     if (typeof msg.params === 'undefined') { msg.params = {}; }
+                        //     twit.updateStatus(msg.payload, msg.params, function (err, data) {
+                        //         if (err) {
+                        //             node.status({fill:"red",shape:"ring",text:"twitter.status.failed"});
+                        //             node.error(err,msg);
+                        //         }
+                        //         node.status({});
+                        //     });
+                        // }
                     }
                 }
-                else { node.warn(RED._("twitter.errors.nopayload")); }
             });
         } else {
             this.error(RED._("twitter.errors.missingcredentials"));
diff --git a/social/twitter/package.json b/social/twitter/package.json
index 2499af52..99feedfe 100644
--- a/social/twitter/package.json
+++ b/social/twitter/package.json
@@ -1,11 +1,10 @@
 {
     "name": "node-red-node-twitter",
-    "version": "1.0.1",
+    "version": "1.1.0",
     "description": "A Node-RED node to talk to Twitter",
     "dependencies": {
         "twitter-ng": "0.6.2",
-        "oauth": "0.9.14",
-        "request": "^2.75.0"
+        "request": "^2.88.0"
     },
     "repository": {
         "type": "git",
@@ -21,9 +20,12 @@
             "twitter": "27-twitter.js"
         }
     },
-    "author": {
-        "name": "Dave Conway-Jones",
-        "email": "ceejay@vnet.ibm.com",
-        "url": "http://nodered.org"
-    }
+    "contributors": [
+      {
+        "name": "Nick O'Leary"
+      },
+      {
+        "name": "Dave Conway-Jones"
+      }
+    ]
 }