From 3107080b19c34c74629691c6ff0e7ab75c1a53e0 Mon Sep 17 00:00:00 2001
From: Neil Kolban <kolban1@kolban.com>
Date: Wed, 20 Apr 2016 13:47:23 -0500
Subject: [PATCH] Email rework (#195)

* Rework of Node-RED email nodes
---
 .gitignore                               |   1 +
 social/email/61-email.html               |  77 ++++-
 social/email/61-email.js                 | 380 ++++++++++++++++-------
 social/email/README.md                   |   4 +-
 social/email/locales/en-US/61-email.json |   7 +-
 social/email/package.json                |   6 +-
 6 files changed, 358 insertions(+), 117 deletions(-)

diff --git a/.gitignore b/.gitignore
index c1ee2880..b9c6d903 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ coverall
 puball.sh
 
 setenv.sh
+/.project
diff --git a/social/email/61-email.html b/social/email/61-email.html
index ed0feee4..4ebc4865 100644
--- a/social/email/61-email.html
+++ b/social/email/61-email.html
@@ -123,6 +123,17 @@
         <label for="node-input-repeat"><i class="fa fa-repeat"></i> <span data-i18n="email.label.repeat"></span></label>
         <input type="text" id="node-input-repeat" style="width: 80px"> <span data-i18n="email.label.seconds">seconds</span>
     </div>
+    <div class="form-row">
+        <label for="node-input-protocol"><i class="fa fa-envelope"></i> <span data-i18n="email.label.protocol"></span></label>
+        <select type="text" id="node-input-protocol">
+            <option value="IMAP">IMAP</option>
+            <option value="POP3">POP3</option>
+        </select>
+    </div>
+    <div class="form-row">
+		<label for="node-input-useSSL"><i class="fa fa-lock"></i> <span data-i18n="email.label.useSSL"></span></label>
+		<input type="checkbox" id="node-input-useSSL" style="width: auto;">
+    </div>
     <div class="form-row">
         <label for="node-input-server"><i class="fa fa-globe"></i> <span data-i18n="email.label.server"></span></label>
         <input type="text" id="node-input-server" placeholder="imap.gmail.com">
@@ -139,17 +150,53 @@
         <label for="node-input-password"><i class="fa fa-lock"></i> <span data-i18n="email.label.password"></span></label>
         <input type="password" id="node-input-password">
     </div>
-    <div class="form-row">
+    <div class="form-row node-input-box">
         <label for="node-input-box"><i class="fa fa-inbox"></i> <span data-i18n="email.label.folder"></span></label>
         <input type="text" id="node-input-box">
     </div>
+    <div class="form-row node-input-disposition">
+		    <label for="node-input-disposition"><i class="fa fa-trash"></i> <span data-i18n="email.label.disposition"></span></label>
+        <select type="text" id="node-input-disposition">
+            <option value="None" selected="selected">None</option>
+            <option value="Read">Mark Read/Answered</option>
+            <option value="Delete">Delete</option>
+        </select>
+    </div>
     <br/>
     <div class="form-row">
         <label for="node-input-name"><i class="fa fa-tag"></i> <span data-i18n="node-red:common.label.name"></span></label>
         <input type="text" id="node-input-name" data-i18n="[placeholder]node-red:common.label.name">
     </div>
-    <div class="form-tips" id="node-tip"><span data-i18n="[html]email.tip.cred"></span></div>
-    <div id="node-input-tip" class="form-tips"><span data-i18n="[html]email.tip.recent"></span></div>
+    <script>
+        var checkPorts = function() {
+            var currentPort = $("#node-input-port").val();
+            if (currentPort === "143" || currentPort === "993" || currentPort === "110" || currentPort == "995") {
+                if ($("#node-input-useSSL").prop("checked") === true) {
+                    $("#node-input-port").val($("#node-input-protocol").val() === "IMAP"?"993":"995");
+                } else {
+                    $("#node-input-port").val($("#node-input-protocol").val() === "IMAP"?"143":"110");
+                }
+            }
+        };
+        
+        $("#node-input-useSSL").change(function(x, y) {
+          console.log("useSSL: x="+ JSON.stringify(x) + ", y=" + y);
+          console.log("Value: " + $("#node-input-useSSL").prop("checked"));
+          checkPorts();
+        });
+        
+        $("#node-input-protocol").change(function() {
+            var protocol = $("#node-input-protocol").val();
+            if (protocol === "IMAP") {
+                $(".node-input-box").show();
+                $(".node-input-disposition").show();
+            } else {
+                $(".node-input-box").hide();
+                $(".node-input-disposition").hide();                
+            }
+            checkPorts();
+        });
+    </script>
 </script>
 
 <script type="text/x-red" data-help-name="e-mail in">
@@ -161,6 +208,25 @@
     <p>Uses the imap module.</p>
     <p><b>Note:</b> this node <i>only</i> gets the most recent single email from the inbox, so set the repeat (polling) time appropriately.</p>
     <p>Note: uses IMAP with SSL to port 993.</p>
+    <p>Any attachments supplied in the incoming email can be found in the <code>msg.attachments</code> property.  This will be an array of objects where
+    each object represents a specific attachments.  The format of the object is:</p>
+    
+<pre>
+{
+  contentType:        // The MIME content description
+  fileName:           // A suggested file name associated with this attachment
+  transferEncoding:   // How was the original email attachment encodded?
+  contentDisposition: // Unknown
+  generatedFileName:  // A suggested file name associated with this attachment
+  contentId:          // A unique generated ID for this attachment
+  checksum:           // A checksum against the data
+  length:             // Size of data in bytes
+  content:            // The actual content of the data contained in a Node.js Buffer object
+                      // We can turn this into a base64 data string with content.toString('base64')
+}
+</pre>
+<p>For POP3, the default port numbers are 110 for plain TCP and 995 for SSL.  For IMAP the port numbers are 143 for plain TCP and 993 for SSL.</p>
+
 </script>
 
 <script type="text/javascript">
@@ -170,9 +236,12 @@
         color:"#c7e9c0",
         defaults: {
             name: {value:""},
+            protocol: {value: "IMAP", required:true}, // Which protocol to use to connect to the mail server ("IMAP" or "POP3")
             server: {value:"imap.gmail.com",required:true},
+            useSSL: {value: true},
             port: {value:"993",required:true},
-            box: {value:"INBOX"},
+            box: {value:"INBOX"}, // For IMAP, The mailbox to process 
+            disposition: { value: "None" }, // For IMAP, the disposition of the read email
             repeat: {value:"300",required:true}
         },
         credentials: {
diff --git a/social/email/61-email.js b/social/email/61-email.js
index d340b7a7..7c67bfda 100644
--- a/social/email/61-email.js
+++ b/social/email/61-email.js
@@ -14,12 +14,23 @@
  * limitations under the License.
  **/
 
+/**
+ * POP3 protocol - RFC1939 - https://www.ietf.org/rfc/rfc1939.txt
+ * 
+ * Dependencies:
+ * * poplib     - https://www.npmjs.com/package/poplib
+ * * nodemailer - https://www.npmjs.com/package/nodemailer
+ * * imap       - https://www.npmjs.com/package/imap
+ * * mailparser - https://www.npmjs.com/package/mailparser
+ */
+
 module.exports = function(RED) {
     "use strict";
     var nodemailer = require("nodemailer");
-    var Imap = require('imap');
-
-    //console.log(nodemailer.Transport.transports.SMTP.wellKnownHosts);
+    var Imap = require('imap');   
+    var POP3Client = require("poplib");
+    var MailParser = require("mailparser").MailParser;
+    var util = require("util");
 
     try {
         var globalkeys = RED.settings.email || require(process.env.NODE_RED_HOME+"/../emailkeys.js");
@@ -122,14 +133,26 @@ module.exports = function(RED) {
             global: { type:"boolean"}
         }
     });
+    
 
+  
+//  
+// EmailInNode
+//
+// Setup the EmailInNode
     function EmailInNode(n) {
+    var imap;
+    
         RED.nodes.createNode(this,n);
-        this.name = n.name;
-        this.repeat = n.repeat * 1000 || 300000;
-        this.inserver = n.server || (globalkeys && globalkeys.server) || "imap.gmail.com";
-        this.inport = n.port || (globalkeys && globalkeys.port) || "993";
-        this.box = n.box || "INBOX";
+        this.name        = n.name;
+        this.repeat      = n.repeat * 1000 || 300000;
+        this.inserver    = n.server || (globalkeys && globalkeys.server) || "imap.gmail.com";
+        this.inport      = n.port || (globalkeys && globalkeys.port) || "993";
+        this.box         = n.box || "INBOX";
+        this.useSSL      = n.useSSL;
+        this.protocol    = n.protocol || "IMAP";
+        this.disposition = n.disposition || "None"; // "None", "Delete", "Read"
+        
         var flag = false;
 
         if (this.credentials && this.credentials.hasOwnProperty("userid")) {
@@ -158,105 +181,242 @@ module.exports = function(RED) {
 
         var node = this;
         this.interval_id = null;
-        var oldmail = {};
+ 
+ // Process a new email message by building a Node-RED message to be passed onwards
+ // in the message flow.  The parameter called `msg` is the template message we
+ // start with while `mailMessage` is an object returned from `mailparser` that
+ // will be used to populate the email.       
+        function processNewMessage(msg, mailMessage) {
+            msg = JSON.parse(JSON.stringify(msg)); // Clone the message
+            
+            // Populate the msg fields from the content of the email message
+            // that we have just parsed.
+            msg.html = mailMessage.html;
+            msg.payload = mailMessage.text;
+            if (mailMessage.attachments) {
+                msg.attachments = mailMessage.attachments;
+            } else {
+                msg.attachments = [];
+            }
+            msg.topic = mailMessage.subject;
+            msg.header = mailMessage.headers;
+            msg.date = mailMessage.date;
+            if (mailMessage.from && mailMessage.from.length > 0) {
+                msg.from = mailMessage.from[0].address;
+            }
+            
+            node.send(msg); // Propagate the message down the flow
+        }; // End of processNewMessage
+        
+// Check the POP3 email mailbox for any new messages.  For any that are found,
+// retrieve each message, call processNewMessage to process it and then delete
+// the messages from the server.
+        function checkPOP3(msg) {
+            var currentMessage;
+            var maxMessage;
 
-        var imap = new Imap({
-            user: node.userid,
-            password: node.password,
-            host: node.inserver,
-            port: node.inport,
-            tls: true,
-            tlsOptions: { rejectUnauthorized: false },
-            connTimeout: node.repeat,
-            authTimeout: node.repeat
-        });
+// Form a new connection to our email server using POP3.                    
+            var pop3Client = new POP3Client(
+                node.inport, node.inserver,
+                {enabletls: node.useSSL} // Should we use SSL to connect to our email server?
+            );
 
-        if (!isNaN(this.repeat) && this.repeat > 0) {
-            this.interval_id = setInterval( function() {
-                node.emit("input",{});
-            }, this.repeat );
-        }
+// If we have a next message to retrieve, ask to retrieve it otherwise issue a 
+// quit request.
+            function nextMessage() {
+                if (currentMessage > maxMessage) {
+                    pop3Client.quit();
+                    return;
+                }
+                pop3Client.retr(currentMessage);
+                currentMessage++;
+            } // End of nextMessage
+            
+            pop3Client.on("stat", function(status, data) {
+// Data contains:
+// {
+//   count: <Number of messages to be read>
+//   octect: <size of messages to be read>
+// }
+                if (status) {
+                  currentMessage = 1;
+                  maxMessage = data.count;
+                  nextMessage();
+                } else {
+                  node.log(util.format("stat error: %s %j", status, data));
+                }
+            });
+            
+            pop3Client.on("error", function(err) {
+                node.log("We caught an error: " + JSON.stringify(err));
+            });
+
+            pop3Client.on("connect", function() {
+                //node.log("We are now connected");
+                pop3Client.login("kolban@test.com", "password");
+            });
+
+            pop3Client.on("login", function(status, rawData) {
+                //node.log("login: " + status + ", " + rawData);
+                if (status) {
+                    pop3Client.stat();
+                } else {
+                    node.log(util.format("login error: %s %j", status, rawData));
+                    pop3Client.quit();
+                }
+            });
+
+            pop3Client.on("retr", function(status, msgNumber, data, rawData) {
+                node.log(util.format("retr: status=%s, msgNumber=%d, data=%j", status, msgNumber, data));
+                if (status) {
+
+// We have now received a new email message.  Create an instance of a mail parser
+// and pass in the email message.  The parser will signal when it has parsed the message.                    
+                    var mailparser = new MailParser();
+                    mailparser.on("end", function(mailObject) {
+                        //node.log(util.format("mailparser: on(end): %j", mailObject));
+                        processNewMessage(msg, mailObject);
+                    });
+                    mailparser.write(data);
+                    mailparser.end();
+                    pop3Client.dele(msgNumber);
+                } 
+                else {
+                    node.log(util.format("retr error: %s %j", status, rawData));
+                    pop3Client.quit();
+                }
+            });
+
+            pop3Client.on("invalid-state", function(cmd) {
+                node.log("Invalid state: " + cmd);
+            });
+
+            pop3Client.on("locked", function(cmd) {
+                node.log("We were locked: " + cmd);
+            });
+            
+// When we have deleted the last processed message, we can move on to
+// processing the next message.
+            pop3Client.on("dele", function(status, msgNumber) {
+                nextMessage();
+            });
+        }; // End of checkPOP3
+
+
+//
+// checkIMAP
+//        
+// Check the email sever using the IMAP protocol for new messages.        
+        function checkIMAP(msg) {
+      node.log("Checkimg IMAP for new messages");   
+      // We get back a 'ready' event once we have connected to imap
+      imap.once("ready", function() {
+        node.status({fill:"blue", shape:"dot", text:"email.status.fetching"});
+        console.log("> ready");
+        // Open the inbox folder
+        imap.openBox('INBOX', // Mailbox name
+          false, // Open readonly?
+          function(err, box) {
+          console.log("> Inbox open: %j", box);
+          imap.search([ 'UNSEEN' ], function(err, results) {
+            if (err) {
+              node.status({fill:"red", shape:"ring", text:"email.status.foldererror"});
+              node.error(RED._("email.errors.fetchfail", {folder:node.box}),err);
+              return;
+            }
+            console.log("> search - err=%j, results=%j", err, results);
+            if (results.length === 0) {
+              console.log(" [X] - Nothing to fetch");
+              return;
+            }
+
+            // We have the search results that contain the list of unseen messages and can now fetch those messages.
+            var fetch = imap.fetch(results, {
+              bodies : ["HEADER", "TEXT"],
+              markSeen : true
+            });
+
+            // For each fetched message returned ...
+            fetch.on('message', function(imapMessage, seqno) {
+              node.log(RED._("email.status.message",{number:seqno}));
+              var messageText = "";
+              console.log("> Fetch message - msg=%j, seqno=%d", imapMessage, seqno);
+              imapMessage.on('body', function(stream, info) {
+                console.log("> message - body - stream=?, info=%j", info);
+                // Info defined which part of the message this is ... for example
+                // 'TEXT' or 'HEADER'
+                stream.on('data', function(chunk) {
+                  console.log("> stream - data - chunk=??");
+                  messageText += chunk.toString('utf8');
+                });
+              }); // End of msg->body
+              // When the `end` event is raised on the message
+              imapMessage.once('end', function() {
+                console.log("> msg - end : %j", messageText);
+                var mailparser = new MailParser();
+                mailparser.on("end", function(mailMessage) {
+                  //console.log("mailparser: on(end): %j", mailMessage);
+                  processNewMessage(msg, mailMessage);
+                });
+                mailparser.write(messageText);
+                mailparser.end();
+              }); // End of msg->end
+            }); // End of fetch->message
+
+            // When we have fetched all the messages, we don't need the imap connection any more.
+            fetch.on('end', function() {
+               var cleanup = function() {
+                   node.status({});
+                   imap.end();
+               };
+               if (this.disposition == "Delete") {
+                  imap.addFlags(results, "\Deleted", cleanup);
+               } else if (this.disposition == "Read") {
+                  imap.addFlags(results, "\Answered", cleanup);
+               } else {
+                  cleanup();
+               }
+            });
+
+            fetch.once('error', function(err) {
+                console.log('Fetch error: ' + err);
+            });
+          }); // End of imap->search
+        }); // End of imap->openInbox
+      }); // End of imap->ready   
+      imap.connect();
+      node.status({fill:"grey",shape:"dot",text:"node-red:common.status.connecting"});
+    }; // End of checkIMAP
+
+        
+// Perform a check of the email inboxes using either POP3 or IMAP        
+        function checkEmail(msg) {
+            if (node.protocol === "POP3") {
+                checkPOP3(msg);
+            } else if (node.protocol === "IMAP") {
+                checkIMAP(msg);
+            }
+        }; // End of checkEmail
+
+        if (node.protocol === "IMAP") {
+      imap = new Imap({
+        user: node.userid,
+        password: node.password,
+        host: node.inserver,
+        port: node.inport,
+        tls: node.useSSL,
+        tlsOptions: { rejectUnauthorized: false },
+        connTimeout: node.repeat,
+        authTimeout: node.repeat
+      });
+      imap.on('error', function(err) {
+        node.log(err);
+        node.status({fill:"red",shape:"ring",text:"email.status.connecterror"});
+      });
+    };     
 
         this.on("input", function(msg) {
-            imap.once('ready', function() {
-                node.status({fill:"blue",shape:"dot",text:"email.status.fetching"});
-                var pay = {};
-                imap.openBox(node.box, false, function(err, box) {
-                    if (err) {
-                        node.status({fill:"red",shape:"ring",text:"email.status.foldererror"});
-                        node.error(RED._("email.errors.fetchfail",{folder:node.box}),err);
-                    }
-                    else {
-                        if (box.messages.total > 0) {
-                            //var f = imap.seq.fetch(box.messages.total + ':*', { markSeen:true, bodies: ['HEADER.FIELDS (FROM SUBJECT DATE TO CC BCC)','TEXT'] });
-                            var f = imap.seq.fetch(box.messages.total + ':*', { markSeen:true, bodies: ['HEADER','TEXT'] });
-                            f.on('message', function(msg, seqno) {
-                                node.log(RED._("email.status.message",{number:seqno}));
-                                var prefix = '(#' + seqno + ') ';
-                                msg.on('body', function(stream, info) {
-                                    var buffer = '';
-                                    stream.on('data', function(chunk) {
-                                        buffer += chunk.toString('utf8');
-                                    });
-                                    stream.on('end', function() {
-                                        if (info.which !== 'TEXT') {
-                                            var head = Imap.parseHeader(buffer);
-                                            if (head.hasOwnProperty("from")) { pay.from = head.from[0]; }
-                                            if (head.hasOwnProperty("subject")) { pay.topic = head.subject[0]; }
-                                            if (head.hasOwnProperty("date")) { pay.date = head.date[0]; }
-                                            pay.header = head;
-                                        } else {
-                                            var parts = buffer.split("Content-Type");
-                                            for (var p = 0; p < parts.length; p++) {
-                                                if (parts[p].indexOf("text/plain") >= 0) {
-                                                    pay.payload = parts[p].split("\n").slice(1,-2).join("\n").trim();
-                                                }
-                                                else if (parts[p].indexOf("text/html") >= 0) {
-                                                    pay.html = parts[p].split("\n").slice(1,-2).join("\n").trim();
-                                                } else {
-                                                    pay.payload = parts[0];
-                                                }
-                                            }
-                                            //pay.body = buffer;
-                                        }
-                                    });
-                                });
-                                msg.on('end', function() {
-                                    //node.log('finished: '+prefix);
-                                });
-                            });
-                            f.on('error', function(err) {
-                                node.warn(RED._("email.errors.messageerror",{error:err}));
-                                node.status({fill:"red",shape:"ring",text:"email.status.messageerror"});
-                            });
-                            f.on('end', function() {
-                                delete(pay._msgid);
-                                if (JSON.stringify(pay) !== oldmail) {
-                                    oldmail = JSON.stringify(pay);
-                                    node.send(pay);
-                                    node.log(RED._("email.status.newemail",{topic:pay.topic}));
-                                }
-                                else { node.log(RED._("email.status.duplicate",{topic:pay.topic})); }
-                                //node.status({fill:"green",shape:"dot",text:"node-red:common.status.ok"});
-                                node.status({});
-                            });
-                        }
-                        else {
-                            node.log(RED._("email.status.inboxzero"));
-                            //node.status({fill:"green",shape:"dot",text:"node-red:common.status.ok"});
-                            node.status({});
-                        }
-                    }
-                    imap.end();
-                });
-            });
-            node.status({fill:"grey",shape:"dot",text:"node-red:common.status.connecting"});
-            imap.connect();
-        });
-
-        imap.on('error', function(err) {
-            node.log(err);
-            node.status({fill:"red",shape:"ring",text:"email.status.connecterror"});
+          checkEmail(msg);
         });
 
         this.on("close", function() {
@@ -266,13 +426,21 @@ module.exports = function(RED) {
             if (imap) { imap.destroy(); }
         });
 
+// Set the repetition timer as needed
+    if (!isNaN(this.repeat) && this.repeat > 0) {
+      this.interval_id = setInterval( function() {
         node.emit("input",{});
+      }, this.repeat );
     }
+    
+    node.emit("input",{});
+    }
+    
     RED.nodes.registerType("e-mail in",EmailInNode,{
         credentials: {
-            userid: {type:"text"},
-            password: {type: "password"},
-            global: { type:"boolean"}
+            userid:   { type:"text" },
+            password: { type: "password" },
+            global:   { type:"boolean" }
         }
     });
 };
diff --git a/social/email/README.md b/social/email/README.md
index 029214f6..2e1c2175 100644
--- a/social/email/README.md
+++ b/social/email/README.md
@@ -3,8 +3,6 @@ node-red-node-email
 
 <a href="http://nodered.org" target="_new">Node-RED</a> nodes to send and receive simple emails.
 
-**Note** : This is the same node as was in the core of Node-RED.
-As of v0.10.8 it will be installed from here instead.
 
 Pre-requisite
 -------------
@@ -25,7 +23,7 @@ Nodes to send and receive simple emails.
 
 ### Input
 
-Repeatedly gets a **single email** from an IMAP server and forwards on as a msg if not already seen.
+Repeatedly gets emails from an IMAP or POP3 server and forwards them onwards as messages if not already seen.
 
 The subject is loaded into `msg.topic` and `msg.payload` is the plain text body.
 If there is text/html then that is returned in `msg.html`. `msg.from` and
diff --git a/social/email/locales/en-US/61-email.json b/social/email/locales/en-US/61-email.json
index 453c13ca..b87560e5 100644
--- a/social/email/locales/en-US/61-email.json
+++ b/social/email/locales/en-US/61-email.json
@@ -8,7 +8,10 @@
             "password": "Password",
             "repeat": "Refresh",
             "seconds": "seconds",
-            "folder": "Folder"
+            "folder": "Folder",
+            "protocol": "Protocol",
+            "useSSL": "Use SSL?",
+            "disposition": "Disposition"
         },
         
         "default-message": "Your file from Node-RED is attached: __filename__ __description__",
@@ -40,4 +43,4 @@
             "messageerror": "Fetch message error: __error__"
         }
     }
-}
\ No newline at end of file
+}
diff --git a/social/email/package.json b/social/email/package.json
index d5b0ea6e..902fc9aa 100644
--- a/social/email/package.json
+++ b/social/email/package.json
@@ -1,10 +1,12 @@
 {
     "name"          : "node-red-node-email",
-    "version"       : "0.1.3",
+    "version"       : "0.1.4",
     "description"   : "Node-RED nodes to send and receive simple emails",
     "dependencies"  : {
         "nodemailer"   : "1.3.4",
-        "imap" : "0.8.16"
+        "imap" : "0.8.16",
+        "poplib": "^0.1.7",
+        "mailparser": "^0.5.3"        
     },
     "repository" : {
         "type":"git",