Merge pull request #1736 from node-red-hitachi/0.19-httpreq

Move to request module

nodes/core/io/21-httprequest.js

@@ -16,9 +16,7 @@
 
 module.exports = function(RED) {
     "use strict";
-    var http = require("follow-redirects").http;
+    var request = require("request");
-    var https = require("follow-redirects").https;
-    var urllib = require("url");
     var mustache = require("mustache");
     var querystring = require("querystring");
     var cookie = require("cookie");
@@ -78,9 +76,13 @@ module.exports = function(RED) {
             if (msg.method && n.method && (n.method === "use")) {
                 method = msg.method.toUpperCase();          // use the msg parameter
             }
-            var opts = urllib.parse(url);
+            var opts = {};
+            opts.url = url;
+            opts.timeout = node.reqTimeout;
             opts.method = method;
             opts.headers = {};
+            opts.encoding = null;  // Force NodeJs to return a Buffer (instead of a string)
+            opts.maxRedirects = 21;
             var ctSet = "Content-Type"; // set default camel case
             var clSet = "Content-Length";
             if (msg.headers) {
@@ -109,7 +111,7 @@ module.exports = function(RED) {
                 }
             }
             if (msg.hasOwnProperty('followRedirects')) {
-                opts.followRedirects = msg.followRedirects;
+                opts.followRedirect = msg.followRedirects;
                 }
             if (msg.cookies) {
                 var cookies = [];
@@ -134,7 +136,10 @@ module.exports = function(RED) {
                 }
             }
             if (this.credentials && this.credentials.user) {
-                opts.auth = this.credentials.user+":"+(this.credentials.password||"");
+                opts.auth = {
+                    user: this.credentials.user,
+                    pass: this.credentials.password||""
+                };
             }
             var payload = null;
 
@@ -160,6 +165,7 @@ module.exports = function(RED) {
                         opts.headers[clSet] = Buffer.byteLength(payload);
                     }
                 }
+                opts.body = payload;
             }
             // revert to user supplied Capitalisation if needed.
             if (opts.headers.hasOwnProperty('content-type') && (ctSet !== 'content-type')) {
@@ -170,7 +176,6 @@ module.exports = function(RED) {
                 opts.headers[clSet] = opts.headers['content-length'];
                 delete opts.headers['content-length'];
             }
-            var urltotest = url;
             var noproxy;
             if (noprox) {
                 for (var i in noprox) {
@@ -180,22 +185,11 @@ module.exports = function(RED) {
             if (prox && !noproxy) {
                 var match = prox.match(/^(http:\/\/)?(.+)?:([0-9]+)?/i);
                 if (match) {
-                    //opts.protocol = "http:";
+                    opts.proxy = prox;
-                    //opts.host = opts.hostname = match[2];
+                } else {
-                    //opts.port = (match[3] != null ? match[3] : 80);
+                    node.warn("Bad proxy url: "+ prox);
-                    opts.headers['Host'] = opts.host;
+                    opts.proxy = null;
-                    var heads = opts.headers;
-                    var path = opts.pathname = opts.href;
-                    opts = urllib.parse(prox);
-                    opts.path = opts.pathname = path;
-                    opts.headers = heads;
-                    opts.method = method;
-                    urltotest = match[0];
-                    if (opts.auth) {
-                        opts.headers['Proxy-Authorization'] = "Basic "+new Buffer(opts.auth).toString('Base64')
-                    }
                 }
-                else { node.warn("Bad proxy url: "+process.env.http_proxy); }
             }
             if (tlsNode) {
                 tlsNode.addTLSOptions(opts);
@@ -204,42 +198,37 @@ module.exports = function(RED) {
                     opts.rejectUnauthorized = msg.rejectUnauthorized;
                 }
             }
-            var req = ((/^https/.test(urltotest))?https:http).request(opts,function(res) {
+            request(opts, function(err, res, body) {
-                // Force NodeJs to return a Buffer (instead of a string)
+                if(err){
-                // See https://github.com/nodejs/node/issues/6038
+                    if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
-                res.setEncoding(null);
+                        node.error(RED._("common.notification.errors.no-response"), msg);
-                delete res._readableState.decoder;
+                        node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
-
+                    }else{
-                msg.statusCode = res.statusCode;
+                        node.error(err,msg);
-                msg.headers = res.headers;
+                        node.status({fill:"red", shape:"ring", text:err.code});
-                msg.responseUrl = res.responseUrl;
-                msg.payload = [];
-
-                if (msg.headers.hasOwnProperty('set-cookie')) {
-                    msg.responseCookies = {};
-                    msg.headers['set-cookie'].forEach(function(c) {
-                        var parsedCookie = cookie.parse(c);
-                        var eq_idx = c.indexOf('=');
-                        var key = c.substr(0, eq_idx).trim()
-                        parsedCookie.value = parsedCookie[key];
-                        delete parsedCookie[key];
-                        msg.responseCookies[key] = parsedCookie;
-
-                    })
-
-                }
-                msg.headers['x-node-red-request-node'] = hashSum(msg.headers);
-                // msg.url = url;   // revert when warning above finally removed
-                res.on('data',function(chunk) {
-                    if (!Buffer.isBuffer(chunk)) {
-                        // if the 'setEncoding(null)' fix above stops working in
-                        // a new Node.js release, throw a noisy error so we know
-                        // about it.
-                        throw new Error("HTTP Request data chunk not a Buffer");
                     }
-                    msg.payload.push(chunk);
+                    msg.payload = err.toString() + " : " + url;
-                });
+                    msg.statusCode = err.code;
-                res.on('end',function() {
+                    node.send(msg);
+                }else{
+                    msg.statusCode = res.statusCode;
+                    msg.headers = res.headers;
+                    msg.responseUrl = res.request.uri.href;
+                    msg.payload = body;
+
+                    if (msg.headers.hasOwnProperty('set-cookie')) {
+                        msg.responseCookies = {};
+                        msg.headers['set-cookie'].forEach(function(c) {
+                            var parsedCookie = cookie.parse(c);
+                            var eq_idx = c.indexOf('=');
+                            var key = c.substr(0, eq_idx).trim()
+                            parsedCookie.value = parsedCookie[key];
+                            delete parsedCookie[key];
+                            msg.responseCookies[key] = parsedCookie;
+                        });
+                    }
+                    msg.headers['x-node-red-request-node'] = hashSum(msg.headers);
+                    // msg.url = url;   // revert when warning above finally removed
                     if (node.metric()) {
                         // Calculate request time
                         var diff = process.hrtime(preRequestTimestamp);
@@ -251,44 +240,19 @@ module.exports = function(RED) {
                         }
                     }
 
-                    // Check that msg.payload is an array - if the req error
+                    // Convert the payload to the required return type
-                    // handler has been called, it will have been set to a string
+                    if (node.ret !== "bin") {
-                    // and the error already handled - so no further action should
+                        msg.payload = msg.payload.toString('utf8'); // txt
-                    // be taken. #1344
-                    if (Array.isArray(msg.payload)) {
-                        // Convert the payload to the required return type
-                        msg.payload = Buffer.concat(msg.payload); // bin
-                        if (node.ret !== "bin") {
-                            msg.payload = msg.payload.toString('utf8'); // txt
 
-                            if (node.ret === "obj") {
+                        if (node.ret === "obj") {
-                                try { msg.payload = JSON.parse(msg.payload); } // obj
+                            try { msg.payload = JSON.parse(msg.payload); } // obj
-                                catch(e) { node.warn(RED._("httpin.errors.json-error")); }
+                            catch(e) { node.warn(RED._("httpin.errors.json-error")); }
-                            }
                         }
-                        node.status({});
-                        node.send(msg);
                     }
-                });
+                    node.status({});
+                    node.send(msg);
+                }
             });
-            req.setTimeout(node.reqTimeout, function() {
-                node.error(RED._("common.notification.errors.no-response"),msg);
-                setTimeout(function() {
-                    node.status({fill:"red",shape:"ring",text:"common.notification.errors.no-response"});
-                },10);
-                req.abort();
-            });
-            req.on('error',function(err) {
-                node.error(err,msg);
-                msg.payload = err.toString() + " : " + url;
-                msg.statusCode = err.code;
-                node.status({fill:"red",shape:"ring",text:err.code});
-                node.send(msg);
-            });
-            if (payload) {
-                req.write(payload);
-            }
-            req.end();
         });
 
         this.on("close",function() {

package.json

@@ -44,7 +44,6 @@
     "cron": "1.3.0",
     "express": "4.16.3",
     "express-session": "1.15.6",
-    "follow-redirects": "1.4.1",
     "fs-extra": "5.0.0",
     "fs.notify": "0.0.4",
     "hash-sum": "1.0.2",
@@ -69,6 +68,7 @@
     "passport-http-bearer": "1.0.1",
     "passport-oauth2-client-password": "0.1.2",
     "raw-body": "2.3.3",
+    "request": "2.87.0",
     "semver": "5.5.0",
     "sentiment": "2.1.0",
     "uglify-js": "3.3.25",

test/nodes/core/io/21-httprequest_spec.js

@@ -796,6 +796,25 @@ describe('HTTP Request Node', function() {
             });
         });
 
+        it('should prevent following redirect when msg.followRedirects is false', function(done) {
+            var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"txt",url:getTestURL('/redirectToText')},
+                {id:"n2", type:"helper"}];
+            helper.load(httpRequestNode, flow, function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('statusCode',302);
+                        msg.should.have.property('responseUrl', getTestURL('/redirectToText'));
+                        done();
+                    } catch(err) {
+                        done(err);
+                    }
+                });
+                n1.receive({payload:"foo",followRedirects:false});
+            });
+        });
+
         it('shuold output an error when request timeout occurred', function(done) {
             var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/timeout')},
                 {id:"n2", type:"helper"}];
@@ -806,7 +825,13 @@ describe('HTTP Request Node', function() {
                 var n2 = helper.getNode("n2");
                 n2.on("input", function(msg) {
                     try {
-                        msg.should.have.property('statusCode','ECONNRESET');
+                        msg.should.have.property('statusCode','ESOCKETTIMEDOUT');
+                        var logEvents = helper.log().args.filter(function(evt) {
+                            return evt[0].type == 'http request';
+                        });
+                        logEvents.should.have.length(1);
+                        var tstmp = logEvents[0][0].timestamp;
+                        logEvents[0][0].should.eql({level:helper.log().ERROR, id:'n1',type:'http request',msg:'common.notification.errors.no-response', timestamp:tstmp});
                         done();
                     } catch(err) {
                         done(err);