Initial migration from request to got for http-request node

package.json

@@ -41,9 +41,12 @@
         "denque": "1.5.0",
         "express": "4.17.1",
         "express-session": "1.17.1",
+        "form-data": "4.0.0",
         "fs-extra": "9.1.0",
         "fs.notify": "0.0.4",
+        "got": "11.8.2",
         "hash-sum": "2.0.0",
+        "hpagent": "0.1.1",
         "https-proxy-agent": "5.0.0",
         "i18next": "20.2.1",
         "iconv-lite": "0.6.2",
@@ -70,7 +73,9 @@
         "request": "2.88.0",
         "semver": "7.3.5",
         "tar": "6.1.0",
+        "tough-cookie": "4.0.0",
         "uglify-js": "3.13.3",
+        "uuid": "8.3.2",
         "ws": "6.2.1",
         "xml2js": "0.4.23"
     },

packages/node_modules/@node-red/nodes/core/network/21-httprequest.js

@@ -16,7 +16,13 @@
 
 module.exports = function(RED) {
     "use strict";
-    var request = require("request");
+    const got = require("got");
+    const {CookieJar} = require("tough-cookie");
+    const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent');
+    const FormData = require('form-data');
+    const { v4: uuid } = require('uuid');
+    const crypto = require('crypto');
+    const URL = require("url").URL
     var mustache = require("mustache");
     var querystring = require("querystring");
     var cookie = require("cookie");
@@ -30,6 +36,8 @@ module.exports = function(RED) {
         var nodeMethod = n.method || "GET";
         var paytoqs = false;
         var paytobody = false;
+        var redirectList = [];
+
         var nodeHTTPPersistent = n["persist"];
         if (n.tls) {
             var tlsNode = RED.nodes.getNode(n.tls);
@@ -63,6 +71,7 @@ module.exports = function(RED) {
             if (msg.url && nodeUrl && (nodeUrl !== msg.url)) {  // revert change below when warning is finally removed
                 node.warn(RED._("common.errors.nooverride"));
             }
+
             if (isTemplatedUrl) {
                 url = mustache.render(nodeUrl,msg);
             }
@@ -71,6 +80,8 @@ module.exports = function(RED) {
                 nodeDone();
                 return;
             }
+
+
             // url must start http:// or https:// so assume http:// if not set
             if (url.indexOf("://") !== -1 && url.indexOf("http") !== 0) {
                 node.warn(RED._("httpin.errors.invalid-transport"));
@@ -86,6 +97,7 @@ module.exports = function(RED) {
                 }
             }
 
+
             var method = nodeMethod.toUpperCase() || "GET";
             if (msg.method && n.method && (n.method !== "use")) {     // warn if override option not set
                 node.warn(RED._("common.errors.nooverride"));
@@ -97,16 +109,15 @@ module.exports = function(RED) {
             var isHttps = (/^https/i.test(url));
 
             var opts = {};
-            opts.url = url;
             // set defaultport, else when using HttpsProxyAgent, it's defaultPort of 443 will be used :(.
             opts.defaultPort = isHttps?443:80;
             opts.timeout = node.reqTimeout;
             opts.method = method;
             opts.headers = {};
-            opts.encoding = null;  // Force NodeJs to return a Buffer (instead of a string)
+            opts.retry = 0;
+            opts.responseType = 'buffer';
             opts.maxRedirects = 21;
-            opts.jar = request.jar();
+            opts.cookieJar = new CookieJar();
-            opts.proxy = null;
             opts.forever = nodeHTTPPersistent;
             if (msg.requestTimeout !== undefined) {
                 if (isNaN(msg.requestTimeout)) {
@@ -117,6 +128,21 @@ module.exports = function(RED) {
                     opts.timeout = msg.requestTimeout;
                 }
             }
+
+            opts.hooks = {
+                beforeRedirect: [
+                    (options, response) => {
+                        let redirectInfo = {
+                            location: response.headers.location
+                        }
+                        if (response.headers.hasOwnProperty('set-cookie')) {
+                            redirectInfo.cookies = extractCookies(response.headers['set-cookie']);
+                        }
+                        redirectList.push(redirectInfo)
+                    }
+                ]
+            }
+
             var ctSet = "Content-Type"; // set default camel case
             var clSet = "Content-Length";
             if (msg.headers) {
@@ -144,30 +170,15 @@ module.exports = function(RED) {
                     }
                 }
             }
+
             if (msg.hasOwnProperty('followRedirects')) {
-                opts.followRedirect = msg.followRedirects;
+                opts.followRedirect = !!msg.followRedirects;
-                opts.followAllRedirects = !!opts.followRedirect;
-            }
-            var redirectList = [];
-            if (!opts.hasOwnProperty('followRedirect') || opts.followRedirect) {
-                opts.followRedirect = function(res) {
-                    var redirectInfo = {
-                        location: res.headers.location,
-                    };
-                    if (res.headers.hasOwnProperty('set-cookie')) {
-                        redirectInfo.cookies = extractCookies(res.headers['set-cookie']);
-                    }
-                    redirectList.push(redirectInfo);
-                    if (this.headers.cookie) {
-                        delete this.headers.cookie;
-                    }
-                    return true;
-                };
             }
+
             if (opts.headers.hasOwnProperty('cookie')) {
                 var cookies = cookie.parse(opts.headers.cookie, {decode:String});
                 for (var name in cookies) {
-                    opts.jar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url);
+                    opts.cookieJar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url);
                 }
                 delete opts.headers.cookie;
             }
@@ -180,13 +191,13 @@ module.exports = function(RED) {
                         } else if (typeof msg.cookies[name] === 'object') {
                             if(msg.cookies[name].encode === false){
                                 // If the encode option is false, the value is not encoded.
-                                opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url);
+                                opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url);
                             } else {
                                 // The value is encoded by encodeURIComponent().
-                                opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value), url);
+                                opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value), url);
                             }
                         } else {
-                            opts.jar.setCookie(cookie.serialize(name, msg.cookies[name]), url);
+                            opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name]), url);
                         }
                     }
                 }
@@ -194,52 +205,62 @@ module.exports = function(RED) {
             if (this.credentials) {
                 if (this.authType === "basic") {
                     if (this.credentials.user) {
-                        opts.auth = {
+                        opts.username = this.credentials.user;
-                            user: this.credentials.user,
+                    }
-                            pass: this.credentials.password || ""
+                    if (this.credentials.password) {
-                        };
+                        opts.password = this.credentials.password;
                     }
                 } else if (this.authType === "digest") {
-                    if (this.credentials.user) {
+                    let digestCreds = this.credentials;
-                        // The first request will be sent without auth information.  Based on the 401 response, the library can determine
+                    let sentCreds = false;
-                        // which auth type is required by the server.  Then the request is resubmitted with the appropriate auth header.
+                    opts.hooks.afterResponse = [(response, retry) => {
-                        opts.auth = {
+                        if (response.statusCode === 401) {
-                            user: this.credentials.user,
+                            if (sentCreds) {
-                            pass: this.credentials.password || "",
+                                return response
-                            sendImmediately: false
-                        };
                             }
+                            const requestUrl = new URL(response.request.requestUrl);
+                            const options = response.request.options;
+                            const normalisedHeaders = {};
+                            Object.keys(response.headers).forEach(k => {
+                                normalisedHeaders[k.toLowerCase()] = response.headers[k]
+                            })
+                            if (normalisedHeaders['www-authenticate']) {
+                                let authHeader = buildDigestHeader(digestCreds.user,digestCreds.password, options.method, requestUrl.pathname, normalisedHeaders['www-authenticate'])
+                                options.headers.authorization = authHeader;
+                            }
+                            sentCreds = true;
+                            return retry(options);
+                        }
+                        return response
+                    }];
                 } else if (this.authType === "bearer") {
-                    opts.auth = {
+                    opts.headers.Authorization = `Bearer ${this.credentials.password||""}`
-                        bearer: this.credentials.password || ""
-                    };
                 }
             }
             var payload = null;
 
+
             if (method !== 'GET' && method !== 'HEAD' && typeof msg.payload !== "undefined") {
                 if (opts.headers['content-type'] == 'multipart/form-data' && typeof msg.payload === "object") {
-                    opts.formData = {};
+                    let formData = new FormData();
-
                     for (var opt in msg.payload) {
                         if (msg.payload.hasOwnProperty(opt)) {
                             var val = msg.payload[opt];
                             if (val !== undefined && val !== null) {
                                 if (typeof val === 'string' || Buffer.isBuffer(val)) {
-                                    opts.formData[opt] = val;
+                                    formData.append(opt, val);
                                 } else if (typeof val === 'object' && val.hasOwnProperty('value')) {
-                                    // Treat as file to upload - ensure it has an options object
+                                    formData.append(opt,val.value,val.options || {});
-                                    // as request complains if it doesn't
-                                    if (!val.hasOwnProperty('options')) {
-                                        val.options = {};
-                                    }
-                                    opts.formData[opt] = val;
                                 } else {
-                                    opts.formData[opt] = JSON.stringify(val);
+                                    formData.append(opt,JSON.stringify(val));
                                 }
                             }
                         }
                     }
+                    // GOT will only set the content-type header with the correct boundary
+                    // if the header isn't set. So we delete it here, for GOT to reset it.
+                    delete opts.headers['content-type'];
+                    opts.body = formData;
                 } else {
                     if (typeof msg.payload === "string" || Buffer.isBuffer(msg.payload)) {
                         payload = msg.payload;
@@ -266,25 +287,29 @@ module.exports = function(RED) {
                 }
             }
 
+
             if (method == 'GET' && typeof msg.payload !== "undefined" && paytoqs) {
                 if (typeof msg.payload === "object") {
                     try {
-                        if (opts.url.indexOf("?") !== -1) {
+                        if (url.indexOf("?") !== -1) {
-                            opts.url += (opts.url.endsWith("?")?"":"&") + querystring.stringify(msg.payload);
+                            url += (url.endsWith("?")?"":"&") + querystring.stringify(msg.payload);
                         } else {
-                            opts.url += "?" + querystring.stringify(msg.payload);
+                            url += "?" + querystring.stringify(msg.payload);
                         }
                     } catch(err) {
+
                         node.error(RED._("httpin.errors.invalid-payload"),msg);
                         nodeDone();
                         return;
                     }
                 } else {
+
                     node.error(RED._("httpin.errors.invalid-payload"),msg);
                     nodeDone();
                     return;
                 }
             } else if ( method == "GET" && typeof msg.payload !== "undefined" && paytobody) {
+                opts.allowGetBody = true;
                 if (typeof msg.payload === "object") {
                     opts.body = JSON.stringify(msg.payload);
                 } else if (typeof msg.payload == "number") {
@@ -311,49 +336,55 @@ module.exports = function(RED) {
                 }
             }
             if (prox && !noproxy) {
-                var match = prox.match(/^(http:\/\/)?(.+)?:([0-9]+)?/i);
+                var match = prox.match(/^(https?:\/\/)?(.+)?:([0-9]+)?/i);
                 if (match) {
-                    opts.proxy = prox;
+                    let proxyAgent;
+                    let proxyOptions = {
+                        proxy: prox,
+                        maxFreeSockets: 256,
+                        maxSockets: 256,
+                        keepAlive: true
+                    }
+                    if (proxyConfig && proxyConfig.credentials) {
+                        let proxyUsername = proxyConfig.credentials.username || '';
+                        let proxyPassword = proxyConfig.credentials.password || '';
+                        if (proxyUsername || proxyPassword) {
+                            var m = /^(https?:\/\/)(.*)$/.exec(prox);
+                            proxyOptions.proxy = `${m[1]}${proxyUsername}:${proxyPassword}@${m[2]}`
+                        }
+                    }
+                    opts.agent = {};
+                    if (/^http:/.test(url)) {
+                        opts.agent.http = new HttpProxyAgent(proxyOptions)
+                    } else {
+                        opts.agent.https = new HttpsProxyAgent(proxyOptions)
+                    }
+                    console.log("ProxyOptions:",proxyOptions);
                 } else {
                     node.warn("Bad proxy url: "+ prox);
-                    opts.proxy = null;
-                }
-            }
-            if (proxyConfig && proxyConfig.credentials && opts.proxy == proxyConfig.url) {
-                var proxyUsername = proxyConfig.credentials.username || '';
-                var proxyPassword = proxyConfig.credentials.password || '';
-                if (proxyUsername || proxyPassword) {
-                    opts.headers['proxy-authorization'] =
-                        'Basic ' +
-                        Buffer.from(proxyUsername + ':' + proxyPassword).toString('base64');
                 }
             }
             if (tlsNode) {
-                tlsNode.addTLSOptions(opts);
+                opts.https = {};
+                tlsNode.addTLSOptions(opts.https);
             } else {
                 if (msg.hasOwnProperty('rejectUnauthorized')) {
-                    opts.rejectUnauthorized = msg.rejectUnauthorized;
+                    opts.https = { rejectUnauthorized: msg.rejectUnauthorized };
                 }
             }
-            request(opts, function(err, res, body) {
+            got(url,opts).then(res => {
-                if(err){
-                    if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
-                        node.error(RED._("common.notification.errors.no-response"), msg);
-                        node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
-                    }else{
-                        node.error(err,msg);
-                        node.status({fill:"red", shape:"ring", text:err.code});
-                    }
-                    msg.payload = err.toString() + " : " + url;
-                    msg.statusCode = err.code;
-                    nodeSend(msg);
-                    nodeDone();
-                }else{
                 msg.statusCode = res.statusCode;
                 msg.headers = res.headers;
-                    msg.responseUrl = res.request.uri.href;
+                let hostname = res.req.host;
-                    msg.payload = body;
+                if (res.req.protocol === "http:" && res.socket.remotePort !== 80) {
+                    hostname = `${hostname}:${res.socket.remotePort}`
+                } else if (res.req.protocol === "https:" && res.socket.remotePort !== 443) {
+                    hostname = `${hostname}:${res.socket.remotePort}`
+                }
+                msg.responseUrl = `${res.req.protocol}//${hostname}${res.req.path}`;
+                msg.payload = res.body;
                 msg.redirectList = redirectList;
+                msg.retry = 0;
 
                 if (msg.headers.hasOwnProperty('set-cookie')) {
                     msg.responseCookies = extractCookies(msg.headers['set-cookie']);
@@ -383,7 +414,18 @@ module.exports = function(RED) {
                 node.status({});
                 nodeSend(msg);
                 nodeDone();
+            }).catch(err => {
+                if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
+                    node.error(RED._("common.notification.errors.no-response"), msg);
+                    node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
+                }else{
+                    node.error(err,msg);
+                    node.status({fill:"red", shape:"ring", text:err.code});
                 }
+                msg.payload = err.toString() + " : " + url;
+                msg.statusCode = err.code || (err.response?err.response.statusCode:undefined);
+                nodeSend(msg);
+                nodeDone();
             });
         });
 
@@ -411,4 +453,69 @@ module.exports = function(RED) {
             password: {type: "password"}
         }
     });
+
+    const md5 = (value) => { return crypto.createHash('md5').update(value).digest('hex') }
+
+    function ha1Compute(algorithm, user, realm, pass, nonce, cnonce) {
+        /**
+        * RFC 2617: handle both MD5 and MD5-sess algorithms.
+        *
+        * If the algorithm directive's value is "MD5" or unspecified, then HA1 is
+        *   HA1=MD5(username:realm:password)
+        * If the algorithm directive's value is "MD5-sess", then HA1 is
+        *   HA1=MD5(MD5(username:realm:password):nonce:cnonce)
+        */
+        var ha1 = md5(user + ':' + realm + ':' + pass)
+        if (algorithm && algorithm.toLowerCase() === 'md5-sess') {
+            return md5(ha1 + ':' + nonce + ':' + cnonce)
+        } else {
+            return ha1
+        }
+    }
+
+
+    function buildDigestHeader(user, pass, method, path, authHeader) {
+        var challenge = {}
+        var re = /([a-z0-9_-]+)=(?:"([^"]+)"|([a-z0-9_-]+))/gi
+        for (;;) {
+            var match = re.exec(authHeader)
+            if (!match) {
+                break
+            }
+            challenge[match[1]] = match[2] || match[3]
+        }
+        var qop = /(^|,)\s*auth\s*($|,)/.test(challenge.qop) && 'auth'
+        var nc = qop && '00000001'
+        var cnonce = qop && uuid().replace(/-/g, '')
+        var ha1 = ha1Compute(challenge.algorithm, user, challenge.realm, pass, challenge.nonce, cnonce)
+        var ha2 = md5(method + ':' + path)
+        var digestResponse = qop
+        ? md5(ha1 + ':' + challenge.nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2)
+        : md5(ha1 + ':' + challenge.nonce + ':' + ha2)
+        var authValues = {
+            username: user,
+            realm: challenge.realm,
+            nonce: challenge.nonce,
+            uri: path,
+            qop: qop,
+            response: digestResponse,
+            nc: nc,
+            cnonce: cnonce,
+            algorithm: challenge.algorithm,
+            opaque: challenge.opaque
+        }
+
+        authHeader = []
+        for (var k in authValues) {
+            if (authValues[k]) {
+                if (k === 'qop' || k === 'nc' || k === 'algorithm') {
+                    authHeader.push(k + '=' + authValues[k])
+                } else {
+                    authHeader.push(k + '="' + authValues[k] + '"')
+                }
+            }
+        }
+        authHeader = 'Digest ' + authHeader.join(', ')
+        return authHeader
+    }
 }

packages/node_modules/@node-red/nodes/package.json

@@ -24,9 +24,12 @@
         "cors": "2.8.5",
         "cron": "1.7.2",
         "denque": "1.5.0",
+        "form-data": "4.0.0",
         "fs-extra": "9.1.0",
         "fs.notify": "0.0.4",
+        "got": "11.8.2",
         "hash-sum": "2.0.0",
+        "hpagent": "0.1.1",
         "https-proxy-agent": "5.0.0",
         "is-utf8": "0.2.1",
         "js-yaml": "3.14.0",
@@ -37,6 +40,8 @@
         "on-headers": "1.0.2",
         "raw-body": "2.4.1",
         "request": "2.88.0",
+        "tough-cookie": "4.0.0",
+        "uuid":"8.3.2",
         "ws": "6.2.1",
         "xml2js": "0.4.23",
         "iconv-lite": "0.6.2"

test/nodes/core/network/21-httprequest_spec.js

@@ -138,8 +138,6 @@ describe('HTTP Request Node', function() {
         });
         testApp.use(fileUploadApp);
 
-
-
         testApp.use(bodyParser.raw({type:"*/*"}));
         testApp.use(cookieParser(undefined,{decode:String}));
         testApp.get('/statusCode204', function(req,res) { res.status(204).end();});
@@ -166,11 +164,16 @@ describe('HTTP Request Node', function() {
             res.send("");
         });
         testApp.get('/authenticate', function(req, res){
-            var user = auth.parse(req.headers['authorization']);
+            let result;
-            var result = {
+            let authHeader = req.headers['authorization'];
-                user: user.name,
+            if (/^Basic/.test(authHeader)) {
-                pass: user.pass,
+                result = auth.parse(authHeader);
-            };
+                result.user = result.name;
+            } else if (/^Bearer/.test(authHeader)) {
+                result = {
+                    token: authHeader.substring(7)
+                }
+            }
             res.json(result);
         });
         testApp.get('/proxyAuthenticate', function(req, res){
@@ -891,7 +894,8 @@ describe('HTTP Request Node', function() {
                 var n2 = helper.getNode("n2");
                 n2.on("input", function(msg) {
                     try {
-                        msg.should.have.property('statusCode','ESOCKETTIMEDOUT');
+                        msg.should.have.property('statusCode');
+                        /TIMEDOUT/.test(msg.statusCode).should.be.true();
                         var logEvents = helper.log().args.filter(function(evt) {
                             return evt[0].type == 'http request';
                         });
@@ -917,7 +921,8 @@ describe('HTTP Request Node', function() {
                 var n2 = helper.getNode("n2");
                 n2.on("input", function(msg) {
                     try {
-                        msg.should.have.property('statusCode','ESOCKETTIMEDOUT');
+                        msg.should.have.property('statusCode');
+                        /TIMEDOUT/.test(msg.statusCode).should.be.true();
                         var logEvents = helper.log().args.filter(function(evt) {
                             return evt[0].type == 'http request';
                         });
@@ -1380,7 +1385,6 @@ describe('HTTP Request Node', function() {
                 n1.receive({payload:"foo"});
             });
         });
-
         it('should use http_proxy', function(done) {
             var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"POST",ret:"obj",url:getTestURL('/postInspect')},
                 {id:"n2", type:"helper"}];
@@ -1565,9 +1569,8 @@ describe('HTTP Request Node', function() {
             });
         });
     });
-
     describe('authentication', function() {
-        it('should authenticate on server', function(done) {
+        it('should authenticate on server - basic', function(done) {
             var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/authenticate')},
                 {id:"n2", type:"helper"}];
             helper.load(httpRequestNode, flow, function() {
@@ -1587,7 +1590,25 @@ describe('HTTP Request Node', function() {
                 n1.receive({payload:"foo"});
             });
         });
-
+        it('should authenticate on server - bearer', function(done) {
+            var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",authType:"bearer", url:getTestURL('/authenticate')},
+                {id:"n2", type:"helper"}];
+            helper.load(httpRequestNode, flow, function() {
+                var n1 = helper.getNode("n1");
+                var n2 = helper.getNode("n2");
+                n1.credentials = {password:'passwordfoo'};
+                n2.on("input", function(msg) {
+                    try {
+                        msg.should.have.property('statusCode',200);
+                        msg.payload.should.have.property('token', 'passwordfoo');
+                        done();
+                    } catch(err) {
+                        done(err);
+                    }
+                });
+                n1.receive({payload:"foo"});
+            });
+        });
         it('should authenticate on proxy server', function(done) {
             var flow = [{id:"n1",type:"http request", wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/proxyAuthenticate')},
                 {id:"n2", type:"helper"}];
@@ -1740,6 +1761,7 @@ describe('HTTP Request Node', function() {
                 var n1 = helper.getNode("n1");
                 var n2 = helper.getNode("n2");
                 n2.on("input", function(msg) {
+                    try {
                         var cookies1 = receivedCookies['localhost:'+testPort+'/redirectToSameDomain'];
                         var cookies2 = receivedCookies['localhost:'+testPort+'/redirectReturn'];
                         if (cookies1 && Object.keys(cookies1).length != 0) {
@@ -1756,6 +1778,7 @@ describe('HTTP Request Node', function() {
                         redirect1.cookies.redirectToSameDomainCookie.Path.should.equal('/');
                         redirect1.cookies.redirectToSameDomainCookie.value.should.equal('same1');
                         done();
+                    } catch(err) { done(err)}
                 });
                 n1.receive({});
             });