diff --git a/package.json b/package.json index 9df66ddf4..409a91c5b 100644 --- a/package.json +++ b/package.json @@ -41,9 +41,12 @@ "denque": "1.5.0", "express": "4.17.1", "express-session": "1.17.1", + "form-data": "4.0.0", "fs-extra": "9.1.0", "fs.notify": "0.0.4", + "got": "11.8.2", "hash-sum": "2.0.0", + "hpagent": "0.1.1", "https-proxy-agent": "5.0.0", "i18next": "20.2.1", "iconv-lite": "0.6.2", @@ -70,7 +73,9 @@ "request": "2.88.0", "semver": "7.3.5", "tar": "6.1.0", + "tough-cookie": "4.0.0", "uglify-js": "3.13.3", + "uuid": "8.3.2", "ws": "6.2.1", "xml2js": "0.4.23" }, diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js b/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js index afb17e3da..5f1be6be9 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js @@ -16,7 +16,13 @@ module.exports = function(RED) { "use strict"; - var request = require("request"); + const got = require("got"); + const {CookieJar} = require("tough-cookie"); + const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent'); + const FormData = require('form-data'); + const { v4: uuid } = require('uuid'); + const crypto = require('crypto'); + const URL = require("url").URL var mustache = require("mustache"); var querystring = require("querystring"); var cookie = require("cookie"); @@ -30,6 +36,8 @@ module.exports = function(RED) { var nodeMethod = n.method || "GET"; var paytoqs = false; var paytobody = false; + var redirectList = []; + var nodeHTTPPersistent = n["persist"]; if (n.tls) { var tlsNode = RED.nodes.getNode(n.tls); @@ -63,6 +71,7 @@ module.exports = function(RED) { if (msg.url && nodeUrl && (nodeUrl !== msg.url)) { // revert change below when warning is finally removed node.warn(RED._("common.errors.nooverride")); } + if (isTemplatedUrl) { url = mustache.render(nodeUrl,msg); } @@ -71,6 +80,8 @@ module.exports = function(RED) { nodeDone(); return; } + + // url must start http:// or https:// so assume http:// if not set if (url.indexOf("://") !== -1 && url.indexOf("http") !== 0) { node.warn(RED._("httpin.errors.invalid-transport")); @@ -86,6 +97,7 @@ module.exports = function(RED) { } } + var method = nodeMethod.toUpperCase() || "GET"; if (msg.method && n.method && (n.method !== "use")) { // warn if override option not set node.warn(RED._("common.errors.nooverride")); @@ -97,16 +109,15 @@ module.exports = function(RED) { var isHttps = (/^https/i.test(url)); var opts = {}; - opts.url = url; // set defaultport, else when using HttpsProxyAgent, it's defaultPort of 443 will be used :(. opts.defaultPort = isHttps?443:80; opts.timeout = node.reqTimeout; opts.method = method; opts.headers = {}; - opts.encoding = null; // Force NodeJs to return a Buffer (instead of a string) + opts.retry = 0; + opts.responseType = 'buffer'; opts.maxRedirects = 21; - opts.jar = request.jar(); - opts.proxy = null; + opts.cookieJar = new CookieJar(); opts.forever = nodeHTTPPersistent; if (msg.requestTimeout !== undefined) { if (isNaN(msg.requestTimeout)) { @@ -117,6 +128,21 @@ module.exports = function(RED) { opts.timeout = msg.requestTimeout; } } + + opts.hooks = { + beforeRedirect: [ + (options, response) => { + let redirectInfo = { + location: response.headers.location + } + if (response.headers.hasOwnProperty('set-cookie')) { + redirectInfo.cookies = extractCookies(response.headers['set-cookie']); + } + redirectList.push(redirectInfo) + } + ] + } + var ctSet = "Content-Type"; // set default camel case var clSet = "Content-Length"; if (msg.headers) { @@ -144,30 +170,15 @@ module.exports = function(RED) { } } } + if (msg.hasOwnProperty('followRedirects')) { - opts.followRedirect = msg.followRedirects; - opts.followAllRedirects = !!opts.followRedirect; - } - var redirectList = []; - if (!opts.hasOwnProperty('followRedirect') || opts.followRedirect) { - opts.followRedirect = function(res) { - var redirectInfo = { - location: res.headers.location, - }; - if (res.headers.hasOwnProperty('set-cookie')) { - redirectInfo.cookies = extractCookies(res.headers['set-cookie']); - } - redirectList.push(redirectInfo); - if (this.headers.cookie) { - delete this.headers.cookie; - } - return true; - }; + opts.followRedirect = !!msg.followRedirects; } + if (opts.headers.hasOwnProperty('cookie')) { var cookies = cookie.parse(opts.headers.cookie, {decode:String}); for (var name in cookies) { - opts.jar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url); + opts.cookieJar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url); } delete opts.headers.cookie; } @@ -180,13 +191,13 @@ module.exports = function(RED) { } else if (typeof msg.cookies[name] === 'object') { if(msg.cookies[name].encode === false){ // If the encode option is false, the value is not encoded. - opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url); + opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url); } else { // The value is encoded by encodeURIComponent(). - opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value), url); + opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value), url); } } else { - opts.jar.setCookie(cookie.serialize(name, msg.cookies[name]), url); + opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name]), url); } } } @@ -194,52 +205,62 @@ module.exports = function(RED) { if (this.credentials) { if (this.authType === "basic") { if (this.credentials.user) { - opts.auth = { - user: this.credentials.user, - pass: this.credentials.password || "" - }; + opts.username = this.credentials.user; + } + if (this.credentials.password) { + opts.password = this.credentials.password; } } else if (this.authType === "digest") { - if (this.credentials.user) { - // The first request will be sent without auth information. Based on the 401 response, the library can determine - // which auth type is required by the server. Then the request is resubmitted with the appropriate auth header. - opts.auth = { - user: this.credentials.user, - pass: this.credentials.password || "", - sendImmediately: false - }; - } + let digestCreds = this.credentials; + let sentCreds = false; + opts.hooks.afterResponse = [(response, retry) => { + if (response.statusCode === 401) { + if (sentCreds) { + return response + } + const requestUrl = new URL(response.request.requestUrl); + const options = response.request.options; + const normalisedHeaders = {}; + Object.keys(response.headers).forEach(k => { + normalisedHeaders[k.toLowerCase()] = response.headers[k] + }) + if (normalisedHeaders['www-authenticate']) { + let authHeader = buildDigestHeader(digestCreds.user,digestCreds.password, options.method, requestUrl.pathname, normalisedHeaders['www-authenticate']) + options.headers.authorization = authHeader; + } + sentCreds = true; + return retry(options); + } + return response + }]; } else if (this.authType === "bearer") { - opts.auth = { - bearer: this.credentials.password || "" - }; + opts.headers.Authorization = `Bearer ${this.credentials.password||""}` } } var payload = null; + if (method !== 'GET' && method !== 'HEAD' && typeof msg.payload !== "undefined") { if (opts.headers['content-type'] == 'multipart/form-data' && typeof msg.payload === "object") { - opts.formData = {}; - + let formData = new FormData(); for (var opt in msg.payload) { if (msg.payload.hasOwnProperty(opt)) { var val = msg.payload[opt]; if (val !== undefined && val !== null) { if (typeof val === 'string' || Buffer.isBuffer(val)) { - opts.formData[opt] = val; + formData.append(opt, val); } else if (typeof val === 'object' && val.hasOwnProperty('value')) { - // Treat as file to upload - ensure it has an options object - // as request complains if it doesn't - if (!val.hasOwnProperty('options')) { - val.options = {}; - } - opts.formData[opt] = val; + formData.append(opt,val.value,val.options || {}); } else { - opts.formData[opt] = JSON.stringify(val); + formData.append(opt,JSON.stringify(val)); } } } } + // GOT will only set the content-type header with the correct boundary + // if the header isn't set. So we delete it here, for GOT to reset it. + delete opts.headers['content-type']; + opts.body = formData; } else { if (typeof msg.payload === "string" || Buffer.isBuffer(msg.payload)) { payload = msg.payload; @@ -266,25 +287,29 @@ module.exports = function(RED) { } } + if (method == 'GET' && typeof msg.payload !== "undefined" && paytoqs) { if (typeof msg.payload === "object") { try { - if (opts.url.indexOf("?") !== -1) { - opts.url += (opts.url.endsWith("?")?"":"&") + querystring.stringify(msg.payload); + if (url.indexOf("?") !== -1) { + url += (url.endsWith("?")?"":"&") + querystring.stringify(msg.payload); } else { - opts.url += "?" + querystring.stringify(msg.payload); + url += "?" + querystring.stringify(msg.payload); } } catch(err) { + node.error(RED._("httpin.errors.invalid-payload"),msg); nodeDone(); return; } } else { + node.error(RED._("httpin.errors.invalid-payload"),msg); nodeDone(); return; } } else if ( method == "GET" && typeof msg.payload !== "undefined" && paytobody) { + opts.allowGetBody = true; if (typeof msg.payload === "object") { opts.body = JSON.stringify(msg.payload); } else if (typeof msg.payload == "number") { @@ -311,79 +336,96 @@ module.exports = function(RED) { } } if (prox && !noproxy) { - var match = prox.match(/^(http:\/\/)?(.+)?:([0-9]+)?/i); + var match = prox.match(/^(https?:\/\/)?(.+)?:([0-9]+)?/i); if (match) { - opts.proxy = prox; + let proxyAgent; + let proxyOptions = { + proxy: prox, + maxFreeSockets: 256, + maxSockets: 256, + keepAlive: true + } + if (proxyConfig && proxyConfig.credentials) { + let proxyUsername = proxyConfig.credentials.username || ''; + let proxyPassword = proxyConfig.credentials.password || ''; + if (proxyUsername || proxyPassword) { + var m = /^(https?:\/\/)(.*)$/.exec(prox); + proxyOptions.proxy = `${m[1]}${proxyUsername}:${proxyPassword}@${m[2]}` + } + } + opts.agent = {}; + if (/^http:/.test(url)) { + opts.agent.http = new HttpProxyAgent(proxyOptions) + } else { + opts.agent.https = new HttpsProxyAgent(proxyOptions) + } + console.log("ProxyOptions:",proxyOptions); } else { node.warn("Bad proxy url: "+ prox); - opts.proxy = null; - } - } - if (proxyConfig && proxyConfig.credentials && opts.proxy == proxyConfig.url) { - var proxyUsername = proxyConfig.credentials.username || ''; - var proxyPassword = proxyConfig.credentials.password || ''; - if (proxyUsername || proxyPassword) { - opts.headers['proxy-authorization'] = - 'Basic ' + - Buffer.from(proxyUsername + ':' + proxyPassword).toString('base64'); } } if (tlsNode) { - tlsNode.addTLSOptions(opts); + opts.https = {}; + tlsNode.addTLSOptions(opts.https); } else { if (msg.hasOwnProperty('rejectUnauthorized')) { - opts.rejectUnauthorized = msg.rejectUnauthorized; + opts.https = { rejectUnauthorized: msg.rejectUnauthorized }; } } - request(opts, function(err, res, body) { - if(err){ - if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { - node.error(RED._("common.notification.errors.no-response"), msg); - node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"}); - }else{ - node.error(err,msg); - node.status({fill:"red", shape:"ring", text:err.code}); - } - msg.payload = err.toString() + " : " + url; - msg.statusCode = err.code; - nodeSend(msg); - nodeDone(); - }else{ - msg.statusCode = res.statusCode; - msg.headers = res.headers; - msg.responseUrl = res.request.uri.href; - msg.payload = body; - msg.redirectList = redirectList; - - if (msg.headers.hasOwnProperty('set-cookie')) { - msg.responseCookies = extractCookies(msg.headers['set-cookie']); - } - msg.headers['x-node-red-request-node'] = hashSum(msg.headers); - // msg.url = url; // revert when warning above finally removed - if (node.metric()) { - // Calculate request time - var diff = process.hrtime(preRequestTimestamp); - var ms = diff[0] * 1e3 + diff[1] * 1e-6; - var metricRequestDurationMillis = ms.toFixed(3); - node.metric("duration.millis", msg, metricRequestDurationMillis); - if (res.client && res.client.bytesRead) { - node.metric("size.bytes", msg, res.client.bytesRead); - } - } - - // Convert the payload to the required return type - if (node.ret !== "bin") { - msg.payload = msg.payload.toString('utf8'); // txt - - if (node.ret === "obj") { - try { msg.payload = JSON.parse(msg.payload); } // obj - catch(e) { node.warn(RED._("httpin.errors.json-error")); } - } - } - node.status({}); - nodeSend(msg); - nodeDone(); + got(url,opts).then(res => { + msg.statusCode = res.statusCode; + msg.headers = res.headers; + let hostname = res.req.host; + if (res.req.protocol === "http:" && res.socket.remotePort !== 80) { + hostname = `${hostname}:${res.socket.remotePort}` + } else if (res.req.protocol === "https:" && res.socket.remotePort !== 443) { + hostname = `${hostname}:${res.socket.remotePort}` } + msg.responseUrl = `${res.req.protocol}//${hostname}${res.req.path}`; + msg.payload = res.body; + msg.redirectList = redirectList; + msg.retry = 0; + + if (msg.headers.hasOwnProperty('set-cookie')) { + msg.responseCookies = extractCookies(msg.headers['set-cookie']); + } + msg.headers['x-node-red-request-node'] = hashSum(msg.headers); + // msg.url = url; // revert when warning above finally removed + if (node.metric()) { + // Calculate request time + var diff = process.hrtime(preRequestTimestamp); + var ms = diff[0] * 1e3 + diff[1] * 1e-6; + var metricRequestDurationMillis = ms.toFixed(3); + node.metric("duration.millis", msg, metricRequestDurationMillis); + if (res.client && res.client.bytesRead) { + node.metric("size.bytes", msg, res.client.bytesRead); + } + } + + // Convert the payload to the required return type + if (node.ret !== "bin") { + msg.payload = msg.payload.toString('utf8'); // txt + + if (node.ret === "obj") { + try { msg.payload = JSON.parse(msg.payload); } // obj + catch(e) { node.warn(RED._("httpin.errors.json-error")); } + } + } + node.status({}); + nodeSend(msg); + nodeDone(); + }).catch(err => { + if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { + node.error(RED._("common.notification.errors.no-response"), msg); + node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"}); + }else{ + node.error(err,msg); + node.status({fill:"red", shape:"ring", text:err.code}); + } + msg.payload = err.toString() + " : " + url; + msg.statusCode = err.code || (err.response?err.response.statusCode:undefined); + nodeSend(msg); + nodeDone(); }); }); @@ -411,4 +453,69 @@ module.exports = function(RED) { password: {type: "password"} } }); + + const md5 = (value) => { return crypto.createHash('md5').update(value).digest('hex') } + + function ha1Compute(algorithm, user, realm, pass, nonce, cnonce) { + /** + * RFC 2617: handle both MD5 and MD5-sess algorithms. + * + * If the algorithm directive's value is "MD5" or unspecified, then HA1 is + * HA1=MD5(username:realm:password) + * If the algorithm directive's value is "MD5-sess", then HA1 is + * HA1=MD5(MD5(username:realm:password):nonce:cnonce) + */ + var ha1 = md5(user + ':' + realm + ':' + pass) + if (algorithm && algorithm.toLowerCase() === 'md5-sess') { + return md5(ha1 + ':' + nonce + ':' + cnonce) + } else { + return ha1 + } + } + + + function buildDigestHeader(user, pass, method, path, authHeader) { + var challenge = {} + var re = /([a-z0-9_-]+)=(?:"([^"]+)"|([a-z0-9_-]+))/gi + for (;;) { + var match = re.exec(authHeader) + if (!match) { + break + } + challenge[match[1]] = match[2] || match[3] + } + var qop = /(^|,)\s*auth\s*($|,)/.test(challenge.qop) && 'auth' + var nc = qop && '00000001' + var cnonce = qop && uuid().replace(/-/g, '') + var ha1 = ha1Compute(challenge.algorithm, user, challenge.realm, pass, challenge.nonce, cnonce) + var ha2 = md5(method + ':' + path) + var digestResponse = qop + ? md5(ha1 + ':' + challenge.nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2) + : md5(ha1 + ':' + challenge.nonce + ':' + ha2) + var authValues = { + username: user, + realm: challenge.realm, + nonce: challenge.nonce, + uri: path, + qop: qop, + response: digestResponse, + nc: nc, + cnonce: cnonce, + algorithm: challenge.algorithm, + opaque: challenge.opaque + } + + authHeader = [] + for (var k in authValues) { + if (authValues[k]) { + if (k === 'qop' || k === 'nc' || k === 'algorithm') { + authHeader.push(k + '=' + authValues[k]) + } else { + authHeader.push(k + '="' + authValues[k] + '"') + } + } + } + authHeader = 'Digest ' + authHeader.join(', ') + return authHeader + } } diff --git a/packages/node_modules/@node-red/nodes/package.json b/packages/node_modules/@node-red/nodes/package.json index a1a8fc5ce..4fa384b80 100644 --- a/packages/node_modules/@node-red/nodes/package.json +++ b/packages/node_modules/@node-red/nodes/package.json @@ -24,9 +24,12 @@ "cors": "2.8.5", "cron": "1.7.2", "denque": "1.5.0", + "form-data": "4.0.0", "fs-extra": "9.1.0", "fs.notify": "0.0.4", + "got": "11.8.2", "hash-sum": "2.0.0", + "hpagent": "0.1.1", "https-proxy-agent": "5.0.0", "is-utf8": "0.2.1", "js-yaml": "3.14.0", @@ -37,6 +40,8 @@ "on-headers": "1.0.2", "raw-body": "2.4.1", "request": "2.88.0", + "tough-cookie": "4.0.0", + "uuid":"8.3.2", "ws": "6.2.1", "xml2js": "0.4.23", "iconv-lite": "0.6.2" diff --git a/test/nodes/core/network/21-httprequest_spec.js b/test/nodes/core/network/21-httprequest_spec.js index e9792dd54..cbb0c9ad5 100644 --- a/test/nodes/core/network/21-httprequest_spec.js +++ b/test/nodes/core/network/21-httprequest_spec.js @@ -138,8 +138,6 @@ describe('HTTP Request Node', function() { }); testApp.use(fileUploadApp); - - testApp.use(bodyParser.raw({type:"*/*"})); testApp.use(cookieParser(undefined,{decode:String})); testApp.get('/statusCode204', function(req,res) { res.status(204).end();}); @@ -166,11 +164,16 @@ describe('HTTP Request Node', function() { res.send(""); }); testApp.get('/authenticate', function(req, res){ - var user = auth.parse(req.headers['authorization']); - var result = { - user: user.name, - pass: user.pass, - }; + let result; + let authHeader = req.headers['authorization']; + if (/^Basic/.test(authHeader)) { + result = auth.parse(authHeader); + result.user = result.name; + } else if (/^Bearer/.test(authHeader)) { + result = { + token: authHeader.substring(7) + } + } res.json(result); }); testApp.get('/proxyAuthenticate', function(req, res){ @@ -891,7 +894,8 @@ describe('HTTP Request Node', function() { var n2 = helper.getNode("n2"); n2.on("input", function(msg) { try { - msg.should.have.property('statusCode','ESOCKETTIMEDOUT'); + msg.should.have.property('statusCode'); + /TIMEDOUT/.test(msg.statusCode).should.be.true(); var logEvents = helper.log().args.filter(function(evt) { return evt[0].type == 'http request'; }); @@ -917,7 +921,8 @@ describe('HTTP Request Node', function() { var n2 = helper.getNode("n2"); n2.on("input", function(msg) { try { - msg.should.have.property('statusCode','ESOCKETTIMEDOUT'); + msg.should.have.property('statusCode'); + /TIMEDOUT/.test(msg.statusCode).should.be.true(); var logEvents = helper.log().args.filter(function(evt) { return evt[0].type == 'http request'; }); @@ -1380,7 +1385,6 @@ describe('HTTP Request Node', function() { n1.receive({payload:"foo"}); }); }); - it('should use http_proxy', function(done) { var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"POST",ret:"obj",url:getTestURL('/postInspect')}, {id:"n2", type:"helper"}]; @@ -1565,9 +1569,8 @@ describe('HTTP Request Node', function() { }); }); }); - describe('authentication', function() { - it('should authenticate on server', function(done) { + it('should authenticate on server - basic', function(done) { var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/authenticate')}, {id:"n2", type:"helper"}]; helper.load(httpRequestNode, flow, function() { @@ -1587,7 +1590,25 @@ describe('HTTP Request Node', function() { n1.receive({payload:"foo"}); }); }); - + it('should authenticate on server - bearer', function(done) { + var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",authType:"bearer", url:getTestURL('/authenticate')}, + {id:"n2", type:"helper"}]; + helper.load(httpRequestNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n1.credentials = {password:'passwordfoo'}; + n2.on("input", function(msg) { + try { + msg.should.have.property('statusCode',200); + msg.payload.should.have.property('token', 'passwordfoo'); + done(); + } catch(err) { + done(err); + } + }); + n1.receive({payload:"foo"}); + }); + }); it('should authenticate on proxy server', function(done) { var flow = [{id:"n1",type:"http request", wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/proxyAuthenticate')}, {id:"n2", type:"helper"}]; @@ -1740,22 +1761,24 @@ describe('HTTP Request Node', function() { var n1 = helper.getNode("n1"); var n2 = helper.getNode("n2"); n2.on("input", function(msg) { - var cookies1 = receivedCookies['localhost:'+testPort+'/redirectToSameDomain']; - var cookies2 = receivedCookies['localhost:'+testPort+'/redirectReturn']; - if (cookies1 && Object.keys(cookies1).length != 0) { - done(new Error('Invalid cookie(path:/rediectToSame)')); - return; - } - if ((cookies2 && Object.keys(cookies2).length != 1) || + try { + var cookies1 = receivedCookies['localhost:'+testPort+'/redirectToSameDomain']; + var cookies2 = receivedCookies['localhost:'+testPort+'/redirectReturn']; + if (cookies1 && Object.keys(cookies1).length != 0) { + done(new Error('Invalid cookie(path:/rediectToSame)')); + return; + } + if ((cookies2 && Object.keys(cookies2).length != 1) || cookies2['redirectToSameDomainCookie'] !== 'same1') { - done(new Error('Invalid cookie(path:/rediectReurn)')); - return; - } - var redirect1 = msg.redirectList[0]; - redirect1.location.should.equal('http://localhost:'+testPort+'/redirectReturn'); - redirect1.cookies.redirectToSameDomainCookie.Path.should.equal('/'); - redirect1.cookies.redirectToSameDomainCookie.value.should.equal('same1'); - done(); + done(new Error('Invalid cookie(path:/rediectReurn)')); + return; + } + var redirect1 = msg.redirectList[0]; + redirect1.location.should.equal('http://localhost:'+testPort+'/redirectReturn'); + redirect1.cookies.redirectToSameDomainCookie.Path.should.equal('/'); + redirect1.cookies.redirectToSameDomainCookie.value.should.equal('same1'); + done(); + } catch(err) { done(err)} }); n1.receive({}); });