frodovdr
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Initial migration from request to got for http-request node

This commit is contained in:
Nick O'Leary 2021-04-23 14:19:15 +01:00
parent 719aea2a58
commit e23f20227a
No known key found for this signature in database
GPG Key ID: 4F2157149161A6C9
4 changed files with 286 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
package.json
View File

@ -41,9 +41,12 @@
"denque": "1.5.0",
"express": "4.17.1",
"express-session": "1.17.1",
"form-data": "4.0.0",
"fs-extra": "9.1.0",
"fs.notify": "0.0.4",
"got": "11.8.2",
"hash-sum": "2.0.0",
"hpagent": "0.1.1",
"https-proxy-agent": "5.0.0",
"i18next": "20.2.1",
"iconv-lite": "0.6.2",
@ -70,7 +73,9 @@
"request": "2.88.0",
"semver": "7.3.5",
"tar": "6.1.0",
"tough-cookie": "4.0.0",
"uglify-js": "3.13.3",
"uuid": "8.3.2",
"ws": "6.2.1",
"xml2js": "0.4.23"
},

343
packages/node_modules/@node-red/nodes/core/network/21-httprequest.js vendored
View File

@ -16,7 +16,13 @@
module.exports = function(RED) {
"use strict";
var request = require("request");
const got = require("got");
const {CookieJar} = require("tough-cookie");
const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent');
const FormData = require('form-data');
const { v4: uuid } = require('uuid');
const crypto = require('crypto');
const URL = require("url").URL
var mustache = require("mustache");
var querystring = require("querystring");
var cookie = require("cookie");
@ -30,6 +36,8 @@ module.exports = function(RED) {
var nodeMethod = n.method || "GET";
var paytoqs = false;
var paytobody = false;
var redirectList = [];
var nodeHTTPPersistent = n["persist"];
if (n.tls) {
var tlsNode = RED.nodes.getNode(n.tls);
@ -63,6 +71,7 @@ module.exports = function(RED) {
if (msg.url && nodeUrl && (nodeUrl !== msg.url)) { // revert change below when warning is finally removed
node.warn(RED._("common.errors.nooverride"));
}
if (isTemplatedUrl) {
url = mustache.render(nodeUrl,msg);
}
@ -71,6 +80,8 @@ module.exports = function(RED) {
nodeDone();
return;
}
// url must start http:// or https:// so assume http:// if not set
if (url.indexOf("://") !== -1 && url.indexOf("http") !== 0) {
node.warn(RED._("httpin.errors.invalid-transport"));
@ -86,6 +97,7 @@ module.exports = function(RED) {
}
}
var method = nodeMethod.toUpperCase() || "GET";
if (msg.method && n.method && (n.method !== "use")) { // warn if override option not set
node.warn(RED._("common.errors.nooverride"));
@ -97,16 +109,15 @@ module.exports = function(RED) {
var isHttps = (/^https/i.test(url));
var opts = {};
opts.url = url;
// set defaultport, else when using HttpsProxyAgent, it's defaultPort of 443 will be used :(.
opts.defaultPort = isHttps?443:80;
opts.timeout = node.reqTimeout;
opts.method = method;
opts.headers = {};
opts.encoding = null; // Force NodeJs to return a Buffer (instead of a string)
opts.retry = 0;
opts.responseType = 'buffer';
opts.maxRedirects = 21;
opts.jar = request.jar();
opts.proxy = null;
opts.cookieJar = new CookieJar();
opts.forever = nodeHTTPPersistent;
if (msg.requestTimeout !== undefined) {
if (isNaN(msg.requestTimeout)) {
@ -117,6 +128,21 @@ module.exports = function(RED) {
opts.timeout = msg.requestTimeout;
}
}
opts.hooks = {
beforeRedirect: [
(options, response) => {
let redirectInfo = {
location: response.headers.location
}
if (response.headers.hasOwnProperty('set-cookie')) {
redirectInfo.cookies = extractCookies(response.headers['set-cookie']);
}
redirectList.push(redirectInfo)
}
]
}
var ctSet = "Content-Type"; // set default camel case
var clSet = "Content-Length";
if (msg.headers) {
@ -144,30 +170,15 @@ module.exports = function(RED) {
}
}
}
if (msg.hasOwnProperty('followRedirects')) {
opts.followRedirect = msg.followRedirects;
opts.followAllRedirects = !!opts.followRedirect;
}
var redirectList = [];
if (!opts.hasOwnProperty('followRedirect') || opts.followRedirect) {
opts.followRedirect = function(res) {
var redirectInfo = {
location: res.headers.location,
};
if (res.headers.hasOwnProperty('set-cookie')) {
redirectInfo.cookies = extractCookies(res.headers['set-cookie']);
}
redirectList.push(redirectInfo);
if (this.headers.cookie) {
delete this.headers.cookie;
}
return true;
};
opts.followRedirect = !!msg.followRedirects;
}
if (opts.headers.hasOwnProperty('cookie')) {
var cookies = cookie.parse(opts.headers.cookie, {decode:String});
for (var name in cookies) {
opts.jar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url);
opts.cookieJar.setCookie(cookie.serialize(name, cookies[name], {encode:String}), url);
}
delete opts.headers.cookie;
}
@ -180,13 +191,13 @@ module.exports = function(RED) {
} else if (typeof msg.cookies[name] === 'object') {
if(msg.cookies[name].encode === false){
// If the encode option is false, the value is not encoded.
opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url);
opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value, {encode: String}), url);
} else {
// The value is encoded by encodeURIComponent().
opts.jar.setCookie(cookie.serialize(name, msg.cookies[name].value), url);
opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name].value), url);
}
} else {
opts.jar.setCookie(cookie.serialize(name, msg.cookies[name]), url);
opts.cookieJar.setCookie(cookie.serialize(name, msg.cookies[name]), url);
}
}
}
@ -194,52 +205,62 @@ module.exports = function(RED) {
if (this.credentials) {
if (this.authType === "basic") {
if (this.credentials.user) {
opts.auth = {
user: this.credentials.user,
pass: this.credentials.password || ""
};
opts.username = this.credentials.user;
}
if (this.credentials.password) {
opts.password = this.credentials.password;
}
} else if (this.authType === "digest") {
if (this.credentials.user) {
// The first request will be sent without auth information. Based on the 401 response, the library can determine
// which auth type is required by the server. Then the request is resubmitted with the appropriate auth header.
opts.auth = {
user: this.credentials.user,
pass: this.credentials.password || "",
sendImmediately: false
};
}
let digestCreds = this.credentials;
let sentCreds = false;
opts.hooks.afterResponse = [(response, retry) => {
if (response.statusCode === 401) {
if (sentCreds) {
return response
}
const requestUrl = new URL(response.request.requestUrl);
const options = response.request.options;
const normalisedHeaders = {};
Object.keys(response.headers).forEach(k => {
normalisedHeaders[k.toLowerCase()] = response.headers[k]
})
if (normalisedHeaders['www-authenticate']) {
let authHeader = buildDigestHeader(digestCreds.user,digestCreds.password, options.method, requestUrl.pathname, normalisedHeaders['www-authenticate'])
options.headers.authorization = authHeader;
}
sentCreds = true;
return retry(options);
}
return response
}];
} else if (this.authType === "bearer") {
opts.auth = {
bearer: this.credentials.password || ""
};
opts.headers.Authorization = `Bearer ${this.credentials.password||""}`
}
}
var payload = null;
if (method !== 'GET' && method !== 'HEAD' && typeof msg.payload !== "undefined") {
if (opts.headers['content-type'] == 'multipart/form-data' && typeof msg.payload === "object") {
opts.formData = {};
let formData = new FormData();
for (var opt in msg.payload) {
if (msg.payload.hasOwnProperty(opt)) {
var val = msg.payload[opt];
if (val !== undefined && val !== null) {
if (typeof val === 'string' || Buffer.isBuffer(val)) {
opts.formData[opt] = val;
formData.append(opt, val);
} else if (typeof val === 'object' && val.hasOwnProperty('value')) {
// Treat as file to upload - ensure it has an options object
// as request complains if it doesn't
if (!val.hasOwnProperty('options')) {
val.options = {};
}
opts.formData[opt] = val;
formData.append(opt,val.value,val.options || {});
} else {
opts.formData[opt] = JSON.stringify(val);
formData.append(opt,JSON.stringify(val));
}
}
}
}
// GOT will only set the content-type header with the correct boundary
// if the header isn't set. So we delete it here, for GOT to reset it.
delete opts.headers['content-type'];
opts.body = formData;
} else {
if (typeof msg.payload === "string" || Buffer.isBuffer(msg.payload)) {
payload = msg.payload;
@ -266,25 +287,29 @@ module.exports = function(RED) {
}
}
if (method == 'GET' && typeof msg.payload !== "undefined" && paytoqs) {
if (typeof msg.payload === "object") {
try {
if (opts.url.indexOf("?") !== -1) {
opts.url += (opts.url.endsWith("?")?"":"&") + querystring.stringify(msg.payload);
if (url.indexOf("?") !== -1) {
url += (url.endsWith("?")?"":"&") + querystring.stringify(msg.payload);
} else {
opts.url += "?" + querystring.stringify(msg.payload);
url += "?" + querystring.stringify(msg.payload);
}
} catch(err) {
node.error(RED._("httpin.errors.invalid-payload"),msg);
nodeDone();
return;
}
} else {
node.error(RED._("httpin.errors.invalid-payload"),msg);
nodeDone();
return;
}
} else if ( method == "GET" && typeof msg.payload !== "undefined" && paytobody) {
opts.allowGetBody = true;
if (typeof msg.payload === "object") {
opts.body = JSON.stringify(msg.payload);
} else if (typeof msg.payload == "number") {
@ -311,79 +336,96 @@ module.exports = function(RED) {
}
}
if (prox && !noproxy) {
var match = prox.match(/^(http:\/\/)?(.+)?:([0-9]+)?/i);
var match = prox.match(/^(https?:\/\/)?(.+)?:([0-9]+)?/i);
if (match) {
opts.proxy = prox;
let proxyAgent;
let proxyOptions = {
proxy: prox,
maxFreeSockets: 256,
maxSockets: 256,
keepAlive: true
}
if (proxyConfig && proxyConfig.credentials) {
let proxyUsername = proxyConfig.credentials.username || '';
let proxyPassword = proxyConfig.credentials.password || '';
if (proxyUsername || proxyPassword) {
var m = /^(https?:\/\/)(.*)$/.exec(prox);
proxyOptions.proxy = `${m[1]}${proxyUsername}:${proxyPassword}@${m[2]}`
}
}
opts.agent = {};
if (/^http:/.test(url)) {
opts.agent.http = new HttpProxyAgent(proxyOptions)
} else {
opts.agent.https = new HttpsProxyAgent(proxyOptions)
}
console.log("ProxyOptions:",proxyOptions);
} else {
node.warn("Bad proxy url: "+ prox);
opts.proxy = null;
}
}
if (proxyConfig && proxyConfig.credentials && opts.proxy == proxyConfig.url) {
var proxyUsername = proxyConfig.credentials.username || '';
var proxyPassword = proxyConfig.credentials.password || '';
if (proxyUsername || proxyPassword) {
opts.headers['proxy-authorization'] =
'Basic ' +
Buffer.from(proxyUsername + ':' + proxyPassword).toString('base64');
}
}
if (tlsNode) {
tlsNode.addTLSOptions(opts);
opts.https = {};
tlsNode.addTLSOptions(opts.https);
} else {
if (msg.hasOwnProperty('rejectUnauthorized')) {
opts.rejectUnauthorized = msg.rejectUnauthorized;
opts.https = { rejectUnauthorized: msg.rejectUnauthorized };
}
}
request(opts, function(err, res, body) {
if(err){
if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
node.error(RED._("common.notification.errors.no-response"), msg);
node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
}else{
node.error(err,msg);
node.status({fill:"red", shape:"ring", text:err.code});
}
msg.payload = err.toString() + " : " + url;
msg.statusCode = err.code;
nodeSend(msg);
nodeDone();
}else{
msg.statusCode = res.statusCode;
msg.headers = res.headers;
msg.responseUrl = res.request.uri.href;
msg.payload = body;
msg.redirectList = redirectList;
if (msg.headers.hasOwnProperty('set-cookie')) {
msg.responseCookies = extractCookies(msg.headers['set-cookie']);
}
msg.headers['x-node-red-request-node'] = hashSum(msg.headers);
// msg.url = url; // revert when warning above finally removed
if (node.metric()) {
// Calculate request time
var diff = process.hrtime(preRequestTimestamp);
var ms = diff[0] * 1e3 + diff[1] * 1e-6;
var metricRequestDurationMillis = ms.toFixed(3);
node.metric("duration.millis", msg, metricRequestDurationMillis);
if (res.client && res.client.bytesRead) {
node.metric("size.bytes", msg, res.client.bytesRead);
}
}
// Convert the payload to the required return type
if (node.ret !== "bin") {
msg.payload = msg.payload.toString('utf8'); // txt
if (node.ret === "obj") {
try { msg.payload = JSON.parse(msg.payload); } // obj
catch(e) { node.warn(RED._("httpin.errors.json-error")); }
}
}
node.status({});
nodeSend(msg);
nodeDone();
got(url,opts).then(res => {
msg.statusCode = res.statusCode;
msg.headers = res.headers;
let hostname = res.req.host;
if (res.req.protocol === "http:" && res.socket.remotePort !== 80) {
hostname = `${hostname}:${res.socket.remotePort}`
} else if (res.req.protocol === "https:" && res.socket.remotePort !== 443) {
hostname = `${hostname}:${res.socket.remotePort}`
}
msg.responseUrl = `${res.req.protocol}//${hostname}${res.req.path}`;
msg.payload = res.body;
msg.redirectList = redirectList;
msg.retry = 0;
if (msg.headers.hasOwnProperty('set-cookie')) {
msg.responseCookies = extractCookies(msg.headers['set-cookie']);
}
msg.headers['x-node-red-request-node'] = hashSum(msg.headers);
// msg.url = url; // revert when warning above finally removed
if (node.metric()) {
// Calculate request time
var diff = process.hrtime(preRequestTimestamp);
var ms = diff[0] * 1e3 + diff[1] * 1e-6;
var metricRequestDurationMillis = ms.toFixed(3);
node.metric("duration.millis", msg, metricRequestDurationMillis);
if (res.client && res.client.bytesRead) {
node.metric("size.bytes", msg, res.client.bytesRead);
}
}
// Convert the payload to the required return type
if (node.ret !== "bin") {
msg.payload = msg.payload.toString('utf8'); // txt
if (node.ret === "obj") {
try { msg.payload = JSON.parse(msg.payload); } // obj
catch(e) { node.warn(RED._("httpin.errors.json-error")); }
}
}
node.status({});
nodeSend(msg);
nodeDone();
}).catch(err => {
if(err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
node.error(RED._("common.notification.errors.no-response"), msg);
node.status({fill:"red", shape:"ring", text:"common.notification.errors.no-response"});
}else{
node.error(err,msg);
node.status({fill:"red", shape:"ring", text:err.code});
}
msg.payload = err.toString() + " : " + url;
msg.statusCode = err.code || (err.response?err.response.statusCode:undefined);
nodeSend(msg);
nodeDone();
});
});
@ -411,4 +453,69 @@ module.exports = function(RED) {
password: {type: "password"}
}
});
const md5 = (value) => { return crypto.createHash('md5').update(value).digest('hex') }
function ha1Compute(algorithm, user, realm, pass, nonce, cnonce) {
/**
* RFC 2617: handle both MD5 and MD5-sess algorithms.
*
* If the algorithm directive's value is "MD5" or unspecified, then HA1 is
* HA1=MD5(username:realm:password)
* If the algorithm directive's value is "MD5-sess", then HA1 is
* HA1=MD5(MD5(username:realm:password):nonce:cnonce)
*/
var ha1 = md5(user + ':' + realm + ':' + pass)
if (algorithm && algorithm.toLowerCase() === 'md5-sess') {
return md5(ha1 + ':' + nonce + ':' + cnonce)
} else {
return ha1
}
}
function buildDigestHeader(user, pass, method, path, authHeader) {
var challenge = {}
var re = /([a-z0-9_-]+)=(?:"([^"]+)"|([a-z0-9_-]+))/gi
for (;;) {
var match = re.exec(authHeader)
if (!match) {
break
}
challenge[match[1]] = match[2] || match[3]
}
var qop = /(^|,)\s*auth\s*($|,)/.test(challenge.qop) && 'auth'
var nc = qop && '00000001'
var cnonce = qop && uuid().replace(/-/g, '')
var ha1 = ha1Compute(challenge.algorithm, user, challenge.realm, pass, challenge.nonce, cnonce)
var ha2 = md5(method + ':' + path)
var digestResponse = qop
? md5(ha1 + ':' + challenge.nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2)
: md5(ha1 + ':' + challenge.nonce + ':' + ha2)
var authValues = {
username: user,
realm: challenge.realm,
nonce: challenge.nonce,
uri: path,
qop: qop,
response: digestResponse,
nc: nc,
cnonce: cnonce,
algorithm: challenge.algorithm,
opaque: challenge.opaque
}
authHeader = []
for (var k in authValues) {
if (authValues[k]) {
if (k === 'qop' || k === 'nc' || k === 'algorithm') {
authHeader.push(k + '=' + authValues[k])
} else {
authHeader.push(k + '="' + authValues[k] + '"')
}
}
}
authHeader = 'Digest ' + authHeader.join(', ')
return authHeader
}
}

5
packages/node_modules/@node-red/nodes/package.json vendored
View File

@ -24,9 +24,12 @@
"cors": "2.8.5",
"cron": "1.7.2",
"denque": "1.5.0",
"form-data": "4.0.0",
"fs-extra": "9.1.0",
"fs.notify": "0.0.4",
"got": "11.8.2",
"hash-sum": "2.0.0",
"hpagent": "0.1.1",
"https-proxy-agent": "5.0.0",
"is-utf8": "0.2.1",
"js-yaml": "3.14.0",
@ -37,6 +40,8 @@
"on-headers": "1.0.2",
"raw-body": "2.4.1",
"request": "2.88.0",
"tough-cookie": "4.0.0",
"uuid":"8.3.2",
"ws": "6.2.1",
"xml2js": "0.4.23",
"iconv-lite": "0.6.2"

79
test/nodes/core/network/21-httprequest_spec.js
View File

@ -138,8 +138,6 @@ describe('HTTP Request Node', function() {
});
testApp.use(fileUploadApp);
testApp.use(bodyParser.raw({type:"*/*"}));
testApp.use(cookieParser(undefined,{decode:String}));
testApp.get('/statusCode204', function(req,res) { res.status(204).end();});
@ -166,11 +164,16 @@ describe('HTTP Request Node', function() {
res.send("");
});
testApp.get('/authenticate', function(req, res){
var user = auth.parse(req.headers['authorization']);
var result = {
user: user.name,
pass: user.pass,
};
let result;
let authHeader = req.headers['authorization'];
if (/^Basic/.test(authHeader)) {
result = auth.parse(authHeader);
result.user = result.name;
} else if (/^Bearer/.test(authHeader)) {
result = {
token: authHeader.substring(7)
}
}
res.json(result);
});
testApp.get('/proxyAuthenticate', function(req, res){
@ -891,7 +894,8 @@ describe('HTTP Request Node', function() {
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
try {
msg.should.have.property('statusCode','ESOCKETTIMEDOUT');
msg.should.have.property('statusCode');
/TIMEDOUT/.test(msg.statusCode).should.be.true();
var logEvents = helper.log().args.filter(function(evt) {
return evt[0].type == 'http request';
});
@ -917,7 +921,8 @@ describe('HTTP Request Node', function() {
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
try {
msg.should.have.property('statusCode','ESOCKETTIMEDOUT');
msg.should.have.property('statusCode');
/TIMEDOUT/.test(msg.statusCode).should.be.true();
var logEvents = helper.log().args.filter(function(evt) {
return evt[0].type == 'http request';
});
@ -1380,7 +1385,6 @@ describe('HTTP Request Node', function() {
n1.receive({payload:"foo"});
});
});
it('should use http_proxy', function(done) {
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"POST",ret:"obj",url:getTestURL('/postInspect')},
{id:"n2", type:"helper"}];
@ -1565,9 +1569,8 @@ describe('HTTP Request Node', function() {
});
});
});
describe('authentication', function() {
it('should authenticate on server', function(done) {
it('should authenticate on server - basic', function(done) {
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/authenticate')},
{id:"n2", type:"helper"}];
helper.load(httpRequestNode, flow, function() {
@ -1587,7 +1590,25 @@ describe('HTTP Request Node', function() {
n1.receive({payload:"foo"});
});
});
it('should authenticate on server - bearer', function(done) {
var flow = [{id:"n1",type:"http request",wires:[["n2"]],method:"GET",ret:"obj",authType:"bearer", url:getTestURL('/authenticate')},
{id:"n2", type:"helper"}];
helper.load(httpRequestNode, flow, function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n1.credentials = {password:'passwordfoo'};
n2.on("input", function(msg) {
try {
msg.should.have.property('statusCode',200);
msg.payload.should.have.property('token', 'passwordfoo');
done();
} catch(err) {
done(err);
}
});
n1.receive({payload:"foo"});
});
});
it('should authenticate on proxy server', function(done) {
var flow = [{id:"n1",type:"http request", wires:[["n2"]],method:"GET",ret:"obj",url:getTestURL('/proxyAuthenticate')},
{id:"n2", type:"helper"}];
@ -1740,22 +1761,24 @@ describe('HTTP Request Node', function() {
var n1 = helper.getNode("n1");
var n2 = helper.getNode("n2");
n2.on("input", function(msg) {
var cookies1 = receivedCookies['localhost:'+testPort+'/redirectToSameDomain'];
var cookies2 = receivedCookies['localhost:'+testPort+'/redirectReturn'];
if (cookies1 && Object.keys(cookies1).length != 0) {
done(new Error('Invalid cookie(path:/rediectToSame)'));
return;
}
if ((cookies2 && Object.keys(cookies2).length != 1) ||
try {
var cookies1 = receivedCookies['localhost:'+testPort+'/redirectToSameDomain'];
var cookies2 = receivedCookies['localhost:'+testPort+'/redirectReturn'];
if (cookies1 && Object.keys(cookies1).length != 0) {
done(new Error('Invalid cookie(path:/rediectToSame)'));
return;
}
if ((cookies2 && Object.keys(cookies2).length != 1) ||
cookies2['redirectToSameDomainCookie'] !== 'same1') {
done(new Error('Invalid cookie(path:/rediectReurn)'));
return;
}
var redirect1 = msg.redirectList[0];
redirect1.location.should.equal('http://localhost:'+testPort+'/redirectReturn');
redirect1.cookies.redirectToSameDomainCookie.Path.should.equal('/');
redirect1.cookies.redirectToSameDomainCookie.value.should.equal('same1');
done();
done(new Error('Invalid cookie(path:/rediectReurn)'));
return;
}
var redirect1 = msg.redirectList[0];
redirect1.location.should.equal('http://localhost:'+testPort+'/redirectReturn');
redirect1.cookies.redirectToSameDomainCookie.Path.should.equal('/');
redirect1.cookies.redirectToSameDomainCookie.value.should.equal('same1');
done();
} catch(err) { done(err)}
});
n1.receive({});
});