From 20abe4a40cfd4d24b71e5bda69c433bf57155448 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Fri, 28 Apr 2023 21:37:03 +0100 Subject: [PATCH 1/3] Update dependecies include got --- package.json | 16 ++++++------- .../nodes/core/network/21-httprequest.js | 24 +++++++++++-------- .../node_modules/@node-red/nodes/package.json | 6 ++--- .../@node-red/registry/package.json | 4 ++-- .../@node-red/runtime/package.json | 2 +- .../node_modules/@node-red/util/package.json | 4 ++-- packages/node_modules/node-red/package.json | 4 ++-- .../nodes/core/network/21-httprequest_spec.js | 4 ++-- 8 files changed, 34 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index f4a87961c..a92965e54 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ "express": "4.18.2", "express-session": "1.17.3", "form-data": "4.0.0", - "fs-extra": "10.1.0", - "got": "11.8.6", + "fs-extra": "11.1.1", + "got": "12.6.0", "hash-sum": "2.0.0", "hpagent": "1.2.0", "https-proxy-agent": "5.0.1", @@ -60,7 +60,7 @@ "memorystore": "1.6.7", "mime": "3.0.0", "moment": "2.29.4", - "moment-timezone": "0.5.41", + "moment-timezone": "0.5.43", "mqtt": "4.3.7", "multer": "1.4.5-lts.1", "mustache": "4.2.0", @@ -73,13 +73,13 @@ "passport-http-bearer": "1.0.1", "passport-oauth2-client-password": "0.1.2", "raw-body": "2.5.2", - "semver": "7.3.8", + "semver": "7.5.0", "tar": "6.1.13", "tough-cookie": "4.1.2", "uglify-js": "3.17.4", "uuid": "9.0.0", "ws": "7.5.6", - "xml2js": "0.4.23" + "xml2js": "0.5.0" }, "optionalDependencies": { "bcrypt": "5.1.0" @@ -108,14 +108,14 @@ "i18next-http-backend": "1.4.1", "jquery-i18next": "1.2.1", "jsdoc-nr-template": "github:node-red/jsdoc-nr-template", - "marked": "4.2.12", - "mermaid": "^9.3.0", + "marked": "4.3.0", + "mermaid": "^9.4.3", "minami": "1.2.3", "mocha": "9.2.2", "node-red-node-test-helper": "^0.3.0", "nodemon": "2.0.20", "proxy": "^1.0.2", - "sass": "1.58.3", + "sass": "1.62.1", "should": "13.2.3", "sinon": "11.1.2", "stoppable": "^1.1.0", diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js b/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js index bcf3b72b9..4543c3655 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httprequest.js @@ -14,9 +14,9 @@ * limitations under the License. **/ -module.exports = function(RED) { +module.exports = async function(RED) { "use strict"; - const got = require("got"); + const { got } = await import('got') const {CookieJar} = require("tough-cookie"); const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent'); const FormData = require('form-data'); @@ -210,24 +210,24 @@ in your Node-RED user directory (${RED.settings.userDir}). // set defaultport, else when using HttpsProxyAgent, it's defaultPort of 443 will be used :(. // Had to remove this to get http->https redirect to work // opts.defaultPort = isHttps?443:80; - opts.timeout = node.reqTimeout; + opts.timeout = { request: node.reqTimeout || 5000 }; opts.throwHttpErrors = false; // TODO: add UI option to auto decompress. Setting to false for 1.x compatibility opts.decompress = false; opts.method = method; - opts.retry = 0; + opts.retry = { limit: 0 }; opts.responseType = 'buffer'; opts.maxRedirects = 21; opts.cookieJar = new CookieJar(); opts.ignoreInvalidCookies = true; - opts.forever = nodeHTTPPersistent; + // opts.forever = nodeHTTPPersistent; if (msg.requestTimeout !== undefined) { if (isNaN(msg.requestTimeout)) { node.warn(RED._("httpin.errors.timeout-isnan")); } else if (msg.requestTimeout < 1) { node.warn(RED._("httpin.errors.timeout-isnegative")); } else { - opts.timeout = msg.requestTimeout; + opts.timeout = { request: msg.requestTimeout }; } } const originalHeaderMap = {}; @@ -245,9 +245,12 @@ in your Node-RED user directory (${RED.settings.userDir}). delete options.headers[h]; } }) - if (node.insecureHTTPParser) { - options.insecureHTTPParser = true + // Setting the property under _unixOptions as pretty + // much the only hack available to get got to apply + // a core http option it doesn't think we should be + // allowed to set + options._unixOptions = { ...options.unixOptions, insecureHTTPParser: true } } } ], @@ -403,15 +406,16 @@ in your Node-RED user directory (${RED.settings.userDir}). return response } const requestUrl = new URL(response.request.requestUrl); - const options = response.request.options; + const options = { headers: {} } const normalisedHeaders = {}; Object.keys(response.headers).forEach(k => { normalisedHeaders[k.toLowerCase()] = response.headers[k] }) if (normalisedHeaders['www-authenticate']) { - let authHeader = buildDigestHeader(digestCreds.user,digestCreds.password, options.method, requestUrl.pathname, normalisedHeaders['www-authenticate']) + let authHeader = buildDigestHeader(digestCreds.user,digestCreds.password, response.request.options.method, requestUrl.pathname, normalisedHeaders['www-authenticate']) options.headers.Authorization = authHeader; } + // response.request.options.merge(options) sentCreds = true; return retry(options); } diff --git a/packages/node_modules/@node-red/nodes/package.json b/packages/node_modules/@node-red/nodes/package.json index 197223089..6bed85bd9 100644 --- a/packages/node_modules/@node-red/nodes/package.json +++ b/packages/node_modules/@node-red/nodes/package.json @@ -27,8 +27,8 @@ "cronosjs": "1.7.1", "denque": "2.1.0", "form-data": "4.0.0", - "fs-extra": "10.1.0", - "got": "11.8.6", + "fs-extra": "11.1.1", + "got": "12.6.0", "hash-sum": "2.0.0", "hpagent": "1.2.0", "https-proxy-agent": "5.0.1", @@ -44,7 +44,7 @@ "tough-cookie": "4.1.2", "uuid": "9.0.0", "ws": "7.5.6", - "xml2js": "0.4.23", + "xml2js": "0.5.0", "iconv-lite": "0.6.3" } } diff --git a/packages/node_modules/@node-red/registry/package.json b/packages/node_modules/@node-red/registry/package.json index 205f97631..fae0d624a 100644 --- a/packages/node_modules/@node-red/registry/package.json +++ b/packages/node_modules/@node-red/registry/package.json @@ -18,8 +18,8 @@ "dependencies": { "@node-red/util": "3.1.0-beta.2", "clone": "2.1.2", - "fs-extra": "10.1.0", - "semver": "7.3.8", + "fs-extra": "11.1.1", + "semver": "7.5.0", "tar": "6.1.13", "uglify-js": "3.17.4" } diff --git a/packages/node_modules/@node-red/runtime/package.json b/packages/node_modules/@node-red/runtime/package.json index e5fcf52a8..7504a7497 100644 --- a/packages/node_modules/@node-red/runtime/package.json +++ b/packages/node_modules/@node-red/runtime/package.json @@ -21,7 +21,7 @@ "async-mutex": "0.4.0", "clone": "2.1.2", "express": "4.18.2", - "fs-extra": "10.1.0", + "fs-extra": "11.1.1", "json-stringify-safe": "5.0.1" } } diff --git a/packages/node_modules/@node-red/util/package.json b/packages/node_modules/@node-red/util/package.json index 1ef2dcf08..184e96f38 100644 --- a/packages/node_modules/@node-red/util/package.json +++ b/packages/node_modules/@node-red/util/package.json @@ -15,12 +15,12 @@ } ], "dependencies": { - "fs-extra": "10.1.0", + "fs-extra": "11.1.1", "i18next": "21.10.0", "json-stringify-safe": "5.0.1", "jsonata": "1.8.6", "lodash.clonedeep": "^4.5.0", "moment": "2.29.4", - "moment-timezone": "0.5.41" + "moment-timezone": "0.5.43" } } diff --git a/packages/node_modules/node-red/package.json b/packages/node_modules/node-red/package.json index 3ea18092c..0b899f655 100644 --- a/packages/node_modules/node-red/package.json +++ b/packages/node_modules/node-red/package.json @@ -38,10 +38,10 @@ "basic-auth": "2.0.1", "bcryptjs": "2.4.3", "express": "4.18.2", - "fs-extra": "10.1.0", + "fs-extra": "11.1.1", "node-red-admin": "^3.0.0", "nopt": "5.0.0", - "semver": "7.3.8" + "semver": "7.5.0" }, "optionalDependencies": { "bcrypt": "5.1.0" diff --git a/test/nodes/core/network/21-httprequest_spec.js b/test/nodes/core/network/21-httprequest_spec.js index 37e282bcf..f02ba8153 100644 --- a/test/nodes/core/network/21-httprequest_spec.js +++ b/test/nodes/core/network/21-httprequest_spec.js @@ -223,7 +223,7 @@ describe('HTTP Request Node', function() { } authFields[match[1]] = match[2] || match[3]; } - console.log(JSON.stringify(authFields)); + // console.log(JSON.stringify(authFields)); if (qop && authFields['qop'] != qop) { console.log('test1'); @@ -250,7 +250,7 @@ describe('HTTP Request Node', function() { req, algorithm, sess, realm, username, nonce, nc, cnonce, qop ); if (!response || expectedResponse.toLowerCase() !== response.toLowerCase()) { - console.log('test3'); + console.log('test3', response, expectedResponse); res.status(401).end(); return; } From 46ae66c8b2ff2c070800e07a3826695cb91af611 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Fri, 28 Apr 2023 21:42:36 +0100 Subject: [PATCH 2/3] Bump test helper version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a92965e54..f53cad7c6 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,7 @@ "mermaid": "^9.4.3", "minami": "1.2.3", "mocha": "9.2.2", - "node-red-node-test-helper": "^0.3.0", + "node-red-node-test-helper": "^0.3.1", "nodemon": "2.0.20", "proxy": "^1.0.2", "sass": "1.62.1", From 6e1b298282d57cc882ae2475943618b1852c3ce9 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 22 May 2023 13:57:12 +0100 Subject: [PATCH 3/3] Reconstruct xml2js output as proper object --- .../@node-red/nodes/core/parsers/70-XML.js | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-XML.js b/packages/node_modules/@node-red/nodes/core/parsers/70-XML.js index a778c4d72..538368730 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-XML.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-XML.js @@ -33,7 +33,13 @@ module.exports = function(RED) { parseString(value, options, function (err, result) { if (err) { done(err); } else { - value = result; + // TODO: With xml2js@0.5.0, they return an object with + // a null prototype. This could cause unexpected + // issues. So for now, we have to reconstruct + // the object with a proper prototype. + // Once https://github.com/Leonidas-from-XIV/node-xml2js/pull/674 + // is merged, we can revisit and hopefully remove this hack + value = fixObj(result) RED.util.setMessageProperty(msg,node.property,value); send(msg); done(); @@ -46,4 +52,18 @@ module.exports = function(RED) { }); } RED.nodes.registerType("xml",XMLNode); + + + function fixObj(obj) { + const res = {} + const keys = Object.keys(obj) + keys.forEach(k => { + if (typeof obj[k] === 'object' && obj[k]) { + res[k] = fixObj(obj[k]) + } else { + res[k] = obj[k] + } + }) + return res + } }