From 356c0c241634370f1461229d89fc602621985325 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Mon, 3 Feb 2025 14:11:29 +0530 Subject: [PATCH 1/9] Add include raw data option in request body. --- .../nodes/core/network/21-httpin.html | 36 +++-- .../@node-red/nodes/core/network/21-httpin.js | 138 +++++++++++++++--- .../@node-red/nodes/locales/de/messages.json | 1 + .../nodes/locales/en-US/messages.json | 1 + .../nodes/locales/es-ES/messages.json | 1 + .../@node-red/nodes/locales/fr/messages.json | 1 + .../@node-red/nodes/locales/ja/messages.json | 1 + .../@node-red/nodes/locales/ko/messages.json | 3 +- .../nodes/locales/pt-BR/messages.json | 1 + .../@node-red/nodes/locales/ru/messages.json | 1 + .../nodes/locales/zh-CN/messages.json | 1 + .../nodes/locales/zh-TW/messages.json | 3 +- packages/node_modules/node-red/red.js | 2 +- 13 files changed, 154 insertions(+), 36 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.html b/packages/node_modules/@node-red/nodes/core/network/21-httpin.html index f828077a1..c7724d9b4 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.html +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.html @@ -25,11 +25,22 @@ -
+ +
- - +
+
+ + +
+
+ + +
+
+ +
@@ -74,6 +85,7 @@ label:RED._("node-red:httpin.label.url")}, method: {value:"get",required:true}, upload: {value:false}, + includeRawBody: {value:false}, swaggerDoc: {type:"swagger-doc", required:false} }, inputs:0, @@ -115,16 +127,22 @@ $('.row-swagger-doc').hide(); } $("#node-input-method").on("change", function() { - if ($(this).val() === "post") { - $(".form-row-http-in-upload").show(); + var method = $(this).val(); + if(["post", "put", "patch","delete"].includes(method)){ + $("#form-reqBody-http-in-controller").show(); + $("#form-row-http-in-rawdata").show(); + if (method === "post") { + $("#form-row-http-in-upload").show(); + } else { + $("#form-row-http-in-upload").hide(); + } } else { - $(".form-row-http-in-upload").hide(); + $("#form-row-http-in-rawdata").hide(); + $("#form-row-http-in-upload").hide(); + $("#form-reqBody-http-in-controller").hide(); } }).change(); - - } - }); var headerTypes = [ {value:"content-type",label:"Content-Type",hasValue: false}, diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 22e83b411..28e90771e 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -26,16 +26,13 @@ module.exports = function(RED) { var mediaTyper = require('media-typer'); var isUtf8 = require('is-utf8'); var hashSum = require("hash-sum"); - - function rawBodyParser(req, res, next) { - if (req.skipRawBodyParser) { next(); } // don't parse this if told to skip - if (req._body) { return next(); } - req.body = ""; - req._body = true; - - var isText = true; - var checkUTF = false; - + var PassThrough = require('stream').PassThrough + + function rawBodyParser(req, _res, next) { + if(!req._nodeRedReqStream) { + return next(); + } + var isText = true, checkUTF = false; if (req.headers['content-type']) { var contentType = typer.parse(req.headers['content-type']) if (contentType.type) { @@ -51,27 +48,105 @@ module.exports = function(RED) { && (parsedType.subtype !== "x-protobuf")) { checkUTF = true; } else { - // application/octet-stream or application/cbor isText = false; } - } } - getBody(req, { + getBody(req._nodeRedReqStream, { length: req.headers['content-length'], encoding: isText ? "utf8" : null }, function (err, buf) { - if (err) { return next(err); } + if (err) { + return next(err); + } if (!isText && checkUTF && isUtf8(buf)) { buf = buf.toString() } - req.body = buf; - next(); + Object.defineProperty(req, "rawRequestBody", { + value: buf, + enumerable: true + }) + return next(); }); } - var corsSetup = false; + function getRootApp(app) { + if (typeof app.parent === 'undefined') { + return app; + } + return getRootApp(app.parent) + } + + function createRouteId(req) { + var method = req.method.toLowerCase(), url = req.url; + return `${method}:${url}`; + } + + var rootApp = getRootApp(RED.httpNode) + + // Check if middleware already exists + var isMiddlewarePresent = rootApp._router.stack.some(layer => layer.name === 'setupRawBodyCapture') + + if (!isMiddlewarePresent) { + // Initialize HTTP node settings storage + var httpInNodes = new Map() + + rootApp.set('httpInNodes', httpInNodes); + + // This middleware must always be at the root to handle the raw request body correctly + function setupRawBodyCapture(req, _res, next) { + var httpInNodeId = createRouteId(req) + + // Check if settings for this ID exist + if (httpInNodes.has(httpInNodeId)) { + var httpInNode = httpInNodes.get(httpInNodeId); + // If raw body inclusion is disabled, skip the processing + if (!httpInNode.includeRawBody) { + return next(); + } + // Create a PassThrough stream to capture the request body + var passThrough = new PassThrough(); + + // Function to handle 'data' event + function onData(chunk) { + passThrough.write(chunk); + } + + // Function to handle 'end' or 'error' events + function onEnd(err) { + if (err) { + passThrough.destroy(err); + } else { + passThrough.end(); + } + } + + // Attach event listeners to the request stream + req.on('data', onData); + req.once('end', onEnd); + req.once('error', onEnd); + + // Remove listeners once the request is closed + req.once('close', function () { + req.removeListener('data', onData); + }); + + // Attach the passThrough stream to the request + Object.defineProperty(req, "_nodeRedReqStream", { + value: passThrough + }); + + return next(); + } + // Proceed to the next middleware if no settings found + return next(); + } + // Add middleware to the stack + rootApp.use(setupRawBodyCapture) + // Move the router to top of the stack + rootApp._router.stack.unshift(rootApp._router.stack.pop()) + } function createRequestWrapper(node,req) { // This misses a bunch of properties (eg headers). Before we use this function @@ -125,6 +200,7 @@ module.exports = function(RED) { return wrapper; } + function createResponseWrapper(node,res) { var wrapper = { _res: res @@ -188,6 +264,7 @@ module.exports = function(RED) { } this.method = n.method; this.upload = n.upload; + this.includeRawBody = n.includeRawBody; this.swaggerDoc = n.swaggerDoc; var node = this; @@ -227,7 +304,9 @@ module.exports = function(RED) { } var maxApiRequestSize = RED.settings.apiMaxLength || '5mb'; + var jsonParser = bodyParser.json({limit:maxApiRequestSize}); + var urlencParser = bodyParser.urlencoded({limit:maxApiRequestSize,extended:true}); var metricsHandler = function(req,res,next) { next(); } @@ -254,25 +333,35 @@ module.exports = function(RED) { var mp = multer({ storage: multer.memoryStorage() }).any(); multipartParser = function(req,res,next) { mp(req,res,function(err) { - req._body = true; next(err); }) }; } if (this.method == "get") { - RED.httpNode.get(this.url,cookieParser(),httpMiddleware,corsHandler,metricsHandler,this.callback,this.errorHandler); + RED.httpNode.get(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, this.callback, this.errorHandler); } else if (this.method == "post") { - RED.httpNode.post(this.url,cookieParser(),httpMiddleware,corsHandler,metricsHandler,jsonParser,urlencParser,multipartParser,rawBodyParser,this.callback,this.errorHandler); + RED.httpNode.post(this.url,cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, multipartParser, rawBodyParser, this.callback, this.errorHandler); } else if (this.method == "put") { - RED.httpNode.put(this.url,cookieParser(),httpMiddleware,corsHandler,metricsHandler,jsonParser,urlencParser,rawBodyParser,this.callback,this.errorHandler); + RED.httpNode.put(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, rawBodyParser, this.callback, this.errorHandler); } else if (this.method == "patch") { - RED.httpNode.patch(this.url,cookieParser(),httpMiddleware,corsHandler,metricsHandler,jsonParser,urlencParser,rawBodyParser,this.callback,this.errorHandler); + RED.httpNode.patch(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, rawBodyParser, this.callback, this.errorHandler); } else if (this.method == "delete") { - RED.httpNode.delete(this.url,cookieParser(),httpMiddleware,corsHandler,metricsHandler,jsonParser,urlencParser,rawBodyParser,this.callback,this.errorHandler); + RED.httpNode.delete(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, rawBodyParser, this.callback, this.errorHandler); } + // unique id for httpInNode based on method name and url path + var httpInNodeId = createRouteId({url: this.url, method: this.method}) + + // get httpInNode from RED.httpNode + var httpInNodes = rootApp.get('httpInNodes') + + // Add httpInNode to RED.httpNode + httpInNodes.set(httpInNodeId, this); + this.on("close",function() { + // remove httpInNodeSettings from RED.httpNode + httpInNodes.remove(httpInNodeId) var node = this; RED.httpNode._router.stack.forEach(function(route,i,routes) { if (route.route && route.route.path === node.url && route.route.methods[node.method]) { @@ -284,9 +373,9 @@ module.exports = function(RED) { this.warn(RED._("httpin.errors.not-created")); } } + RED.nodes.registerType("http in",HTTPIn); - function HTTPOut(n) { RED.nodes.createNode(this,n); var node = this; @@ -361,5 +450,6 @@ module.exports = function(RED) { done(); }); } + RED.nodes.registerType("http response",HTTPOut); } diff --git a/packages/node_modules/@node-red/nodes/locales/de/messages.json b/packages/node_modules/@node-red/nodes/locales/de/messages.json index a51e504cf..332712902 100644 --- a/packages/node_modules/@node-red/nodes/locales/de/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/de/messages.json @@ -451,6 +451,7 @@ "upload": "Dateiuploads akzeptieren", "status": "Statuscode", "headers": "Kopfzeilen", + "rawBody": "Rohdaten einbeziehen?", "other": "andere", "paytoqs": { "ignore": "Ignorieren", diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index bc89992e2..76ff2394b 100644 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -515,6 +515,7 @@ "doc": "Docs", "return": "Return", "upload": "Accept file uploads?", + "rawBody": "Include Raw Data?", "status": "Status code", "headers": "Headers", "other": "other", diff --git a/packages/node_modules/@node-red/nodes/locales/es-ES/messages.json b/packages/node_modules/@node-red/nodes/locales/es-ES/messages.json index b8ac84f1c..56465e451 100644 --- a/packages/node_modules/@node-red/nodes/locales/es-ES/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/es-ES/messages.json @@ -518,6 +518,7 @@ "status": "Status code", "headers": "Headers", "other": "otro", + "rawBody": "¿Incluir datos sin procesar?", "paytoqs": { "ignore": "Ignore", "query": "Append to query-string parameters", diff --git a/packages/node_modules/@node-red/nodes/locales/fr/messages.json b/packages/node_modules/@node-red/nodes/locales/fr/messages.json index 99002f48f..b675efcb5 100644 --- a/packages/node_modules/@node-red/nodes/locales/fr/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/fr/messages.json @@ -518,6 +518,7 @@ "status": "Code d'état", "headers": "En-têtes", "other": "Autre", + "rawBody": "Inclure les données brutes ?", "paytoqs": { "ignore": "Ignorer", "query": "Joindre aux paramètres de chaîne de requête", diff --git a/packages/node_modules/@node-red/nodes/locales/ja/messages.json b/packages/node_modules/@node-red/nodes/locales/ja/messages.json index 8d38ac077..eefdf5762 100644 --- a/packages/node_modules/@node-red/nodes/locales/ja/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/ja/messages.json @@ -518,6 +518,7 @@ "status": "ステータスコード", "headers": "ヘッダ", "other": "その他", + "rawBody": "生データを含める？", "paytoqs": { "ignore": "無視", "query": "クエリパラメータに追加", diff --git a/packages/node_modules/@node-red/nodes/locales/ko/messages.json b/packages/node_modules/@node-red/nodes/locales/ko/messages.json index c82e0f51b..abef1c89c 100644 --- a/packages/node_modules/@node-red/nodes/locales/ko/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/ko/messages.json @@ -397,7 +397,8 @@ "binaryBuffer": "바이너리 버퍼", "jsonObject": "JSON오브젝트", "authType": "종류별", - "bearerToken": "토큰" + "bearerToken": "토큰", + "rawBody": "원시 데이터를 포함할까요?" }, "setby": "- msg.method에 정의 -", "basicauth": "인증을 사용", diff --git a/packages/node_modules/@node-red/nodes/locales/pt-BR/messages.json b/packages/node_modules/@node-red/nodes/locales/pt-BR/messages.json index 51e1fd897..003c484af 100644 --- a/packages/node_modules/@node-red/nodes/locales/pt-BR/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/pt-BR/messages.json @@ -506,6 +506,7 @@ "status": "Código de estado", "headers": "Cabeçalhos", "other": "outro", + "rawBody": "Incluir dados brutos?", "paytoqs" : { "ignore": "Ignorar", "query": "Anexar aos parâmetros da cadeia de caracteres de consulta", diff --git a/packages/node_modules/@node-red/nodes/locales/ru/messages.json b/packages/node_modules/@node-red/nodes/locales/ru/messages.json index 2694ac6a5..49ed92239 100644 --- a/packages/node_modules/@node-red/nodes/locales/ru/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/ru/messages.json @@ -411,6 +411,7 @@ "status": "Код состояния", "headers": "Заголовки", "other": "другое", + "rawBody": "Включить необработанные данные?", "paytoqs": { "ignore": "Игнорировать", "query": "Добавлять к параметрам строки запроса", diff --git a/packages/node_modules/@node-red/nodes/locales/zh-CN/messages.json b/packages/node_modules/@node-red/nodes/locales/zh-CN/messages.json index 7d5616a8f..23b6b07ee 100644 --- a/packages/node_modules/@node-red/nodes/locales/zh-CN/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/zh-CN/messages.json @@ -508,6 +508,7 @@ "status": "状态码", "headers": "头", "other": "其他", + "rawBody": "包含原始数据？", "paytoqs": { "ignore": "忽略", "query": "附加到查询字符串参数", diff --git a/packages/node_modules/@node-red/nodes/locales/zh-TW/messages.json b/packages/node_modules/@node-red/nodes/locales/zh-TW/messages.json index 7d16c5817..2f9f3b560 100644 --- a/packages/node_modules/@node-red/nodes/locales/zh-TW/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/zh-TW/messages.json @@ -416,7 +416,8 @@ "binaryBuffer": "二進制buffer", "jsonObject": "解析的JSON對象", "authType": "類型", - "bearerToken": "Token" + "bearerToken": "Token", + "rawBody": "包含原始數據？" }, "setby": "- 用 msg.method 設定 -", "basicauth": "基本認證", diff --git a/packages/node_modules/node-red/red.js b/packages/node_modules/node-red/red.js index 5f3c9da25..fa5c840bc 100755 --- a/packages/node_modules/node-red/red.js +++ b/packages/node_modules/node-red/red.js @@ -65,6 +65,7 @@ var knownOpts = { "version": Boolean, "define": [String, Array] }; + var shortHands = { "?":["--help"], "p":["--port"], @@ -286,7 +287,6 @@ httpsPromise.then(function(startupHttps) { server = http.createServer(function(req,res) {app(req,res);}); } server.setMaxListeners(0); - function formatRoot(root) { if (root[0] != "/") { root = "/" + root; From 90328ef75e64d95f006d401c1da0506bf0282a60 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Fri, 7 Feb 2025 00:09:13 +0530 Subject: [PATCH 2/9] Replace httpInNodes.remove with httpInNodes.delete to fix timeout error --- packages/node_modules/@node-red/nodes/core/network/21-httpin.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 28e90771e..5c6ea0010 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -361,7 +361,7 @@ module.exports = function(RED) { this.on("close",function() { // remove httpInNodeSettings from RED.httpNode - httpInNodes.remove(httpInNodeId) + httpInNodes.delete(httpInNodeId) var node = this; RED.httpNode._router.stack.forEach(function(route,i,routes) { if (route.route && route.route.path === node.url && route.route.methods[node.method]) { From 7321cd20db631cc76b35f40af3e89702d2888f58 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Sat, 8 Feb 2025 10:59:37 +0530 Subject: [PATCH 3/9] Rename createRouteId to getRouteId and update related references; improve raw body capture stream handling --- .../@node-red/nodes/core/network/21-httpin.js | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 5c6ea0010..4d7786838 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -78,7 +78,7 @@ module.exports = function(RED) { return getRootApp(app.parent) } - function createRouteId(req) { + function getRouteId(req) { var method = req.method.toLowerCase(), url = req.url; return `${method}:${url}`; } @@ -94,49 +94,63 @@ module.exports = function(RED) { rootApp.set('httpInNodes', httpInNodes); - // This middleware must always be at the root to handle the raw request body correctly + /** + * This middleware must always be at the root to handle the raw request body correctly + * @param {import('express').Request} req + * @param {import('express').Response} _res + * @param {import('express').NextFunction} next + * @returns + */ function setupRawBodyCapture(req, _res, next) { - var httpInNodeId = createRouteId(req) + var httpInNodeId = getRouteId(req) // Check if settings for this ID exist if (httpInNodes.has(httpInNodeId)) { + // Get the httpInNode by routeId var httpInNode = httpInNodes.get(httpInNodeId); + // If raw body inclusion is disabled, skip the processing if (!httpInNode.includeRawBody) { return next(); } + // Create a PassThrough stream to capture the request body - var passThrough = new PassThrough(); + var cloneStream = new PassThrough(); // Function to handle 'data' event function onData(chunk) { - passThrough.write(chunk); + // Continue pushing chunks into clone stream if it is writable. + if (!cloneStream.writable) { + cloneStream.write(chunk); + } } // Function to handle 'end' or 'error' events function onEnd(err) { if (err) { - passThrough.destroy(err); + // If clone stream is already destroyed don't call destory function again + if (!cloneStream.destroyed) { + cloneStream.destroy(err); + } } else { - passThrough.end(); + cloneStream.end(); } } // Attach event listeners to the request stream - req.on('data', onData); - req.once('end', onEnd); - req.once('error', onEnd); + req.on('data', onData) + .once('end', onEnd) + .once('error', onEnd) + // Remove listeners once the request is closed + .once('close', () => { + req.removeListener('data', onData); + }) - // Remove listeners once the request is closed - req.once('close', function () { - req.removeListener('data', onData); - }); - - // Attach the passThrough stream to the request + // Attach the clone stream to the request Object.defineProperty(req, "_nodeRedReqStream", { - value: passThrough + value: cloneStream }); - + // Proceed to the next middleware if no settings found return next(); } // Proceed to the next middleware if no settings found @@ -351,7 +365,7 @@ module.exports = function(RED) { } // unique id for httpInNode based on method name and url path - var httpInNodeId = createRouteId({url: this.url, method: this.method}) + var httpInNodeId = getRouteId({url: this.url, method: this.method}) // get httpInNode from RED.httpNode var httpInNodes = rootApp.get('httpInNodes') From 9d811c7bce3a0e6a0aa0590ae1313881d0dc6639 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Sat, 8 Feb 2025 11:13:04 +0530 Subject: [PATCH 4/9] Improve clone stream handling by adding safety checks for writable and destroyed states --- .../@node-red/nodes/core/network/21-httpin.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 4d7786838..e8f99d2dc 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -119,8 +119,8 @@ module.exports = function(RED) { // Function to handle 'data' event function onData(chunk) { - // Continue pushing chunks into clone stream if it is writable. - if (!cloneStream.writable) { + // Safely call clone stream write + if (cloneStream.writable) { cloneStream.write(chunk); } } @@ -128,12 +128,15 @@ module.exports = function(RED) { // Function to handle 'end' or 'error' events function onEnd(err) { if (err) { - // If clone stream is already destroyed don't call destory function again + // Safely call clone stream destroy method if (!cloneStream.destroyed) { cloneStream.destroy(err); } } else { - cloneStream.end(); + // Safely call clone stream end method + if(cloneStream.writable) { + cloneStream.end(); + } } } From 9a26ee30522893a6278462d3069e89733c3c97c0 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Sun, 9 Feb 2025 13:26:05 +0530 Subject: [PATCH 5/9] Refactor raw body capture middleware and improve route key generation for HTTP nodes --- .../@node-red/nodes/core/network/21-httpin.js | 180 ++++++++++-------- 1 file changed, 96 insertions(+), 84 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index e8f99d2dc..81d1f4e9b 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -27,9 +27,20 @@ module.exports = function(RED) { var isUtf8 = require('is-utf8'); var hashSum = require("hash-sum"); var PassThrough = require('stream').PassThrough - + var rootApp = getRootApp(RED.httpNode) + + // Check if middleware already exists + var isMiddlewareExists = rootApp._router.stack.some(layer => layer.name === 'setupRawBodyCapture') + + /** + * This middleware parses the raw body if the user enables it. + * @param {import('express').Request} req + * @param {import('express').Response} _res + * @param {import('express').NextFunction} next + * @returns + */ function rawBodyParser(req, _res, next) { - if(!req._nodeRedReqStream) { + if (!req._nodeRedReqStream) { return next(); } var isText = true, checkUTF = false; @@ -57,8 +68,8 @@ module.exports = function(RED) { length: req.headers['content-length'], encoding: isText ? "utf8" : null }, function (err, buf) { - if (err) { - return next(err); + if (err) { + return next(err) } if (!isText && checkUTF && isUtf8(buf)) { buf = buf.toString() @@ -71,6 +82,11 @@ module.exports = function(RED) { }); } + /** + * This method retrieves the root app by traversing the parent hierarchy. + * @param {import('express').Application} app + * @returns {import('express').Application} + */ function getRootApp(app) { if (typeof app.parent === 'undefined') { return app; @@ -78,87 +94,87 @@ module.exports = function(RED) { return getRootApp(app.parent) } - function getRouteId(req) { - var method = req.method.toLowerCase(), url = req.url; + /** + * It provide the unique route key + * @param {{method: string, url: string}} obj + * @returns + */ + function getRouteKey(obj) { + var method = obj.method.toUpperCase() + // Normalize the URL by replacing double slashes with a single slash and removing the trailing slash + var url = obj.url.replace(/\/{2,}/g, '/').replace(/\/$/, '') return `${method}:${url}`; } - var rootApp = getRootApp(RED.httpNode) + /** + * This middleware is for clone the request stream + * @param {import('express').Request} req + * @param {import('express').Response} _res + * @param {import('express').NextFunction} next + * @returns + */ + function setupRawBodyCapture(req, _res, next) { + var routeKey = getRouteKey({method: req.method, url: req._parsedUrl.pathname}) + var httpInNodes = rootApp.get('httpInNodes') + // Check if settings for this ID exist + if (httpInNodes.has(routeKey)) { + // Get the httpInNode by routeId + var httpInNode = httpInNodes.get(routeKey); - // Check if middleware already exists - var isMiddlewarePresent = rootApp._router.stack.some(layer => layer.name === 'setupRawBodyCapture') - - if (!isMiddlewarePresent) { - // Initialize HTTP node settings storage - var httpInNodes = new Map() - - rootApp.set('httpInNodes', httpInNodes); - - /** - * This middleware must always be at the root to handle the raw request body correctly - * @param {import('express').Request} req - * @param {import('express').Response} _res - * @param {import('express').NextFunction} next - * @returns - */ - function setupRawBodyCapture(req, _res, next) { - var httpInNodeId = getRouteId(req) - - // Check if settings for this ID exist - if (httpInNodes.has(httpInNodeId)) { - // Get the httpInNode by routeId - var httpInNode = httpInNodes.get(httpInNodeId); - - // If raw body inclusion is disabled, skip the processing - if (!httpInNode.includeRawBody) { - return next(); - } - - // Create a PassThrough stream to capture the request body - var cloneStream = new PassThrough(); - - // Function to handle 'data' event - function onData(chunk) { - // Safely call clone stream write - if (cloneStream.writable) { - cloneStream.write(chunk); - } - } - - // Function to handle 'end' or 'error' events - function onEnd(err) { - if (err) { - // Safely call clone stream destroy method - if (!cloneStream.destroyed) { - cloneStream.destroy(err); - } - } else { - // Safely call clone stream end method - if(cloneStream.writable) { - cloneStream.end(); - } - } - } - - // Attach event listeners to the request stream - req.on('data', onData) - .once('end', onEnd) - .once('error', onEnd) - // Remove listeners once the request is closed - .once('close', () => { - req.removeListener('data', onData); - }) - - // Attach the clone stream to the request - Object.defineProperty(req, "_nodeRedReqStream", { - value: cloneStream - }); - // Proceed to the next middleware if no settings found + // If raw body inclusion is disabled, skip the processing + if (!httpInNode.includeRawBody) { return next(); } + + // Create a PassThrough stream to capture the request body + var cloneStream = new PassThrough(); + + // Function to handle 'data' event + function onData(chunk) { + // Safely call clone stream write + if (cloneStream.writable) { + cloneStream.write(chunk); + } + } + + // Function to handle 'end' or 'error' events + function onEnd(err) { + if (err) { + // Safely call clone stream destroy method + if (!cloneStream.destroyed) { + cloneStream.destroy(err); + } + } else { + // Safely call clone stream end method + if (cloneStream.writable) { + cloneStream.end(); + } + } + } + + // Attach event listeners to the request stream + req.on('data', onData) + .once('end', onEnd) + .once('error', onEnd) + // Remove listeners once the request is closed + .once('close', () => { + req.removeListener('data', onData); + }) + + // Attach the clone stream to the request + Object.defineProperty(req, "_nodeRedReqStream", { + value: cloneStream + }); // Proceed to the next middleware if no settings found return next(); } + // Proceed to the next middleware if no settings found + return next(); + } + + if (!isMiddlewareExists) { + // Initialize HTTP node storage + rootApp.set('httpInNodes', new Map()); // Add middleware to the stack rootApp.use(setupRawBodyCapture) // Move the router to top of the stack @@ -366,19 +382,15 @@ module.exports = function(RED) { } else if (this.method == "delete") { RED.httpNode.delete(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, rawBodyParser, this.callback, this.errorHandler); } - - // unique id for httpInNode based on method name and url path - var httpInNodeId = getRouteId({url: this.url, method: this.method}) - // get httpInNode from RED.httpNode + var routeKey = getRouteKey({method: this.method, url: RED.httpNode.path() + this.url}) + var httpInNodes = rootApp.get('httpInNodes') - // Add httpInNode to RED.httpNode - httpInNodes.set(httpInNodeId, this); + httpInNodes.set(routeKey, this); this.on("close",function() { - // remove httpInNodeSettings from RED.httpNode - httpInNodes.delete(httpInNodeId) + httpInNodes.delete(routeKey) var node = this; RED.httpNode._router.stack.forEach(function(route,i,routes) { if (route.route && route.route.path === node.url && route.route.methods[node.method]) { From caeb5d3538dfc96aa3d5e04d9ae9aa2bf6ef773f Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Sun, 9 Feb 2025 21:09:30 +0530 Subject: [PATCH 6/9] Refactor raw body capture middleware to use a Set for route management and improve code consistency --- .../@node-red/nodes/core/network/21-httpin.js | 58 ++++++++----------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 81d1f4e9b..34030f13d 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -26,11 +26,9 @@ module.exports = function(RED) { var mediaTyper = require('media-typer'); var isUtf8 = require('is-utf8'); var hashSum = require("hash-sum"); - var PassThrough = require('stream').PassThrough - var rootApp = getRootApp(RED.httpNode) - - // Check if middleware already exists - var isMiddlewareExists = rootApp._router.stack.some(layer => layer.name === 'setupRawBodyCapture') + var PassThrough = require('stream').PassThrough; + var rootApp = getRootApp(RED.httpNode); + var rawDataRoutes = new Set(); /** * This middleware parses the raw body if the user enables it. @@ -69,7 +67,7 @@ module.exports = function(RED) { encoding: isText ? "utf8" : null }, function (err, buf) { if (err) { - return next(err) + return next(err); } if (!isText && checkUTF && isUtf8(buf)) { buf = buf.toString() @@ -77,7 +75,7 @@ module.exports = function(RED) { Object.defineProperty(req, "rawRequestBody", { value: buf, enumerable: true - }) + }); return next(); }); } @@ -91,7 +89,7 @@ module.exports = function(RED) { if (typeof app.parent === 'undefined') { return app; } - return getRootApp(app.parent) + return getRootApp(app.parent); } /** @@ -100,9 +98,9 @@ module.exports = function(RED) { * @returns */ function getRouteKey(obj) { - var method = obj.method.toUpperCase() + var method = obj.method.toUpperCase(); // Normalize the URL by replacing double slashes with a single slash and removing the trailing slash - var url = obj.url.replace(/\/{2,}/g, '/').replace(/\/$/, '') + var url = obj.url.replace(/\/{2,}/g, '/').replace(/\/$/, ''); return `${method}:${url}`; } @@ -114,17 +112,9 @@ module.exports = function(RED) { * @returns */ function setupRawBodyCapture(req, _res, next) { - var routeKey = getRouteKey({method: req.method, url: req._parsedUrl.pathname}) - var httpInNodes = rootApp.get('httpInNodes') + var routeKey = getRouteKey({ method: req.method, url: req._parsedUrl.pathname }); // Check if settings for this ID exist - if (httpInNodes.has(routeKey)) { - // Get the httpInNode by routeId - var httpInNode = httpInNodes.get(routeKey); - - // If raw body inclusion is disabled, skip the processing - if (!httpInNode.includeRawBody) { - return next(); - } + if (rawDataRoutes.has(routeKey)) { // Create a PassThrough stream to capture the request body var cloneStream = new PassThrough(); @@ -159,7 +149,7 @@ module.exports = function(RED) { // Remove listeners once the request is closed .once('close', () => { req.removeListener('data', onData); - }) + }); // Attach the clone stream to the request Object.defineProperty(req, "_nodeRedReqStream", { @@ -172,14 +162,10 @@ module.exports = function(RED) { return next(); } - if (!isMiddlewareExists) { - // Initialize HTTP node storage - rootApp.set('httpInNodes', new Map()); - // Add middleware to the stack - rootApp.use(setupRawBodyCapture) - // Move the router to top of the stack - rootApp._router.stack.unshift(rootApp._router.stack.pop()) - } + // Add middleware to the stack + rootApp.use(setupRawBodyCapture); + // Move the router to top of the stack + rootApp._router.stack.unshift(rootApp._router.stack.pop()); function createRequestWrapper(node,req) { // This misses a bunch of properties (eg headers). Before we use this function @@ -301,6 +287,12 @@ module.exports = function(RED) { this.swaggerDoc = n.swaggerDoc; var node = this; + var routeKey = getRouteKey({method: this.method, url: RED.httpNode.path() + this.url}); + + // If the user enables raw body, add it to the raw data routes. + if(this.includeRawBody) { + rawDataRoutes.add(routeKey); + } this.errorHandler = function(err,req,res,next) { node.warn(err); @@ -383,14 +375,10 @@ module.exports = function(RED) { RED.httpNode.delete(this.url, cookieParser(), httpMiddleware, corsHandler, metricsHandler, jsonParser, urlencParser, rawBodyParser, this.callback, this.errorHandler); } - var routeKey = getRouteKey({method: this.method, url: RED.httpNode.path() + this.url}) - var httpInNodes = rootApp.get('httpInNodes') - - httpInNodes.set(routeKey, this); - this.on("close",function() { - httpInNodes.delete(routeKey) + // Remove it from the raw data routes if the node is closed + rawDataRoutes.delete(routeKey); var node = this; RED.httpNode._router.stack.forEach(function(route,i,routes) { if (route.route && route.route.path === node.url && route.route.methods[node.method]) { From bb43d63b54cd6376e27195c2b79987e9360f3397 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Mon, 10 Feb 2025 19:40:49 +0530 Subject: [PATCH 7/9] Fix condition check for RED.httpNode to ensure proper middleware setup --- .../@node-red/nodes/core/network/21-httpin.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 34030f13d..0083ad315 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -16,6 +16,7 @@ module.exports = function(RED) { "use strict"; + var rootApp; var bodyParser = require("body-parser"); var multer = require("multer"); var cookieParser = require("cookie-parser"); @@ -27,7 +28,6 @@ module.exports = function(RED) { var isUtf8 = require('is-utf8'); var hashSum = require("hash-sum"); var PassThrough = require('stream').PassThrough; - var rootApp = getRootApp(RED.httpNode); var rawDataRoutes = new Set(); /** @@ -162,10 +162,12 @@ module.exports = function(RED) { return next(); } - // Add middleware to the stack - rootApp.use(setupRawBodyCapture); - // Move the router to top of the stack - rootApp._router.stack.unshift(rootApp._router.stack.pop()); + if(typeof RED.httpNode === 'function' && (rootApp = getRootApp(RED.httpNode))) { + // Add middleware to the stack + rootApp.use(setupRawBodyCapture); + // Move the router to top of the stack + rootApp._router.stack.unshift(rootApp._router.stack.pop()); + } function createRequestWrapper(node,req) { // This misses a bunch of properties (eg headers). Before we use this function From 3fc94e7b99957d662eec35501b1dfc1fdbed3fc7 Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Tue, 11 Feb 2025 00:34:32 +0530 Subject: [PATCH 8/9] Clarify comment for route key existence check in raw body capture middleware --- .../node_modules/@node-red/nodes/core/network/21-httpin.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 0083ad315..14bc4d465 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -113,7 +113,7 @@ module.exports = function(RED) { */ function setupRawBodyCapture(req, _res, next) { var routeKey = getRouteKey({ method: req.method, url: req._parsedUrl.pathname }); - // Check if settings for this ID exist + // Check if routeKey exist in rawDataRoutes if (rawDataRoutes.has(routeKey)) { // Create a PassThrough stream to capture the request body @@ -155,10 +155,8 @@ module.exports = function(RED) { Object.defineProperty(req, "_nodeRedReqStream", { value: cloneStream }); - // Proceed to the next middleware if no settings found - return next(); } - // Proceed to the next middleware if no settings found + // Proceed to the next middleware return next(); } From 3a29330021d3f11cb3df403168783deff3c179af Mon Sep 17 00:00:00 2001 From: Debadutta Panda Date: Fri, 14 Feb 2025 13:50:06 +0530 Subject: [PATCH 9/9] Update comment to clarify middleware positioning in HTTP node setup --- packages/node_modules/@node-red/nodes/core/network/21-httpin.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js index 14bc4d465..45c980e2a 100644 --- a/packages/node_modules/@node-red/nodes/core/network/21-httpin.js +++ b/packages/node_modules/@node-red/nodes/core/network/21-httpin.js @@ -163,7 +163,7 @@ module.exports = function(RED) { if(typeof RED.httpNode === 'function' && (rootApp = getRootApp(RED.httpNode))) { // Add middleware to the stack rootApp.use(setupRawBodyCapture); - // Move the router to top of the stack + // Move the middleware to top of the stack rootApp._router.stack.unshift(rootApp._router.stack.pop()); }