diff --git a/include/flatbufserver/FlatBufferServer.h b/include/flatbufserver/FlatBufferServer.h index ba39c647..6120b845 100644 --- a/include/flatbufserver/FlatBufferServer.h +++ b/include/flatbufserver/FlatBufferServer.h @@ -71,5 +71,7 @@ private: quint16 _port; const QJsonDocument _config; + int _pixelDecimation; + QVector _openConnections; }; diff --git a/include/utils/ImageResampler.h b/include/utils/ImageResampler.h index 7aba7a40..4f6dab46 100644 --- a/include/utils/ImageResampler.h +++ b/include/utils/ImageResampler.h @@ -13,6 +13,7 @@ public: void setHorizontalPixelDecimation(int decimator) { _horizontalDecimation = decimator; } void setVerticalPixelDecimation(int decimator) { _verticalDecimation = decimator; } + void setPixelDecimation(int decimator) { _horizontalDecimation = decimator; _verticalDecimation = decimator;} void setCropping(int cropLeft, int cropRight, int cropTop, int cropBottom); void setVideoMode(VideoMode mode) { _videoMode = mode; } void setFlipMode(FlipMode mode) { _flipMode = mode; } diff --git a/libsrc/flatbufserver/FlatBufferClient.cpp b/libsrc/flatbufserver/FlatBufferClient.cpp index 8df752ca..f1659585 100644 --- a/libsrc/flatbufserver/FlatBufferClient.cpp +++ b/libsrc/flatbufserver/FlatBufferClient.cpp @@ -1,4 +1,5 @@ #include "FlatBufferClient.h" +#include // qt #include @@ -15,6 +16,8 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par , _timeout(timeout * 1000) , _priority() { + _imageResampler.setPixelDecimation(1); + // timer setup _timeoutTimer->setSingleShot(true); _timeoutTimer->setInterval(_timeout); @@ -25,6 +28,11 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par connect(_socket, &QTcpSocket::disconnected, this, &FlatBufferClient::disconnected); } +void FlatBufferClient::setPixelDecimation(int decimator) +{ + _imageResampler.setPixelDecimation(decimator); +} + void FlatBufferClient::readyRead() { _timeoutTimer->start(); @@ -141,55 +149,71 @@ void FlatBufferClient::handleRegisterCommand(const hyperionnet::Register *regReq void FlatBufferClient::handleImageCommand(const hyperionnet::Image *image) { + Image imageRGB; + // extract parameters int duration = image->duration(); - const void* reqPtr; if ((reqPtr = image->data_as_RawImage()) != nullptr) { - const auto *img = static_cast(reqPtr); - const auto & imageData = img->data(); - const int width = img->width(); - const int height = img->height(); + const auto* img = static_cast(reqPtr); - if (width <= 0 || height <= 0) + hyperionnet::RawImageT rawImageNative; + img->UnPackTo(&rawImageNative); + + const int width = rawImageNative.width; + const int height = rawImageNative.height; + + if (width <= 0 || height <= 0 || rawImageNative.data.empty()) { - sendErrorReply("Size of image data does not match with the width and height"); + sendErrorReply("Invalid width and/or height or no raw image data provided"); return; } // check consistency of the size of the received data - int channelCount = (int)imageData->size()/(width*height); - if (channelCount != 3 && channelCount != 4) + int bytesPerPixel = rawImageNative.data.size() / (width * height); + if (bytesPerPixel != 3 && bytesPerPixel != 4) { sendErrorReply("Size of image data does not match with the width and height"); return; } - // create ImageRgb - Image imageRGB(width, height); - if (channelCount == 3) - { - memmove(imageRGB.memptr(), imageData->data(), imageData->size()); - } - - if (channelCount == 4) - { - for (int source=0, destination=0; source < width * height * static_cast(sizeof(ColorRgb)); source+=sizeof(ColorRgb), destination+=sizeof(ColorRgba)) - { - memmove((uint8_t*)imageRGB.memptr() + source, imageData->data() + destination, sizeof(ColorRgb)); - } - } - - emit setGlobalInputImage(_priority, imageRGB, duration); - emit setBufferImage("FlatBuffer", imageRGB); + imageRGB.resize(width, height); + processRawImage(rawImageNative, bytesPerPixel, _imageResampler, imageRGB); } + else if ((reqPtr = image->data_as_NV12Image()) != nullptr) + { + const auto* img = static_cast(reqPtr); + + hyperionnet::NV12ImageT nv12ImageNative; + img->UnPackTo(&nv12ImageNative); + + const int width = nv12ImageNative.width; + const int height = nv12ImageNative.height; + + if (width <= 0 || height <= 0 || nv12ImageNative.data_y.empty() || nv12ImageNative.data_uv.empty()) + { + sendErrorReply("Invalid width and/or height or no complete NV12 image data provided"); + return; + } + + imageRGB.resize(width, height); + processNV12Image(nv12ImageNative, _imageResampler, imageRGB); + + } + else + { + sendErrorReply("No or unknown image data provided"); + return; + } + + emit setGlobalInputImage(_priority, imageRGB, duration); + emit setBufferImage("FlatBuffer", imageRGB); // send reply sendSuccessReply(); } - void FlatBufferClient::handleClearCommand(const hyperionnet::Clear *clear) { // extract parameters @@ -242,3 +266,50 @@ void FlatBufferClient::sendErrorReply(const std::string &error) _builder.Clear(); } + +inline void FlatBufferClient::processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image& outputImage) { + + int width = raw_image.width; + int height = raw_image.height; + + int lineLength = width * bytesPerPixel; + PixelFormat pixelFormat = (bytesPerPixel == 4) ? PixelFormat::RGB32 : PixelFormat::RGB24; + + // Process the image + resampler.processImage( + raw_image.data.data(), // Raw RGB/RGBA buffer + width, // Image width + height, // Image height + lineLength, // Line length + pixelFormat, // Pixel format (RGB24/RGB32) + outputImage // Output image + ); +} + +inline void FlatBufferClient::processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image& outputImage) { + // Combine data_y and data_uv into a single buffer + int width = nv12_image.width; + int height = nv12_image.height; + + size_t y_size = nv12_image.data_y.size(); + size_t uv_size = nv12_image.data_uv.size(); + std::vector combined_buffer(y_size + uv_size); + + std::memcpy(combined_buffer.data(), nv12_image.data_y.data(), y_size); + std::memcpy(combined_buffer.data() + y_size, nv12_image.data_uv.data(), uv_size); + + // Determine line length (stride_y) + int lineLength = nv12_image.stride_y > 0 ? nv12_image.stride_y : width; + + PixelFormat pixelFormat = PixelFormat::NV12; + + // Process the image + resampler.processImage( + combined_buffer.data(), // Combined NV12 buffer + width, // Image width + height, // Image height + lineLength, // Line length for Y plane + pixelFormat, // Pixel format (NV12) + outputImage // Output image + ); +} \ No newline at end of file diff --git a/libsrc/flatbufserver/FlatBufferClient.h b/libsrc/flatbufserver/FlatBufferClient.h index 5776df54..24eca77b 100644 --- a/libsrc/flatbufserver/FlatBufferClient.h +++ b/libsrc/flatbufserver/FlatBufferClient.h @@ -6,6 +6,7 @@ #include #include #include +#include "utils/ImageResampler.h" // flatbuffer FBS #include "hyperion_reply_generated.h" @@ -33,6 +34,8 @@ public: /// explicit FlatBufferClient(QTcpSocket* socket, int timeout, QObject *parent = nullptr); + void setPixelDecimation(int decimator); + signals: /// /// @brief forward register data to HyperionDaemon @@ -138,6 +141,9 @@ private: /// void sendErrorReply(const std::string & error); + void processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image& outputImage); + void processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image& outputImage); + private: Logger *_log; QTcpSocket *_socket; @@ -148,6 +154,8 @@ private: QByteArray _receiveBuffer; + ImageResampler _imageResampler; + // Flatbuffers builder flatbuffers::FlatBufferBuilder _builder; }; diff --git a/libsrc/flatbufserver/FlatBufferServer.cpp b/libsrc/flatbufserver/FlatBufferServer.cpp index 33e015a6..cbddf78e 100644 --- a/libsrc/flatbufserver/FlatBufferServer.cpp +++ b/libsrc/flatbufserver/FlatBufferServer.cpp @@ -62,6 +62,12 @@ void FlatBufferServer::handleSettingsUpdate(settings::type type, const QJsonDocu _timeout = obj["timeout"].toInt(5000); // enable check obj["enable"].toBool(true) ? startServer() : stopServer(); + + _pixelDecimation = obj["pixelDecimation"].toInt(1); + for (const auto& client : _openConnections) + { + client->setPixelDecimation(_pixelDecimation); + } } } @@ -75,6 +81,9 @@ void FlatBufferServer::newConnection() { Debug(_log, "New connection from %s", QSTRING_CSTR(socket->peerAddress().toString())); FlatBufferClient *client = new FlatBufferClient(socket, _timeout, this); + + client->setPixelDecimation(_pixelDecimation); + // internal connect(client, &FlatBufferClient::clientDisconnected, this, &FlatBufferServer::clientDisconnected); connect(client, &FlatBufferClient::registerGlobalInput, GlobalSignals::getInstance(), &GlobalSignals::registerGlobalInput); diff --git a/libsrc/flatbufserver/hyperion_request.fbs b/libsrc/flatbufserver/hyperion_request.fbs index 5a076b65..06bd54a2 100644 --- a/libsrc/flatbufserver/hyperion_request.fbs +++ b/libsrc/flatbufserver/hyperion_request.fbs @@ -12,9 +12,17 @@ table RawImage { height:int = -1; } -union ImageType {RawImage} +table NV12Image { + data_y:[ubyte]; + data_uv:[ubyte]; + width:int; + height:int; + stride_y:int = 0; + stride_uv:int = 0; +} + +union ImageType {RawImage, NV12Image} -// Either RGB or RGBA data can be transferred table Image { data:ImageType (required); duration:int = -1; diff --git a/libsrc/hyperion/schema/schema-flatbufServer.json b/libsrc/hyperion/schema/schema-flatbufServer.json index 523bd45b..dc7fe85f 100644 --- a/libsrc/hyperion/schema/schema-flatbufServer.json +++ b/libsrc/hyperion/schema/schema-flatbufServer.json @@ -1,35 +1,41 @@ { "type" : "object", "title" : "edt_conf_fbs_heading_title", - "properties" : - { - "enable" : - { - "type" : "boolean", - "required" : true, - "title" : "edt_conf_general_enable_title", - "default" : true, - "propertyOrder" : 1 + "properties": { + "enable": { + "type": "boolean", + "required": true, + "title": "edt_conf_general_enable_title", + "default": true, + "propertyOrder": 1 }, - "port" : - { - "type" : "integer", - "required" : true, - "title" : "edt_conf_general_port_title", - "minimum" : 1024, - "maximum" : 65535, - "default" : 19400, - "propertyOrder" : 2 + "port": { + "type": "integer", + "required": true, + "title": "edt_conf_general_port_title", + "minimum": 1024, + "maximum": 65535, + "default": 19400, + "propertyOrder": 2 }, - "timeout" : - { - "type" : "integer", - "required" : true, - "title" : "edt_conf_fbs_timeout_title", - "append" : "edt_append_s", - "minimum" : 1, - "default" : 5, - "propertyOrder" : 3 + "timeout": { + "type": "integer", + "required": true, + "title": "edt_conf_fbs_timeout_title", + "append": "edt_append_s", + "minimum": 1, + "default": 5, + "propertyOrder": 3 + }, + "pixelDecimation": { + "type": "integer", + "title": "edt_conf_fg_pixelDecimation_title", + "minimum": 1, + "maximum": 30, + "default": 1, + "required": false, + "access": "advanced", + "propertyOrder": 4 } }, "additionalProperties" : false