frodovdr/hyperion.ng
1
0
Fork 0
mirror of https://github.com/hyperion-project/hyperion.ng.git synced 2025-03-01 10:33:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support NV12 format on Flatbuffer (#1806)

Browse Source
This commit is contained in:
LordGrey 2024-12-01 17:08:00 +01:00 committed by GitHub
parent bc1bfbc83a
commit 6e3357ea2d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 161 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/flatbufserver/FlatBufferServer.h
View File

@ -71,5 +71,7 @@ private:
quint16 _port;
const QJsonDocument _config;
int _pixelDecimation;
QVector<FlatBufferClient*> _openConnections;
};

1
include/utils/ImageResampler.h
View File

@ -13,6 +13,7 @@ public:
void setHorizontalPixelDecimation(int decimator) { _horizontalDecimation = decimator; }
void setVerticalPixelDecimation(int decimator) { _verticalDecimation = decimator; }
void setPixelDecimation(int decimator) { _horizontalDecimation = decimator; _verticalDecimation = decimator;}
void setCropping(int cropLeft, int cropRight, int cropTop, int cropBottom);
void setVideoMode(VideoMode mode) { _videoMode = mode; }
void setFlipMode(FlipMode mode) { _flipMode = mode; }

125
libsrc/flatbufserver/FlatBufferClient.cpp
View File

@ -1,4 +1,5 @@
#include "FlatBufferClient.h"
#include <utils/PixelFormat.h>
// qt
#include <QTcpSocket>
@ -15,6 +16,8 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par
, _timeout(timeout * 1000)
, _priority()
{
_imageResampler.setPixelDecimation(1);
// timer setup
_timeoutTimer->setSingleShot(true);
_timeoutTimer->setInterval(_timeout);
@ -25,6 +28,11 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par
connect(_socket, &QTcpSocket::disconnected, this, &FlatBufferClient::disconnected);
}
void FlatBufferClient::setPixelDecimation(int decimator)
{
_imageResampler.setPixelDecimation(decimator);
}
void FlatBufferClient::readyRead()
{
_timeoutTimer->start();
@ -141,55 +149,71 @@ void FlatBufferClient::handleRegisterCommand(const hyperionnet::Register *regReq
void FlatBufferClient::handleImageCommand(const hyperionnet::Image *image)
{
Image<ColorRgb> imageRGB;
// extract parameters
int duration = image->duration();
const void* reqPtr;
if ((reqPtr = image->data_as_RawImage()) != nullptr)
{
const auto *img = static_cast<const hyperionnet::RawImage*>(reqPtr);
const auto & imageData = img->data();
const int width = img->width();
const int height = img->height();
const auto* img = static_cast<const hyperionnet::RawImage*>(reqPtr);
if (width <= 0 || height <= 0)
hyperionnet::RawImageT rawImageNative;
img->UnPackTo(&rawImageNative);
const int width = rawImageNative.width;
const int height = rawImageNative.height;
if (width <= 0 || height <= 0 || rawImageNative.data.empty())
{
sendErrorReply("Size of image data does not match with the width and height");
sendErrorReply("Invalid width and/or height or no raw image data provided");
return;
}
// check consistency of the size of the received data
int channelCount = (int)imageData->size()/(width*height);
if (channelCount != 3 && channelCount != 4)
int bytesPerPixel = rawImageNative.data.size() / (width * height);
if (bytesPerPixel != 3 && bytesPerPixel != 4)
{
sendErrorReply("Size of image data does not match with the width and height");
return;
}
// create ImageRgb
Image<ColorRgb> imageRGB(width, height);
if (channelCount == 3)
{
memmove(imageRGB.memptr(), imageData->data(), imageData->size());
}
if (channelCount == 4)
{
for (int source=0, destination=0; source < width * height * static_cast<int>(sizeof(ColorRgb)); source+=sizeof(ColorRgb), destination+=sizeof(ColorRgba))
{
memmove((uint8_t*)imageRGB.memptr() + source, imageData->data() + destination, sizeof(ColorRgb));
}
}
emit setGlobalInputImage(_priority, imageRGB, duration);
emit setBufferImage("FlatBuffer", imageRGB);
imageRGB.resize(width, height);
processRawImage(rawImageNative, bytesPerPixel, _imageResampler, imageRGB);
}
else if ((reqPtr = image->data_as_NV12Image()) != nullptr)
{
const auto* img = static_cast<const hyperionnet::NV12Image*>(reqPtr);
hyperionnet::NV12ImageT nv12ImageNative;
img->UnPackTo(&nv12ImageNative);
const int width = nv12ImageNative.width;
const int height = nv12ImageNative.height;
if (width <= 0 || height <= 0 || nv12ImageNative.data_y.empty() || nv12ImageNative.data_uv.empty())
{
sendErrorReply("Invalid width and/or height or no complete NV12 image data provided");
return;
}
imageRGB.resize(width, height);
processNV12Image(nv12ImageNative, _imageResampler, imageRGB);
}
else
{
sendErrorReply("No or unknown image data provided");
return;
}
emit setGlobalInputImage(_priority, imageRGB, duration);
emit setBufferImage("FlatBuffer", imageRGB);
// send reply
sendSuccessReply();
}
void FlatBufferClient::handleClearCommand(const hyperionnet::Clear *clear)
{
// extract parameters
@ -242,3 +266,50 @@ void FlatBufferClient::sendErrorReply(const std::string &error)
_builder.Clear();
}
inline void FlatBufferClient::processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image<ColorRgb>& outputImage) {
int width = raw_image.width;
int height = raw_image.height;
int lineLength = width * bytesPerPixel;
PixelFormat pixelFormat = (bytesPerPixel == 4) ? PixelFormat::RGB32 : PixelFormat::RGB24;
// Process the image
resampler.processImage(
raw_image.data.data(), // Raw RGB/RGBA buffer
width, // Image width
height, // Image height
lineLength, // Line length
pixelFormat, // Pixel format (RGB24/RGB32)
outputImage // Output image
);
}
inline void FlatBufferClient::processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image<ColorRgb>& outputImage) {
// Combine data_y and data_uv into a single buffer
int width = nv12_image.width;
int height = nv12_image.height;
size_t y_size = nv12_image.data_y.size();
size_t uv_size = nv12_image.data_uv.size();
std::vector<uint8_t> combined_buffer(y_size + uv_size);
std::memcpy(combined_buffer.data(), nv12_image.data_y.data(), y_size);
std::memcpy(combined_buffer.data() + y_size, nv12_image.data_uv.data(), uv_size);
// Determine line length (stride_y)
int lineLength = nv12_image.stride_y > 0 ? nv12_image.stride_y : width;
PixelFormat pixelFormat = PixelFormat::NV12;
// Process the image
resampler.processImage(
combined_buffer.data(), // Combined NV12 buffer
width, // Image width
height, // Image height
lineLength, // Line length for Y plane
pixelFormat, // Pixel format (NV12)
outputImage // Output image
);
}

8
libsrc/flatbufserver/FlatBufferClient.h
View File

@ -6,6 +6,7 @@
#include <utils/ColorRgb.h>
#include <utils/ColorRgba.h>
#include <utils/Components.h>
#include "utils/ImageResampler.h"
// flatbuffer FBS
#include "hyperion_reply_generated.h"
@ -33,6 +34,8 @@ public:
///
explicit FlatBufferClient(QTcpSocket* socket, int timeout, QObject *parent = nullptr);
void setPixelDecimation(int decimator);
signals:
///
/// @brief forward register data to HyperionDaemon
@ -138,6 +141,9 @@ private:
///
void sendErrorReply(const std::string & error);
void processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image<ColorRgb>& outputImage);
void processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image<ColorRgb>& outputImage);
private:
Logger *_log;
QTcpSocket *_socket;
@ -148,6 +154,8 @@ private:
QByteArray _receiveBuffer;
ImageResampler _imageResampler;
// Flatbuffers builder
flatbuffers::FlatBufferBuilder _builder;
};

9
libsrc/flatbufserver/FlatBufferServer.cpp
View File

@ -62,6 +62,12 @@ void FlatBufferServer::handleSettingsUpdate(settings::type type, const QJsonDocu
_timeout = obj["timeout"].toInt(5000);
// enable check
obj["enable"].toBool(true) ? startServer() : stopServer();
_pixelDecimation = obj["pixelDecimation"].toInt(1);
for (const auto& client : _openConnections)
{
client->setPixelDecimation(_pixelDecimation);
}
}
}
@ -75,6 +81,9 @@ void FlatBufferServer::newConnection()
{
Debug(_log, "New connection from %s", QSTRING_CSTR(socket->peerAddress().toString()));
FlatBufferClient *client = new FlatBufferClient(socket, _timeout, this);
client->setPixelDecimation(_pixelDecimation);
// internal
connect(client, &FlatBufferClient::clientDisconnected, this, &FlatBufferServer::clientDisconnected);
connect(client, &FlatBufferClient::registerGlobalInput, GlobalSignals::getInstance(), &GlobalSignals::registerGlobalInput);

12
libsrc/flatbufserver/hyperion_request.fbs
View File

@ -12,9 +12,17 @@ table RawImage {
height:int = -1;
}
union ImageType {RawImage}
table NV12Image {
data_y:[ubyte];
data_uv:[ubyte];
width:int;
height:int;
stride_y:int = 0;
stride_uv:int = 0;
}
union ImageType {RawImage, NV12Image}
// Either RGB or RGBA data can be transferred
table Image {
data:ImageType (required);
duration:int = -1;

60
libsrc/hyperion/schema/schema-flatbufServer.json
View File

@ -1,35 +1,41 @@
{
"type" : "object",
"title" : "edt_conf_fbs_heading_title",
"properties" :
{
"enable" :
{
"type" : "boolean",
"required" : true,
"title" : "edt_conf_general_enable_title",
"default" : true,
"propertyOrder" : 1
"properties": {
"enable": {
"type": "boolean",
"required": true,
"title": "edt_conf_general_enable_title",
"default": true,
"propertyOrder": 1
},
"port" :
{
"type" : "integer",
"required" : true,
"title" : "edt_conf_general_port_title",
"minimum" : 1024,
"maximum" : 65535,
"default" : 19400,
"propertyOrder" : 2
"port": {
"type": "integer",
"required": true,
"title": "edt_conf_general_port_title",
"minimum": 1024,
"maximum": 65535,
"default": 19400,
"propertyOrder": 2
},
"timeout" :
{
"type" : "integer",
"required" : true,
"title" : "edt_conf_fbs_timeout_title",
"append" : "edt_append_s",
"minimum" : 1,
"default" : 5,
"propertyOrder" : 3
"timeout": {
"type": "integer",
"required": true,
"title": "edt_conf_fbs_timeout_title",
"append": "edt_append_s",
"minimum": 1,
"default": 5,
"propertyOrder": 3
},
"pixelDecimation": {
"type": "integer",
"title": "edt_conf_fg_pixelDecimation_title",
"minimum": 1,
"maximum": 30,
"default": 1,
"required": false,
"access": "advanced",
"propertyOrder": 4
}
},
"additionalProperties" : false