From 98654e48f6e2ae76d5498ce9c87d830a75280108 Mon Sep 17 00:00:00 2001 From: Markus <16664240+Paulchen-Panther@users.noreply.github.com> Date: Sun, 9 Jan 2022 17:23:50 +0100 Subject: [PATCH] Flatbuffer/Protobuf now able to receive rgba data (#1399) * Flatbuffer/Protobuf now able to receive rgba data * Proto/Flat schema comment added * Prevent diveded by zero * Address LGTM findings * Fix EncoderThread & cleanup Co-authored-by: LordGrey --- README.md | 20 ++++++------ assets/webconfig/css/hyperion.css | 1 - libsrc/flatbufserver/FlatBufferClient.cpp | 29 ++++++++++++++--- libsrc/flatbufserver/FlatBufferClient.h | 1 + libsrc/flatbufserver/hyperion_request.fbs | 1 + libsrc/grabber/video/EncoderThread.cpp | 32 +++++++++++-------- .../video/mediafoundation/MFGrabber.cpp | 4 +++ .../video/mediafoundation/MFSourceReaderCB.h | 14 ++++++++ libsrc/protoserver/ProtoClientConnection.cpp | 26 ++++++++++++--- libsrc/protoserver/ProtoClientConnection.h | 1 + libsrc/protoserver/message.proto | 2 +- 11 files changed, 97 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 6623ac44..82ad2c04 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ ![Hyperion](doc/logo_dark.png#gh-dark-mode-only) ![Hyperion](doc/logo_light.png#gh-light-mode-only) -[![Latest-Release](https://img.shields.io/github/v/release/hyperion-project/hyperion.ng?include_prereleases)](https://github.com/hyperion-project/hyperion.ng/releases) +[![Latest-Release](https://img.shields.io/github/v/release/hyperion-project/hyperion.ng?include_prereleases&label=Latest%20Release&logo=github&logoColor=white&color=0f83e7)](https://github.com/hyperion-project/hyperion.ng/releases) [![GitHub Actions](https://github.com/hyperion-project/hyperion.ng/workflows/Hyperion%20CI%20Build/badge.svg?branch=master)](https://github.com/hyperion-project/hyperion.ng/actions) -[![LGTM](https://img.shields.io/lgtm/alerts/g/hyperion-project/hyperion.ng.svg)](https://lgtm.com/projects/g/hyperion-project/hyperion.ng/alerts/) -[![Forum](https://img.shields.io/website/https/hyperion-project.org.svg?label=Forum&down_color=red&down_message=offline&up_color=green&up_message=online)](https://www.hyperion-project.org) -[![Documentation](https://img.shields.io/website?down_message=offline&label=Documentation%20%20&up_message=online&url=https%3A%2F%2Fdocs.hyperion-project.org)](https://docs.hyperion-project.org) -[![Discord](https://img.shields.io/discord/785578322167463937?label=Discord)](https://discord.gg/khkR8Vx3ff) -![made-with-love](https://img.shields.io/badge/Made%20with-♥-ff0000.svg) +[![LGTM](https://img.shields.io/lgtm/alerts/g/hyperion-project/hyperion.ng.svg?label=LGTM%20Alerts&logo=lgtm&logoColor=white&color=e18200)](https://lgtm.com/projects/g/hyperion-project/hyperion.ng/alerts/) +[![Forum](https://img.shields.io/website/https/hyperion-project.org.svg?label=Forum&down_color=red&down_message=offline&up_color=4bc51d&up_message=online&logo=homeadvisor&logoColor=white)](https://www.hyperion-project.org) +[![Documentation](https://img.shields.io/website/https/docs.hyperion-project.org.svg?label=Documentation&down_color=red&down_message=offline&up_color=4bc51d&up_message=online&logo=read-the-docs)](https://docs.hyperion-project.org) +[![Discord](https://img.shields.io/discord/785578322167463937?label=Discord&logo=discord&logoColor=white&color=4bc51d)](https://discord.gg/khkR8Vx3ff) +![made-with-love](https://img.shields.io/badge/Made%20With-♥-ff0000.svg) ## About Hyperion @@ -26,15 +26,15 @@ * A multi language web interface to configure and remote control hyperion If you need further support please open a topic at the forum!
-[![Forum](https://img.shields.io/website/https/hyperion-project.org.svg?label=Forum&down_color=red&down_message=offline&up_color=green&up_message=online)](https://www.hyperion-project.org) +[![Forum](https://img.shields.io/website/https/hyperion-project.org.svg?label=Forum&down_color=red&down_message=offline&up_color=4bc51d&up_message=online&logo=homeadvisor&logoColor=white)](https://www.hyperion-project.org) ## Contributing Contributions are welcome! Feel free to join us! We are looking always for people who wants to participate.
-[![Contributors](https://img.shields.io/github/contributors/hyperion-project/hyperion.ng.svg?label=Contributors)](https://github.com/hyperion-project/hyperion.ng/graphs/contributors) +[![Contributors](https://img.shields.io/github/contributors/hyperion-project/hyperion.ng.svg?label=Contributors&logo=github&logoColor=white)](https://github.com/hyperion-project/hyperion.ng/graphs/contributors) For an example, you can participate in the translation.
-[![Join Translation](https://img.shields.io/badge/POEditor-translate-green.svg)](https://poeditor.com/join/project/Y4F6vHRFjA) +[![Join Translation](https://img.shields.io/badge/POEditor-4bc51d.svg?label=Join%20Translation)](https://poeditor.com/join/project/Y4F6vHRFjA) ## Supported Platforms @@ -47,7 +47,7 @@ Covers these topics: - [Effect development](https://docs.hyperion-project.org/en/effects/#effect-files) - [JSON API](https://docs.hyperion-project.org/en/json/) -[![Visit Documentation](https://img.shields.io/website?down_message=offline&label=Documentation%20%20&up_message=online&url=https%3A%2F%2Fdocs.hyperion-project.org)](https://docs.hyperion-project.org) +[![Visit Documentation](https://img.shields.io/website/https/docs.hyperion-project.org.svg?label=Documentation&down_color=red&down_message=offline&up_color=4bc51d&up_message=online&logo=read-the-docs)](https://docs.hyperion-project.org) ## Changelog Released and unreleased changes at [CHANGELOG.md](CHANGELOG.md) diff --git a/assets/webconfig/css/hyperion.css b/assets/webconfig/css/hyperion.css index a239a23f..86fc4dd5 100644 --- a/assets/webconfig/css/hyperion.css +++ b/assets/webconfig/css/hyperion.css @@ -30,7 +30,6 @@ body{ padding-right: 10px; } .navbar-brand{ - position:fixed; height: 80px; } #main-nav{ diff --git a/libsrc/flatbufserver/FlatBufferClient.cpp b/libsrc/flatbufserver/FlatBufferClient.cpp index d5d21cdc..1e6acba4 100644 --- a/libsrc/flatbufserver/FlatBufferClient.cpp +++ b/libsrc/flatbufserver/FlatBufferClient.cpp @@ -152,15 +152,36 @@ void FlatBufferClient::handleImageCommand(const hyperionnet::Image *image) const int width = img->width(); const int height = img->height(); - if ((int) imageData->size() != width*height*3) + if (width <= 0 || height <= 0) { sendErrorReply("Size of image data does not match with the width and height"); return; } - Image imageDest(width, height); - memmove(imageDest.memptr(), imageData->data(), imageData->size()); - emit setGlobalInputImage(_priority, imageDest, duration); + // check consistency of the size of the received data + int channelCount = (int)imageData->size()/(width*height); + if (channelCount != 3 && channelCount != 4) + { + sendErrorReply("Size of image data does not match with the width and height"); + return; + } + + // create ImageRgb + Image imageRGB(width, height); + if (channelCount == 3) + { + memmove(imageRGB.memptr(), imageData->data(), imageData->size()); + } + + if (channelCount == 4) + { + for (int source=0, destination=0; source < width * height * static_cast(sizeof(ColorRgb)); source+=sizeof(ColorRgb), destination+=sizeof(ColorRgba)) + { + memmove((uint8_t*)imageRGB.memptr() + source, imageData->data() + destination, sizeof(ColorRgb)); + } + } + + emit setGlobalInputImage(_priority, imageRGB, duration); } // send reply diff --git a/libsrc/flatbufserver/FlatBufferClient.h b/libsrc/flatbufserver/FlatBufferClient.h index 9ac82645..4250559e 100644 --- a/libsrc/flatbufserver/FlatBufferClient.h +++ b/libsrc/flatbufserver/FlatBufferClient.h @@ -4,6 +4,7 @@ #include #include #include +#include #include // flatbuffer FBS diff --git a/libsrc/flatbufserver/hyperion_request.fbs b/libsrc/flatbufserver/hyperion_request.fbs index 1c6e3d49..5a076b65 100644 --- a/libsrc/flatbufserver/hyperion_request.fbs +++ b/libsrc/flatbufserver/hyperion_request.fbs @@ -14,6 +14,7 @@ table RawImage { union ImageType {RawImage} +// Either RGB or RGBA data can be transferred table Image { data:ImageType (required); duration:int = -1; diff --git a/libsrc/grabber/video/EncoderThread.cpp b/libsrc/grabber/video/EncoderThread.cpp index a8098362..943fe693 100644 --- a/libsrc/grabber/video/EncoderThread.cpp +++ b/libsrc/grabber/video/EncoderThread.cpp @@ -22,12 +22,15 @@ EncoderThread::~EncoderThread() tjDestroy(_decompress); #endif - if (_localData) + if (_localData != nullptr) + { #ifdef HAVE_TURBO_JPEG tjFree(_localData); #else delete[] _localData; #endif + _localData = nullptr; + } } void EncoderThread::setup( @@ -55,14 +58,18 @@ void EncoderThread::setup( _imageResampler.setVerticalPixelDecimation(_pixelDecimation); #ifdef HAVE_TURBO_JPEG - if (_localData) + if (_localData != nullptr) tjFree(_localData); _localData = (uint8_t*)tjAlloc(size + 1); #else - delete[] _localData; - _localData = nullptr; - _localData = new uint8_t(size + 1); + if (_localData != nullptr) + { + delete[] _localData; + _localData = nullptr; + } + + _localData = new uint8_t[size]; #endif memcpy(_localData, sharedData, size); @@ -177,23 +184,20 @@ void EncoderThread::processImageMjpeg() else { // calculate the output size - int outputWidth = (_width - _cropLeft - _cropRight); - int outputHeight = (_height - _cropTop - _cropBottom); + int outputWidth = (scaledWidth - _cropLeft - _cropRight); + int outputHeight = (scaledHeight - _cropTop - _cropBottom); if (outputWidth <= 0 || outputHeight <= 0) + { + emit newFrame(srcImage); return; + } Image destImage(outputWidth, outputHeight); for (unsigned int y = 0; y < destImage.height(); y++) { - unsigned char* source = (unsigned char*)srcImage.memptr() + (y + _cropTop)*srcImage.width()*3 + _cropLeft*3; - unsigned char* dest = (unsigned char*)destImage.memptr() + y*destImage.width()*3; - memcpy(dest, source, destImage.width()*3); - free(source); - source = nullptr; - free(dest); - dest = nullptr; + memcpy((unsigned char*)destImage.memptr() + y * destImage.width() * 3, (unsigned char*)srcImage.memptr() + (y + _cropTop) * srcImage.width() * 3 + _cropLeft * 3, destImage.width() * 3); } // emit diff --git a/libsrc/grabber/video/mediafoundation/MFGrabber.cpp b/libsrc/grabber/video/mediafoundation/MFGrabber.cpp index aec004c9..aa017d8d 100644 --- a/libsrc/grabber/video/mediafoundation/MFGrabber.cpp +++ b/libsrc/grabber/video/mediafoundation/MFGrabber.cpp @@ -529,7 +529,11 @@ void MFGrabber::process_image(const void *frameImageBuffer, int size) return; // We do want a new frame... +#ifdef HAVE_TURBO_JPEG if (size < _frameByteSize && _pixelFormat != PixelFormat::MJPEG) +#else + if (size < _frameByteSize) +#endif Error(_log, "Frame too small: %d != %d", size, _frameByteSize); else if (_threadManager != nullptr) { diff --git a/libsrc/grabber/video/mediafoundation/MFSourceReaderCB.h b/libsrc/grabber/video/mediafoundation/MFSourceReaderCB.h index c29dd7ec..8cb10ba4 100644 --- a/libsrc/grabber/video/mediafoundation/MFSourceReaderCB.h +++ b/libsrc/grabber/video/mediafoundation/MFSourceReaderCB.h @@ -30,7 +30,9 @@ static PixelFormat GetPixelFormatForGuid(const GUID guid) if (IsEqualGUID(guid, MFVideoFormat_RGB24)) return PixelFormat::BGR24; if (IsEqualGUID(guid, MFVideoFormat_YUY2)) return PixelFormat::YUYV; if (IsEqualGUID(guid, MFVideoFormat_UYVY)) return PixelFormat::UYVY; +#ifdef HAVE_TURBO_JPEG if (IsEqualGUID(guid, MFVideoFormat_MJPG)) return PixelFormat::MJPEG; +#endif if (IsEqualGUID(guid, MFVideoFormat_NV12)) return PixelFormat::NV12; if (IsEqualGUID(guid, MFVideoFormat_I420)) return PixelFormat::I420; return PixelFormat::NO_CHANGE; @@ -142,7 +144,11 @@ public: goto done; } +#ifdef HAVE_TURBO_JPEG if (_pixelformat != PixelFormat::MJPEG && _pixelformat != PixelFormat::BGR24 && _pixelformat != PixelFormat::NO_CHANGE) +#else + if (_pixelformat != PixelFormat::BGR24 && _pixelformat != PixelFormat::NO_CHANGE) +#endif pSample = TransformSample(_transform, pSample); _hrStatus = pSample->ConvertToContiguousBuffer(&buffer); @@ -174,7 +180,11 @@ public: if (MF_SOURCE_READERF_ENDOFSTREAM & dwStreamFlags) _bEOS = TRUE; // Reached the end of the stream. +#ifdef HAVE_TURBO_JPEG if (_pixelformat != PixelFormat::MJPEG && _pixelformat != PixelFormat::BGR24 && _pixelformat != PixelFormat::NO_CHANGE) +#else + if (_pixelformat != PixelFormat::BGR24 && _pixelformat != PixelFormat::NO_CHANGE) +#endif SAFE_RELEASE(pSample); _isBusy = false; @@ -185,7 +195,11 @@ public: HRESULT InitializeVideoEncoder(IMFMediaType* type, PixelFormat format) { _pixelformat = format; +#ifdef HAVE_TURBO_JPEG if (format == PixelFormat::MJPEG || format == PixelFormat::BGR24 || format == PixelFormat::NO_CHANGE) +#else + if (format == PixelFormat::BGR24 || format == PixelFormat::NO_CHANGE) +#endif return S_OK; // Variable declaration diff --git a/libsrc/protoserver/ProtoClientConnection.cpp b/libsrc/protoserver/ProtoClientConnection.cpp index 051ca3e7..247d4ff1 100644 --- a/libsrc/protoserver/ProtoClientConnection.cpp +++ b/libsrc/protoserver/ProtoClientConnection.cpp @@ -168,18 +168,36 @@ void ProtoClientConnection::handleImageCommand(const proto::ImageRequest &messag _priority = priority; } + if (width <= 0 || height <= 0) + { + sendErrorReply("Size of image data does not match with the width and height"); + return; + } + // check consistency of the size of the received data - if ((int) imageData.size() != width*height*3) + int channelCount = (int)imageData.size()/(width*height); + if (channelCount != 3 && channelCount != 4) { sendErrorReply("Size of image data does not match with the width and height"); return; } // create ImageRgb - Image image(width, height); - memcpy(image.memptr(), imageData.c_str(), imageData.size()); + Image imageRGB(width, height); + if (channelCount == 3) + { + memmove(imageRGB.memptr(), imageData.c_str(), imageData.size()); + } - emit setGlobalInputImage(_priority, image, duration); + if (channelCount == 4) + { + for (int source=0, destination=0; source < width * height * static_cast(sizeof(ColorRgb)); source+=sizeof(ColorRgb), destination+=sizeof(ColorRgba)) + { + memmove((uint8_t*)imageRGB.memptr() + source, imageData.c_str() + destination, sizeof(ColorRgb)); + } + } + + emit setGlobalInputImage(_priority, imageRGB, duration); // send reply sendSuccessReply(); diff --git a/libsrc/protoserver/ProtoClientConnection.h b/libsrc/protoserver/ProtoClientConnection.h index e08658dc..28d0b6b7 100644 --- a/libsrc/protoserver/ProtoClientConnection.h +++ b/libsrc/protoserver/ProtoClientConnection.h @@ -9,6 +9,7 @@ #include #include #include +#include #include class QTcpSocket; diff --git a/libsrc/protoserver/message.proto b/libsrc/protoserver/message.proto index d7593c32..88adba3f 100644 --- a/libsrc/protoserver/message.proto +++ b/libsrc/protoserver/message.proto @@ -45,7 +45,7 @@ message ImageRequest { // height of the image required int32 imageheight = 3; - // image data + // image data (either RGB or RGBA data can be transferred) required bytes imagedata = 4; // duration of the request (negative results in infinite)