Fix v4l2 MJPEG Processing (#1512)

* Fix MJPEG 3D and transform memory leak

* In 3D Crop only half to maintain ratio

* Support turbojpeg v1 and v2
This commit is contained in:
LordGrey 2022-09-27 08:57:19 +02:00 committed by GitHub
parent 9d23d9e2d6
commit cb76afa4d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 275 additions and 110 deletions

View File

@ -1,4 +1,5 @@
#pragma once
#ifndef ENCODERTHREAD_H
#define ENCODERTHREAD_H
// Qt includes
#include <QThread>
@ -13,20 +14,23 @@
// Turbo JPEG decoder
#ifdef HAVE_TURBO_JPEG
#include <turbojpeg.h>
#include "jconfig.h"
#endif
constexpr int DEFAULT_THREAD_COUNT {1};
/// Encoder thread for USB devices
class EncoderThread : public QObject
{
Q_OBJECT
public:
explicit EncoderThread();
~EncoderThread();
~EncoderThread() override;
void setup(
PixelFormat pixelFormat, uint8_t* sharedData,
int size, int width, int height, int lineLength,
unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
int cropLeft, int cropTop, int cropBottom, int cropRight,
VideoMode videoMode, FlipMode flipMode, int pixelDecimation);
void process();
@ -38,29 +42,33 @@ signals:
void newFrame(const Image<ColorRgb>& data);
private:
PixelFormat _pixelFormat;
uint8_t* _localData,
*_flipBuffer;
int _scalingFactorsCount,
_width,
_height,
_lineLength,
_currentFrame,
_pixelDecimation;
unsigned long _size;
unsigned _cropLeft,
_cropTop,
_cropBottom,
_cropRight;
FlipMode _flipMode;
PixelFormat _pixelFormat;
uint8_t* _localData;
int _scalingFactorsCount;
int _width;
int _height;
int _lineLength;
int _currentFrame;
int _pixelDecimation;
unsigned long _size;
int _cropLeft;
int _cropTop;
int _cropBottom;
int _cropRight;
FlipMode _flipMode;
VideoMode _videoMode;
bool _doTransform;
ImageResampler _imageResampler;
#ifdef HAVE_TURBO_JPEG
tjhandle _transform, _decompress;
tjhandle _tjInstance;
tjscalingfactor* _scalingFactors;
tjtransform* _xform;
void processImageMjpeg();
bool onError(const QString context) const;
#endif
};
@ -76,7 +84,7 @@ public:
start();
}
~Thread()
~Thread() override
{
quit();
wait();
@ -87,7 +95,7 @@ public:
void setup(
PixelFormat pixelFormat, uint8_t* sharedData,
int size, int width, int height, int lineLength,
unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
int cropLeft, int cropTop, int cropBottom, int cropRight,
VideoMode videoMode, FlipMode flipMode, int pixelDecimation)
{
auto encThread = qobject_cast<EncoderThread*>(_thread);
@ -128,22 +136,22 @@ class EncoderThreadManager : public QObject
public:
explicit EncoderThreadManager(QObject *parent = nullptr)
: QObject(parent)
, _threadCount(qMax(QThread::idealThreadCount(), 1))
, _threadCount(static_cast<unsigned long>(qMax(QThread::idealThreadCount(), DEFAULT_THREAD_COUNT)))
, _threads(nullptr)
{
_threads = new Thread<EncoderThread>*[_threadCount];
for (int i = 0; i < _threadCount; i++)
for (unsigned long i = 0; i < _threadCount; i++)
{
_threads[i] = new Thread<EncoderThread>(new EncoderThread, this);
_threads[i]->setObjectName("Encoder " + QString::number(i));
}
}
~EncoderThreadManager()
~EncoderThreadManager() override
{
if (_threads != nullptr)
{
for(int i = 0; i < _threadCount; i++)
for(unsigned long i = 0; i < _threadCount; i++)
{
_threads[i]->deleteLater();
_threads[i] = nullptr;
@ -157,20 +165,22 @@ public:
void start()
{
if (_threads != nullptr)
for (int i = 0; i < _threadCount; i++)
for (unsigned long i = 0; i < _threadCount; i++)
connect(_threads[i]->thread(), &EncoderThread::newFrame, this, &EncoderThreadManager::newFrame);
}
void stop()
{
if (_threads != nullptr)
for(int i = 0; i < _threadCount; i++)
for(unsigned long i = 0; i < _threadCount; i++)
disconnect(_threads[i]->thread(), nullptr, nullptr, nullptr);
}
int _threadCount;
unsigned long _threadCount;
Thread<EncoderThread>** _threads;
signals:
void newFrame(const Image<ColorRgb>& data);
};
#endif //ENCODERTHREAD_H

View File

@ -1,25 +1,28 @@
#include "grabber/EncoderThread.h"
#include <QDebug>
EncoderThread::EncoderThread()
: _localData(nullptr)
, _scalingFactorsCount(0)
, _imageResampler()
#ifdef HAVE_TURBO_JPEG
, _transform(nullptr)
, _decompress(nullptr)
, _doTransform(false)
,_imageResampler()
#ifdef HAVE_TURBO_JPEG
, _tjInstance(nullptr)
, _scalingFactors(nullptr)
, _xform(nullptr)
#endif
{
#ifdef HAVE_TURBO_JPEG
_scalingFactors = tjGetScalingFactors(&_scalingFactorsCount);
#endif
{}
}
EncoderThread::~EncoderThread()
{
#ifdef HAVE_TURBO_JPEG
if (_transform)
tjDestroy(_transform);
if (_decompress)
tjDestroy(_decompress);
if (_tjInstance)
tjDestroy(_tjInstance);
#endif
if (_localData != nullptr)
@ -34,14 +37,14 @@ EncoderThread::~EncoderThread()
}
void EncoderThread::setup(
PixelFormat pixelFormat, uint8_t* sharedData,
int size, int width, int height, int lineLength,
unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
VideoMode videoMode, FlipMode flipMode, int pixelDecimation)
PixelFormat pixelFormat, uint8_t* sharedData,
int size, int width, int height, int lineLength,
int cropLeft, int cropTop, int cropBottom, int cropRight,
VideoMode videoMode, FlipMode flipMode, int pixelDecimation)
{
_lineLength = lineLength;
_pixelFormat = pixelFormat;
_size = (unsigned long) size;
_size = static_cast<unsigned long>(size);
_width = width;
_height = height;
_cropLeft = cropLeft;
@ -49,19 +52,47 @@ void EncoderThread::setup(
_cropBottom = cropBottom;
_cropRight = cropRight;
_flipMode = flipMode;
_videoMode = videoMode;
_pixelDecimation = pixelDecimation;
_imageResampler.setVideoMode(videoMode);
bool needTransform {false};
if (_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0 ||
_flipMode != FlipMode::NO_CHANGE ||
_videoMode != VideoMode::VIDEO_2D)
{
needTransform = true;
}
else
{
needTransform = false;
}
#ifdef HAVE_TURBO_JPEG
if (_doTransform != needTransform )
{
if (_tjInstance != nullptr)
{
tjDestroy(_tjInstance);
_tjInstance = nullptr;
}
_doTransform = needTransform;
}
#endif
_imageResampler.setVideoMode(_videoMode);
_imageResampler.setFlipMode(_flipMode);
_imageResampler.setCropping(cropLeft, cropRight, cropTop, cropBottom);
_imageResampler.setCropping(_cropLeft, _cropRight, _cropTop, _cropBottom);
_imageResampler.setHorizontalPixelDecimation(_pixelDecimation);
_imageResampler.setVerticalPixelDecimation(_pixelDecimation);
#ifdef HAVE_TURBO_JPEG
if (_localData != nullptr)
{
tjFree(_localData);
_localData = (uint8_t*)tjAlloc(size + 1);
_localData = nullptr;
}
_localData = static_cast<uint8_t*>(tjAlloc(size + 1));
#else
if (_localData != nullptr)
{
@ -72,7 +103,10 @@ void EncoderThread::setup(
_localData = new uint8_t[size];
#endif
memcpy(_localData, sharedData, size);
if (_localData != nullptr)
{
memcpy(_localData, sharedData, static_cast<size_t>(size));
}
}
void EncoderThread::process()
@ -123,85 +157,201 @@ void EncoderThread::process()
#ifdef HAVE_TURBO_JPEG
void EncoderThread::processImageMjpeg()
{
if (!_transform && _flipMode != FlipMode::NO_CHANGE)
int inSubsamp {0};
int inColorspace {0};
if (_doTransform)
{
_transform = tjInitTransform();
_xform = new tjtransform();
if (!_tjInstance)
{
_tjInstance = tjInitTransform();
_xform = new tjtransform();
}
if (tjDecompressHeader3(_tjInstance, _localData, _size, &_width, &_height, &inSubsamp, &inColorspace) < 0)
{
if (onError("_doTransform - tjDecompressHeader3"))
{
return;
}
}
int transformedWidth {_width};
int transformedHeight {_height};
// handle 3D mode
switch (_videoMode)
{
case VideoMode::VIDEO_3DSBS:
transformedWidth = transformedWidth >> 1;
_cropLeft = _cropLeft >> 1;
_cropRight = _cropRight >> 1;
break;
case VideoMode::VIDEO_3DTAB:
transformedHeight = transformedHeight >> 1;
_cropTop = _cropTop >> 1;
_cropBottom = _cropBottom >> 1;
break;
default:
break;
}
if (_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0)
{
int mcuWidth = tjMCUWidth[inSubsamp];
int mcuHeight = tjMCUHeight[inSubsamp];
_cropLeft = _cropLeft - _cropLeft % mcuWidth;
_cropTop = _cropTop - _cropTop % mcuHeight;
int croppedWidth = transformedWidth - _cropLeft - _cropRight;
int croppeddHeight = transformedHeight - _cropTop - _cropBottom;
if (croppedWidth >= 0)
{
transformedWidth = croppedWidth;
}
if (croppeddHeight >= 0)
{
transformedHeight = croppeddHeight;
}
}
if ( transformedWidth != _width || transformedHeight != _height )
{
_xform->options = TJXOPT_CROP;
_xform->r = tjregion {_cropLeft,_cropTop,transformedWidth,transformedHeight};
//qDebug() << "processImageMjpeg() | _doTransform - Image cropped: transformedWidth: " << transformedWidth << " transformedHeight: " << transformedHeight;
}
else
{
_xform->options = 0;
_xform->r = tjregion {0,0,_width,_height};
//qDebug() << "processImageMjpeg() | _doTransform - Image not cropped: _width: " << _width << " _height: " << _height;
}
_xform->options |= TJXOPT_TRIM;
switch (_flipMode) {
case FlipMode::HORIZONTAL:
_xform->op = TJXOP_HFLIP;
break;
case FlipMode::VERTICAL:
_xform->op = TJXOP_VFLIP;
break;
case FlipMode::BOTH:
_xform->op = TJXOP_ROT180;
break;
default:
_xform->op = TJXOP_NONE;
break;
}
unsigned char *dstBuf = nullptr; /* Dynamically allocate the JPEG buffer */
unsigned long dstSize = 0;
if(tjTransform(_tjInstance, _localData, _size, 1, &dstBuf, &dstSize, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) < 0 )
{
if (onError("_doTransform - tjTransform"))
{
return;
}
}
tjFree(_localData);
_localData = dstBuf;
_size = dstSize;
}
else
{
if (!_tjInstance)
{
_tjInstance = tjInitDecompress();
}
}
if (_flipMode == FlipMode::BOTH || _flipMode == FlipMode::HORIZONTAL)
if (_doTransform)
{
_xform->op = TJXOP_HFLIP;
tjTransform(_transform, _localData, _size, 1, &_localData, &_size, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE);
if (tjDecompressHeader3(_tjInstance, _localData, _size, &_width, &_height, &inSubsamp, &inColorspace) < 0)
{
if (onError("get image details - tjDecompressHeader3"))
{
return;
}
}
}
else
{
if (tjDecompressHeader2(_tjInstance, _localData, _size, &_width, &_height, &inSubsamp) < 0)
{
if (onError("get image details - tjDecompressHeader2"))
{
return;
}
}
}
if (_flipMode == FlipMode::BOTH || _flipMode == FlipMode::VERTICAL)
{
_xform->op = TJXOP_VFLIP;
tjTransform(_transform, _localData, _size, 1, &_localData, &_size, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE);
}
if (!_decompress)
{
_decompress = tjInitDecompress();
_scalingFactors = tjGetScalingFactors(&_scalingFactorsCount);
}
int subsamp = 0;
if (tjDecompressHeader2(_decompress, _localData, _size, &_width, &_height, &subsamp) != 0)
return;
int scaledWidth = _width, scaledHeight = _height;
if(_scalingFactors != nullptr && _pixelDecimation > 1)
{
//Scaling factors will not do map 1:1 to pixel decimation, but do a best match
//In case perfect pixel decimation is required, code needs to make use of a interim image.
bool scaleingFactorFound {false};
for (int i = 0; i < _scalingFactorsCount ; i++)
{
const int tempWidth = TJSCALED(_width, _scalingFactors[i]);
const int tempHeight = TJSCALED(_height, _scalingFactors[i]);
if (tempWidth <= _width/_pixelDecimation && tempHeight <= _height/_pixelDecimation)
{
scaledWidth = tempWidth;
scaledHeight = tempHeight;
_width = tempWidth;
_height = tempHeight;
scaleingFactorFound = true;
break;
}
}
if (scaledWidth == _width && scaledHeight == _height)
if (!scaleingFactorFound)
{
scaledWidth = TJSCALED(_width, _scalingFactors[_scalingFactorsCount-1]);
scaledHeight = TJSCALED(_height, _scalingFactors[_scalingFactorsCount-1]);
//Set to smallest scaling factor
_width = TJSCALED(_width, _scalingFactors[_scalingFactorsCount-1]);
_height = TJSCALED(_height, _scalingFactors[_scalingFactorsCount-1]);
}
}
Image<ColorRgb> srcImage(scaledWidth, scaledHeight);
Image<ColorRgb> srcImage(static_cast<unsigned>(_width), static_cast<unsigned>(_height));
if (tjDecompress2(_decompress, _localData , _size, (unsigned char*)srcImage.memptr(), scaledWidth, 0, scaledHeight, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0)
return;
// got image, process it
if (!(_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0))
emit newFrame(srcImage);
else
{
// calculate the output size
int outputWidth = (scaledWidth - _cropLeft - _cropRight);
int outputHeight = (scaledHeight - _cropTop - _cropBottom);
if (outputWidth <= 0 || outputHeight <= 0)
if (tjDecompress2(_tjInstance, _localData , _size,
reinterpret_cast<unsigned char*>(srcImage.memptr()), _width, 0, _height,
TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE)
< 0)
{
if (onError("get final image - tjDecompress2"))
{
emit newFrame(srcImage);
return;
}
Image<ColorRgb> destImage(outputWidth, outputHeight);
for (unsigned int y = 0; y < destImage.height(); y++)
{
memcpy((unsigned char*)destImage.memptr() + y * destImage.width() * 3, (unsigned char*)srcImage.memptr() + (y + _cropTop) * srcImage.width() * 3 + _cropLeft * 3, destImage.width() * 3);
}
// emit
emit newFrame(destImage);
}
emit newFrame(srcImage);
}
#endif
#ifdef HAVE_TURBO_JPEG
bool EncoderThread::onError(const QString context) const
{
bool treatAsError {false};
#if LIBJPEG_TURBO_VERSION_NUMBER > 2000000
if (tjGetErrorCode(_tjInstance) == TJERR_FATAL)
{
//qDebug() << context << "Error: " << QString(tjGetErrorStr2(_tjInstance));
treatAsError = true;
}
#else
//qDebug() << context << "Error: " << QString(tjGetErrorStr());
treatAsError = true;
#endif
return treatAsError;
}
#endif

View File

@ -209,7 +209,7 @@ bool V4L2Grabber::start()
{
connect(_threadManager, &EncoderThreadManager::newFrame, this, &V4L2Grabber::newThreadFrame);
_threadManager->start();
DebugIf(verbose, _log, "Decoding threads: %d", _threadManager->_threadCount);
DebugIf(verbose, _log, "Decoding threads: %u", _threadManager->_threadCount);
_streamNotifier->setEnabled(true);
start_capturing();
@ -1294,7 +1294,7 @@ QJsonArray V4L2Grabber::discover(const QJsonObject& params)
{
std::pair<int, int> width_height{enc.width, enc.height};
auto &com = combined[width_height];
for (auto framerate : enc.framerates)
for (auto framerate : qAsConst(enc.framerates))
{
com.insert(framerate);
}
@ -1326,7 +1326,7 @@ QJsonArray V4L2Grabber::discover(const QJsonObject& params)
device["video_inputs"] = video_inputs;
QJsonObject controls, controls_default;
for (auto control : _deviceControls[device_property.key()])
for (const auto &control : qAsConst(_deviceControls[device_property.key()]))
{
QJsonObject property;
property["minValue"] = control.minValue;

View File

@ -24,8 +24,11 @@ void ImageResampler::setCropping(int cropLeft, int cropRight, int cropTop, int c
void ImageResampler::processImage(const uint8_t * data, int width, int height, int lineLength, PixelFormat pixelFormat, Image<ColorRgb> &outputImage) const
{
int cropLeft = _cropLeft;
int cropRight = _cropRight;
int cropTop = _cropTop;
int cropBottom = _cropBottom;
int xDestFlip = 0, yDestFlip = 0;
int uOffset = 0, vOffset = 0;
@ -33,22 +36,24 @@ void ImageResampler::processImage(const uint8_t * data, int width, int height, i
switch (_videoMode)
{
case VideoMode::VIDEO_3DSBS:
cropRight = width >> 1;
cropRight = (width >> 1) + (cropRight >> 1);
cropLeft = cropLeft >> 1;
break;
case VideoMode::VIDEO_3DTAB:
cropBottom = height >> 1;
cropBottom = (height >> 1) + (cropBottom >> 1);
cropTop = cropTop >> 1;
break;
default:
break;
}
// calculate the output size
int outputWidth = (width - _cropLeft - cropRight - (_horizontalDecimation >> 1) + _horizontalDecimation - 1) / _horizontalDecimation;
int outputHeight = (height - _cropTop - cropBottom - (_verticalDecimation >> 1) + _verticalDecimation - 1) / _verticalDecimation;
int outputWidth = (width - cropLeft - cropRight - (_horizontalDecimation >> 1) + _horizontalDecimation - 1) / _horizontalDecimation;
int outputHeight = (height - cropTop - cropBottom - (_verticalDecimation >> 1) + _verticalDecimation - 1) / _verticalDecimation;
outputImage.resize(outputWidth, outputHeight);
for (int yDest = 0, ySource = _cropTop + (_verticalDecimation >> 1); yDest < outputHeight; ySource += _verticalDecimation, ++yDest)
for (int yDest = 0, ySource = cropTop + (_verticalDecimation >> 1); yDest < outputHeight; ySource += _verticalDecimation, ++yDest)
{
int yOffset = lineLength * ySource;
if (pixelFormat == PixelFormat::NV12)
@ -61,7 +66,7 @@ void ImageResampler::processImage(const uint8_t * data, int width, int height, i
vOffset = width * height * 1.25 + (ySource/2) * width/2;
}
for (int xDest = 0, xSource = _cropLeft + (_horizontalDecimation >> 1); xDest < outputWidth; xSource += _horizontalDecimation, ++xDest)
for (int xDest = 0, xSource = cropLeft + (_horizontalDecimation >> 1); xDest < outputWidth; xSource += _horizontalDecimation, ++xDest)
{
switch (_flipMode)
{