Fix v4l2 MJPEG Processing (#1512)

* Fix MJPEG 3D and transform memory leak

* In 3D Crop only half to maintain ratio

* Support turbojpeg v1 and v2

include/grabber/EncoderThread.h

@@ -1,4 +1,5 @@
-#pragma once
+#ifndef ENCODERTHREAD_H
+#define ENCODERTHREAD_H
 
 // Qt includes
 #include <QThread>
@@ -13,20 +14,23 @@
 // Turbo JPEG decoder
 #ifdef HAVE_TURBO_JPEG
 	#include <turbojpeg.h>
+	#include "jconfig.h"
 #endif
 
+constexpr int DEFAULT_THREAD_COUNT {1};
+
 /// Encoder thread for USB devices
 class EncoderThread : public QObject
 {
 	Q_OBJECT
 public:
 	explicit EncoderThread();
-	~EncoderThread();
+	~EncoderThread() override;
 
 	void setup(
 		PixelFormat pixelFormat, uint8_t* sharedData,
 		int size, int width, int height, int lineLength,
-		unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
+		int cropLeft, int cropTop, int cropBottom, int cropRight,
 		VideoMode videoMode, FlipMode flipMode, int pixelDecimation);
 
 	void process();
@@ -39,28 +43,32 @@ signals:
 
 private:
 	PixelFormat _pixelFormat;
-	uint8_t*			_localData,
+	uint8_t* _localData;
-						*_flipBuffer;
+	int	_scalingFactorsCount;
-	int					_scalingFactorsCount,
+	int	_width;
-						_width,
+	int	_height;
-						_height,
+	int	_lineLength;
-						_lineLength,
+	int	_currentFrame;
-						_currentFrame,
+	int	_pixelDecimation;
-						_pixelDecimation;
 	unsigned long _size;
-	unsigned			_cropLeft,
+	int	_cropLeft;
-						_cropTop,
+	int _cropTop;
-						_cropBottom,
+	int _cropBottom;
-						_cropRight;
+	int _cropRight;
+
 	FlipMode _flipMode;
+	VideoMode _videoMode;
+	bool _doTransform;
+
 	ImageResampler		_imageResampler;
 
 #ifdef HAVE_TURBO_JPEG
-	tjhandle			_transform, _decompress;
+	tjhandle			_tjInstance;
 	tjscalingfactor*	_scalingFactors;
 	tjtransform*		_xform;
 
 	void processImageMjpeg();
+	bool onError(const QString context) const;
 #endif
 };
 
@@ -76,7 +84,7 @@ public:
 		start();
 	}
 
-	~Thread()
+	~Thread() override
 	{
 		quit();
 		wait();
@@ -87,7 +95,7 @@ public:
 	void setup(
 		PixelFormat pixelFormat, uint8_t* sharedData,
 		int size, int width, int height, int lineLength,
-		unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
+		int cropLeft, int cropTop, int cropBottom, int cropRight,
 		VideoMode videoMode, FlipMode flipMode, int pixelDecimation)
 	{
 		auto encThread = qobject_cast<EncoderThread*>(_thread);
@@ -128,22 +136,22 @@ class EncoderThreadManager : public QObject
 public:
 	explicit EncoderThreadManager(QObject *parent = nullptr)
 		: QObject(parent)
-		, _threadCount(qMax(QThread::idealThreadCount(), 1))
+		, _threadCount(static_cast<unsigned long>(qMax(QThread::idealThreadCount(), DEFAULT_THREAD_COUNT)))
 		, _threads(nullptr)
 	{
 		_threads = new Thread<EncoderThread>*[_threadCount];
-		for (int i = 0; i < _threadCount; i++)
+		for (unsigned long i = 0; i < _threadCount; i++)
 		{
 			_threads[i] = new Thread<EncoderThread>(new EncoderThread, this);
 			_threads[i]->setObjectName("Encoder " + QString::number(i));
 		}
 	}
 
-	~EncoderThreadManager()
+	~EncoderThreadManager() override
 	{
 		if (_threads != nullptr)
 		{
-			for(int i = 0; i < _threadCount; i++)
+			for(unsigned long  i = 0; i < _threadCount; i++)
 			{
 				_threads[i]->deleteLater();
 				_threads[i] = nullptr;
@@ -157,20 +165,22 @@ public:
 	void start()
 	{
 		if (_threads != nullptr)
-			for (int i = 0; i < _threadCount; i++)
+			for (unsigned long  i = 0; i < _threadCount; i++)
 				connect(_threads[i]->thread(), &EncoderThread::newFrame, this, &EncoderThreadManager::newFrame);
 	}
 
 	void stop()
 	{
 		if (_threads != nullptr)
-			for(int i = 0; i < _threadCount; i++)
+			for(unsigned long  i = 0; i < _threadCount; i++)
 				disconnect(_threads[i]->thread(), nullptr, nullptr, nullptr);
 	}
 
-	int					_threadCount;
+	unsigned long _threadCount;
 	Thread<EncoderThread>**	_threads;
 
 signals:
 	void newFrame(const Image<ColorRgb>& data);
 };
+
+#endif //ENCODERTHREAD_H

libsrc/grabber/video/EncoderThread.cpp

@@ -1,25 +1,28 @@
 #include "grabber/EncoderThread.h"
 
+#include <QDebug>
+
 EncoderThread::EncoderThread()
 	: _localData(nullptr)
 	, _scalingFactorsCount(0)
-	, _imageResampler()
+	, _doTransform(false)
-#ifdef HAVE_TURBO_JPEG
+	,_imageResampler()
-	, _transform(nullptr)
+	#ifdef HAVE_TURBO_JPEG
-	, _decompress(nullptr)
+	, _tjInstance(nullptr)
 	, _scalingFactors(nullptr)
 	, _xform(nullptr)
+	#endif
+{
+#ifdef HAVE_TURBO_JPEG
+	_scalingFactors = tjGetScalingFactors(&_scalingFactorsCount);
 #endif
-{}
+}
 
 EncoderThread::~EncoderThread()
 {
 #ifdef HAVE_TURBO_JPEG
-	if (_transform)
+	if (_tjInstance)
-		tjDestroy(_transform);
+		tjDestroy(_tjInstance);
-
-	if (_decompress)
-		tjDestroy(_decompress);
 #endif
 
 	if (_localData != nullptr)
@@ -36,12 +39,12 @@ EncoderThread::~EncoderThread()
 void EncoderThread::setup(
 		PixelFormat pixelFormat, uint8_t* sharedData,
 		int size, int width, int height, int lineLength,
-	unsigned cropLeft, unsigned cropTop, unsigned cropBottom, unsigned cropRight,
+		int cropLeft, int cropTop, int cropBottom, int cropRight,
 		VideoMode videoMode, FlipMode flipMode, int pixelDecimation)
 {
 	_lineLength = lineLength;
 	_pixelFormat = pixelFormat;
-	_size = (unsigned long) size;
+	_size = static_cast<unsigned long>(size);
 	_width = width;
 	_height = height;
 	_cropLeft = cropLeft;
@@ -49,19 +52,47 @@ void EncoderThread::setup(
 	_cropBottom = cropBottom;
 	_cropRight = cropRight;
 	_flipMode = flipMode;
+	_videoMode = videoMode;
 	_pixelDecimation = pixelDecimation;
 
-	_imageResampler.setVideoMode(videoMode);
+	bool needTransform {false};
+
+	if (_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0 ||
+		_flipMode != FlipMode::NO_CHANGE ||
+		_videoMode !=  VideoMode::VIDEO_2D)
+	{
+		needTransform = true;
+	}
+	else
+	{
+		needTransform = false;
+	}
+
+#ifdef HAVE_TURBO_JPEG
+	if (_doTransform != needTransform )
+	{
+		if (_tjInstance != nullptr)
+		{
+			tjDestroy(_tjInstance);
+			_tjInstance = nullptr;
+		}
+		_doTransform = needTransform;
+	}
+#endif
+
+	_imageResampler.setVideoMode(_videoMode);
 	_imageResampler.setFlipMode(_flipMode);
-	_imageResampler.setCropping(cropLeft, cropRight, cropTop, cropBottom);
+	_imageResampler.setCropping(_cropLeft, _cropRight, _cropTop, _cropBottom);
 	_imageResampler.setHorizontalPixelDecimation(_pixelDecimation);
 	_imageResampler.setVerticalPixelDecimation(_pixelDecimation);
 
 #ifdef HAVE_TURBO_JPEG
 	if (_localData != nullptr)
+	{
 		tjFree(_localData);
-
+		_localData = nullptr;
-	_localData = (uint8_t*)tjAlloc(size + 1);
+	}
+	_localData = static_cast<uint8_t*>(tjAlloc(size + 1));
 #else
 	if (_localData != nullptr)
 	{
@@ -72,7 +103,10 @@ void EncoderThread::setup(
 	_localData = new uint8_t[size];
 #endif
 
-	memcpy(_localData, sharedData, size);
+	if (_localData != nullptr)
+	{
+		memcpy(_localData, sharedData, static_cast<size_t>(size));
+	}
 }
 
 void EncoderThread::process()
@@ -123,85 +157,201 @@ void EncoderThread::process()
 #ifdef HAVE_TURBO_JPEG
 void EncoderThread::processImageMjpeg()
 {
-	if (!_transform && _flipMode != FlipMode::NO_CHANGE)
+	int inSubsamp {0};
+	int inColorspace {0};
+
+	if (_doTransform)
 	{
-		_transform = tjInitTransform();
+		if (!_tjInstance)
+		{
+			_tjInstance = tjInitTransform();
 			_xform = new tjtransform();
 		}
 
-	if (_flipMode == FlipMode::BOTH || _flipMode == FlipMode::HORIZONTAL)
+		if (tjDecompressHeader3(_tjInstance, _localData, _size, &_width, &_height, &inSubsamp, &inColorspace) < 0)
 		{
-		_xform->op = TJXOP_HFLIP;
+			if (onError("_doTransform - tjDecompressHeader3"))
-		tjTransform(_transform, _localData, _size, 1, &_localData, &_size, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE);
-	}
-
-	if (_flipMode == FlipMode::BOTH || _flipMode == FlipMode::VERTICAL)
 			{
-		_xform->op = TJXOP_VFLIP;
-		tjTransform(_transform, _localData, _size, 1, &_localData, &_size, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE);
-	}
-
-	if (!_decompress)
-	{
-		_decompress = tjInitDecompress();
-		_scalingFactors = tjGetScalingFactors(&_scalingFactorsCount);
-	}
-
-	int subsamp = 0;
-	if (tjDecompressHeader2(_decompress, _localData, _size, &_width, &_height, &subsamp) != 0)
 				return;
+			}
+		}
+
+		int transformedWidth {_width};
+		int transformedHeight {_height};
+
+		// handle 3D mode
+		switch (_videoMode)
+		{
+		case VideoMode::VIDEO_3DSBS:
+			transformedWidth = transformedWidth >> 1;
+			_cropLeft = _cropLeft >> 1;
+			_cropRight = _cropRight >> 1;
+			break;
+		case VideoMode::VIDEO_3DTAB:
+			transformedHeight = transformedHeight >> 1;
+			_cropTop = _cropTop >> 1;
+			_cropBottom = _cropBottom >> 1;
+			break;
+		default:
+			break;
+		}
+
+		if (_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0)
+		{
+			int mcuWidth = tjMCUWidth[inSubsamp];
+			int mcuHeight = tjMCUHeight[inSubsamp];
+
+			_cropLeft = _cropLeft - _cropLeft % mcuWidth;
+			_cropTop = _cropTop - _cropTop % mcuHeight;
+
+			int croppedWidth = transformedWidth - _cropLeft - _cropRight;
+			int croppeddHeight = transformedHeight - _cropTop - _cropBottom;
+
+			if (croppedWidth >= 0)
+			{
+				transformedWidth = croppedWidth;
+			}
+
+			if (croppeddHeight >= 0)
+			{
+				transformedHeight = croppeddHeight;
+			}
+
+		}
+
+		if ( transformedWidth != _width ||  transformedHeight != _height  )
+		{
+			_xform->options = TJXOPT_CROP;
+			_xform->r = tjregion {_cropLeft,_cropTop,transformedWidth,transformedHeight};
+			//qDebug() << "processImageMjpeg() | _doTransform - Image cropped: transformedWidth: " << transformedWidth << " transformedHeight: " << transformedHeight;
+		}
+		else
+		{
+			_xform->options = 0;
+			_xform->r = tjregion {0,0,_width,_height};
+			//qDebug() << "processImageMjpeg() | _doTransform - Image not cropped: _width: " << _width << " _height: " << _height;
+		}
+		_xform->options |= TJXOPT_TRIM;
+
+		switch (_flipMode) {
+		case FlipMode::HORIZONTAL:
+			_xform->op = TJXOP_HFLIP;
+			break;
+		case FlipMode::VERTICAL:
+			_xform->op = TJXOP_VFLIP;
+			break;
+		case FlipMode::BOTH:
+			_xform->op = TJXOP_ROT180;
+			break;
+		default:
+			_xform->op = TJXOP_NONE;
+			break;
+		}
+
+		unsigned char *dstBuf = nullptr;  /* Dynamically allocate the JPEG buffer */
+		unsigned long dstSize = 0;
+
+		if(tjTransform(_tjInstance, _localData, _size, 1, &dstBuf, &dstSize, _xform, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) < 0 )
+		{
+			if (onError("_doTransform - tjTransform"))
+			{
+				return;
+			}
+		}
+
+		tjFree(_localData);
+		_localData = dstBuf;
+		_size = dstSize;
+	}
+	else
+	{
+		if (!_tjInstance)
+		{
+			_tjInstance = tjInitDecompress();
+		}
+	}
+
+	if (_doTransform)
+	{
+		if (tjDecompressHeader3(_tjInstance, _localData, _size, &_width, &_height,	&inSubsamp, &inColorspace) < 0)
+		{
+			if (onError("get image details - tjDecompressHeader3"))
+			{
+				return;
+			}
+		}
+	}
+	else
+	{
+		if (tjDecompressHeader2(_tjInstance, _localData, _size, &_width, &_height, &inSubsamp) < 0)
+		{
+			if (onError("get image details - tjDecompressHeader2"))
+			{
+				return;
+			}
+		}
+	}
 
-	int scaledWidth = _width, scaledHeight = _height;
 	if(_scalingFactors != nullptr && _pixelDecimation > 1)
 	{
+		//Scaling factors will not do map 1:1 to pixel decimation, but do a best match
+		//In case perfect pixel decimation is required, code needs to make use of a interim image.
+
+		bool scaleingFactorFound {false};
 		for (int i = 0; i < _scalingFactorsCount ; i++)
 		{
 			const int tempWidth = TJSCALED(_width, _scalingFactors[i]);
 			const int tempHeight = TJSCALED(_height, _scalingFactors[i]);
+
 			if (tempWidth <= _width/_pixelDecimation && tempHeight <= _height/_pixelDecimation)
 			{
-				scaledWidth = tempWidth;
+				_width = tempWidth;
-				scaledHeight = tempHeight;
+				_height = tempHeight;
+				scaleingFactorFound = true;
 				break;
 			}
 		}
 
-		if (scaledWidth == _width && scaledHeight == _height)
+		if (!scaleingFactorFound)
 		{
-			scaledWidth = TJSCALED(_width, _scalingFactors[_scalingFactorsCount-1]);
+			//Set to smallest scaling factor
-			scaledHeight = TJSCALED(_height, _scalingFactors[_scalingFactorsCount-1]);
+			_width = TJSCALED(_width, _scalingFactors[_scalingFactorsCount-1]);
+			_height = TJSCALED(_height, _scalingFactors[_scalingFactorsCount-1]);
 		}
 	}
 
-	Image<ColorRgb> srcImage(scaledWidth, scaledHeight);
+	Image<ColorRgb> srcImage(static_cast<unsigned>(_width), static_cast<unsigned>(_height));
 
-	if (tjDecompress2(_decompress, _localData , _size, (unsigned char*)srcImage.memptr(), scaledWidth, 0, scaledHeight, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0)
+	if (tjDecompress2(_tjInstance, _localData , _size,
-			return;
+					  reinterpret_cast<unsigned char*>(srcImage.memptr()), _width, 0, _height,
-
+					  TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE)
-	// got image, process it
+		< 0)
-	if (!(_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0))
-		emit newFrame(srcImage);
-	else
 	{
-		// calculate the output size
+		if (onError("get final image - tjDecompress2"))
-		int outputWidth = (scaledWidth - _cropLeft - _cropRight);
-		int outputHeight = (scaledHeight - _cropTop - _cropBottom);
-
-		if (outputWidth <= 0 || outputHeight <= 0)
 		{
-			emit newFrame(srcImage);
 			return;
 		}
-
-		Image<ColorRgb> destImage(outputWidth, outputHeight);
-
-		for (unsigned int y = 0; y < destImage.height(); y++)
-		{
-			memcpy((unsigned char*)destImage.memptr() + y * destImage.width() * 3, (unsigned char*)srcImage.memptr() + (y + _cropTop) * srcImage.width() * 3 + _cropLeft * 3, destImage.width() * 3);
-		}
-
-    	// emit
-		emit newFrame(destImage);
 	}
+	emit newFrame(srcImage);
+}
+#endif
+
+#ifdef HAVE_TURBO_JPEG
+bool EncoderThread::onError(const QString context) const
+{
+	bool treatAsError {false};
+
+#if LIBJPEG_TURBO_VERSION_NUMBER > 2000000
+	if (tjGetErrorCode(_tjInstance) == TJERR_FATAL)
+	{
+		//qDebug() << context << "Error: " << QString(tjGetErrorStr2(_tjInstance));
+		treatAsError = true;
+	}
+#else
+	//qDebug() << context << "Error: " << QString(tjGetErrorStr());
+	treatAsError = true;
+#endif
+
+return treatAsError;
 }
 #endif

libsrc/grabber/video/v4l2/V4L2Grabber.cpp

@@ -209,7 +209,7 @@ bool V4L2Grabber::start()
 		{
 			connect(_threadManager, &EncoderThreadManager::newFrame, this, &V4L2Grabber::newThreadFrame);
 			_threadManager->start();
-			DebugIf(verbose, _log, "Decoding threads: %d", _threadManager->_threadCount);
+			DebugIf(verbose, _log, "Decoding threads: %u", _threadManager->_threadCount);
 
 			_streamNotifier->setEnabled(true);
 			start_capturing();
@@ -1294,7 +1294,7 @@ QJsonArray V4L2Grabber::discover(const QJsonObject& params)
 					{
 						std::pair<int, int> width_height{enc.width, enc.height};
 						auto &com = combined[width_height];
-						for (auto framerate : enc.framerates)
+						for (auto framerate : qAsConst(enc.framerates))
 						{
 							com.insert(framerate);
 						}
@@ -1326,7 +1326,7 @@ QJsonArray V4L2Grabber::discover(const QJsonObject& params)
 			device["video_inputs"] = video_inputs;
 
 			QJsonObject controls, controls_default;
-			for (auto control : _deviceControls[device_property.key()])
+			for (const auto &control : qAsConst(_deviceControls[device_property.key()]))
 			{
 				QJsonObject property;
 				property["minValue"] = control.minValue;

libsrc/utils/ImageResampler.cpp

@@ -24,8 +24,11 @@ void ImageResampler::setCropping(int cropLeft, int cropRight, int cropTop, int c
 
 void ImageResampler::processImage(const uint8_t * data, int width, int height, int lineLength, PixelFormat pixelFormat, Image<ColorRgb> &outputImage) const
 {
+	int cropLeft = _cropLeft;
 	int cropRight  = _cropRight;
+	int cropTop = _cropTop;
 	int cropBottom = _cropBottom;
+
 	int xDestFlip = 0, yDestFlip = 0;
 	int uOffset = 0, vOffset = 0;
 
@@ -33,22 +36,24 @@ void ImageResampler::processImage(const uint8_t * data, int width, int height, i
 	switch (_videoMode)
 	{
 	case VideoMode::VIDEO_3DSBS:
-		cropRight = width >> 1;
+		cropRight =  (width >> 1) + (cropRight >> 1);
+		cropLeft = cropLeft >> 1;
 		break;
 	case VideoMode::VIDEO_3DTAB:
-		cropBottom = height >> 1;
+		cropBottom = (height >> 1) + (cropBottom >> 1);
+		cropTop = cropTop >> 1;
 		break;
 	default:
 		break;
 	}
 
 	// calculate the output size
-	int outputWidth = (width - _cropLeft - cropRight - (_horizontalDecimation >> 1) + _horizontalDecimation - 1) / _horizontalDecimation;
+	int outputWidth = (width - cropLeft - cropRight - (_horizontalDecimation >> 1) + _horizontalDecimation - 1) / _horizontalDecimation;
-	int outputHeight = (height - _cropTop - cropBottom - (_verticalDecimation >> 1) + _verticalDecimation - 1) / _verticalDecimation;
+	int outputHeight = (height - cropTop - cropBottom - (_verticalDecimation >> 1) + _verticalDecimation - 1) / _verticalDecimation;
 
 	outputImage.resize(outputWidth, outputHeight);
 
-	for (int yDest = 0, ySource = _cropTop + (_verticalDecimation >> 1); yDest < outputHeight; ySource += _verticalDecimation, ++yDest)
+	for (int yDest = 0, ySource = cropTop + (_verticalDecimation >> 1); yDest < outputHeight; ySource += _verticalDecimation, ++yDest)
 	{
 		int yOffset = lineLength * ySource;
 		if (pixelFormat == PixelFormat::NV12)
@@ -61,7 +66,7 @@ void ImageResampler::processImage(const uint8_t * data, int width, int height, i
 			vOffset = width * height * 1.25 + (ySource/2) * width/2;
 		}
 
-		for (int xDest = 0, xSource = _cropLeft + (_horizontalDecimation >> 1); xDest < outputWidth; xSource += _horizontalDecimation, ++xDest)
+		for (int xDest = 0, xSource = cropLeft + (_horizontalDecimation >> 1); xDest < outputWidth; xSource += _horizontalDecimation, ++xDest)
 		{
 			switch (_flipMode)
 			{