implement webui live video (#340)

* implement webui live video

* add missing german translation

assets/webconfig/content/leds.html

@@ -88,6 +88,7 @@
 					<button  type="button" class="btn btn-success" id="leds_toggle"  data-i18n="conf_leds_test_button_toggleleds">toggle leds</button>
 					<button  type="button" class="btn btn-danger" id="leds_toggle_num"  data-i18n="conf_leds_test_button_togglelednumber">toggle led numbers</button>
 					<button  type="button" class="btn btn-danger" id="leds_toggle_live"  data-i18n="conf_leds_test_button_toggleliveleds">toggle live leds</button>
+					<button  type="button" class="btn btn-danger" id="leds_toggle_live_video"  data-i18n="conf_leds_test_button_togglelivevideo">toggle live video</button>
 				</div>
 			</div>
 		</div>
@@ -296,6 +297,7 @@
 					<div id="previewledcount"></div>
 					<div class="col-lg-12 st_helper" style="padding-left:0px; padding-right:0px">
 						<div id="leds_preview"></div>
+						<div>
 					</div>
 				</div>
 				<div class="panel-footer">

assets/webconfig/i18n/de.json

@@ -115,6 +115,7 @@
 		"conf_leds_test_button_toggleleds" : "LEDs",
 		"conf_leds_test_button_togglelednumber" : "LED Nummerierung",
 		"conf_leds_test_button_toggleliveleds" : "LED Echtzeitansicht",
+		"conf_leds_test_button_togglelivevideo" : "Grabber Echtzeitansicht",
 		"conf_grabber_label_intro" : "Hyperion unterstützt 2 Hauptarten wie Bilder aufgenommen werden können. Zum Einen die Plattform Aufnahme, die sich direkt am System bedient auf dem Hyperion läuft (beste Qualität). Zum Anderen die USB Aufnahme, die sich an einem angeschlossenen Gerät bedient die benötigten Informationen für die Verarbeitung und Ausgabe zu erhalten (Mehr Konfigurationsaufwand und Kalibrierung)",
 		"conf_colors_label_intro" : "Neben der Farbkalibrierung, gehört auch die Glättung (sanfte Farbübergänge) und die Erkennung von störenden (schwarzen) Balken zur Bildverarbeitung.",
 		"conf_network_label_intro" : "Alle Einstellungen zu Ports, der Weiterleitung von JSON/PROTO und Boblight sowie UDP Listener.",

assets/webconfig/i18n/en.json

@@ -115,6 +115,7 @@
 		"conf_leds_test_button_toggleleds" : "toggle leds",
 		"conf_leds_test_button_togglelednumber" : "toggle led numbers",
 		"conf_leds_test_button_toggleliveleds" : "toggle live leds",
+		"conf_leds_test_button_togglelivevideo" : "toggle live video",
 		"conf_grabber_label_intro" : "Hyperion supports two ways on how to get captured pictures for processing and output. The platform capture: internal at the device you are running Hyperion on (best qualitiy) and the USB Capture which gathers from a connected device the necessary pictures (more calibration work and configuration).",
 		"conf_colors_label_intro" : "Color calibration, smoothing (color transistions) and detection of blackbars.",
 		"conf_network_label_intro" : "All network based settings are listed here.",

assets/webconfig/js/content_leds.js

@@ -322,10 +322,12 @@ $(document).ready(function() {
 		createClassicLeds();
 	});
 	
+	// ------------------------------------------------------------------
 	$('.ledMAconstr').bind("change", function() {
 		createMatrixLeds();
 	});
 	
+	// ------------------------------------------------------------------
 	$('#btn_cl_generate').off().on("click", function() {
 		if (finalLedArray != ""){
 			$("#ledconfig").text(JSON.stringify(finalLedArray, null, "\t"));
@@ -334,6 +336,7 @@ $(document).ready(function() {
 		}
 	});
 	
+	// ------------------------------------------------------------------
 	$('#btn_ma_generate').off().on("click", function() {
 		if (finalLedArray != ""){
 			$("#ledconfig").text(JSON.stringify(finalLedArray, null, "\t"));
@@ -342,6 +345,19 @@ $(document).ready(function() {
 		}
 	});
 
+	// ------------------------------------------------------------------
+	$(hyperion).on("cmd-ledcolors-imagestream-update",function(event){
+		if ($("#leddevices").length == 0)
+		{
+			requestLedImageStop();
+		}
+		else
+		{
+			imageData = (event.response.result.image);
+			$("#image_preview").attr("src", imageData);
+		}
+	});
+
 	// ------------------------------------------------------------------
 	$(hyperion).on("cmd-ledcolors-ledstream-update",function(event){
 		if ($("#leddevices").length == 0)
@@ -420,7 +436,7 @@ $(document).ready(function() {
 		canvas_height = $('#leds_canvas').innerHeight();
 		canvas_width = $('#leds_canvas').innerWidth();
 
-		leds_html = "";
+		leds_html = '<img src="" id="image_preview" style="position:relative" />"';
 		for(var idx=0; idx<leds.length; idx++)
 		{
 			led = leds[idx];
@@ -436,6 +452,9 @@ $(document).ready(function() {
 		$('#led_0').css({"z-index":"10"});
 
 		$('#leds_custom_updsim').trigger('click');
+		$('#image_preview').hide();
+		$('#image_preview').attr("width" , $('#leds_canvas').innerWidth()-2);
+		$('#image_preview').attr("height", $('#leds_canvas').innerHeight()-2);
 	});
 
 	// ------------------------------------------------------------------
@@ -463,6 +482,21 @@ $(document).ready(function() {
 		}
 	});
 
+	// ------------------------------------------------------------------
+	$('#leds_toggle_live_video').off().on("click", function() {
+		setClassByBool('#leds_toggle_live_video',imageStreamActive,"btn-success","btn-danger");
+		if ( imageStreamActive )
+		{
+			requestLedImageStop();
+			$('#image_preview').hide();
+		}
+		else
+		{
+			$('#image_preview').show();
+			requestLedImageStart();
+		}
+	});
+
 	// ------------------------------------------------------------------
 	$("#leds_custom_updsim").off().on("click", function() {
 		if (validateText()){

assets/webconfig/js/hyperion.js

@@ -13,11 +13,12 @@ var websocket = null;
 var hyperion = {};
 var wsTan = 1;
 var cronId = 0;
-var ledStreamActive=false;
-var loggingStreamActive=false;
+var ledStreamActive  = false;
+var imageStreamActive = false;
+var loggingStreamActive = false;
 var loggingHandlerInstalled = false;
 var watchdog = 0;
-
+var debugMessagesActive = true;
 //
 function cron()
 {
@@ -168,6 +169,18 @@ function requestLedColorsStop()
 	sendToHyperion("ledcolors", "ledstream-stop");
 }
 
+function requestLedImageStart()
+{
+	imageStreamActive=true;
+	sendToHyperion("ledcolors", "imagestream-start");
+}
+
+function requestLedImageStop()
+{
+	imageStreamActive=false;
+	sendToHyperion("ledcolors", "imagestream-stop");
+}
+
 function requestPriorityClear()
 {
 	sendToHyperion("clear", "", '"priority":1');
@@ -205,6 +218,8 @@ function requestWriteConfig(config)
 	});
 
 	var config_str = JSON.stringify(complete_config);
+	console.log("save");
+	console.log(config_str);
 	sendToHyperion("config","setconfig", '"config":'+config_str);
 }
 

assets/webconfig/js/ui_utils.js

@@ -1,4 +1,12 @@
 
+function debugMessage(msg)
+{
+	if (debugMessagesActive)
+	{
+		console.log(msg);
+	}
+}
+
 function bindNavToContent(containerId, fileName, loadNow)
 {
 	$("#page-content").off();

include/hyperion/Hyperion.h

@@ -202,6 +202,15 @@ public slots:
 	///
 	void setColors(int priority, const std::vector<ColorRgb> &ledColors, const int timeout_ms, bool clearEffects = true, hyperion::Components component=hyperion::COMP_INVALID);
 
+	///
+	/// Writes the given colors to all leds for the given time and priority
+	///
+	/// @param[in] priority The priority of the written colors
+	/// @param[in] ledColors The colors to write to the leds
+	/// @param[in] timeout_ms The time the leds are set to the given colors [ms]
+	///
+	void setImage(int priority, const Image<ColorRgb> & image, int duration_ms);
+
 	///
 	/// Returns the list with unique transform identifiers
 	/// @return The list with transform identifiers
@@ -303,6 +312,8 @@ signals:
 
 	void componentStateChanged(const hyperion::Components component, bool enabled);
 
+	void emitImage(int priority, const Image<ColorRgb> & image, const int timeout_ms);
+
 private slots:
 	///
 	/// Updates the priority muxer with the current time and (re)writes the led color with applied

libsrc/grabber/v4l2/V4L2Wrapper.cpp

@@ -88,10 +88,10 @@ void V4L2Wrapper::set3D(VideoMode mode)
 void V4L2Wrapper::newFrame(const Image<ColorRgb> &image)
 {
 	// forward to other hyperions
-	if ( _forward )
-	{
+	//if ( _forward )
+	//{
 		emit emitImage(_priority, image, _timeout_ms);
-	}
+	//}
 
 	// process the new image
 	_processor->process(image, _ledColors);

libsrc/grabber/x11/X11Wrapper.cpp

@@ -64,10 +64,10 @@ void X11Wrapper::action()
 	// Grab frame into the allocated image
 	_grabber->grabFrame(_image);
 
-	if ( _forward )
-	{
+	//if ( _forward )
+	//{
 		emit emitImage(_priority, _image, _timeout_ms);
-	}
+	//}
 
 	_processor->process(_image, _ledColors);
 	setColors(_ledColors, _timeout_ms);

libsrc/hyperion/GrabberWrapper.cpp

@@ -56,7 +56,6 @@ void GrabberWrapper::componentStateChanged(const hyperion::Components component,
 
 			_forward = _hyperion->getForwarder()->protoForwardingEnabled();
 
-
 			if ( enable == _timer.isActive() )
 			{
 				Info(_log, "grabber change state to %s", (_timer.isActive() ? "enabled" : "disabled") );

libsrc/hyperion/Hyperion.cpp

@@ -739,6 +739,14 @@ void Hyperion::setColors(int priority, const std::vector<ColorRgb>& ledColors, c
 	}
 }
 
+void Hyperion::setImage(int priority, const Image<ColorRgb> & image, int duration_ms)
+{
+	if (priority == getCurrentPriority())
+	{
+		emit emitImage(priority, image, duration_ms);
+	}
+}
+
 const std::vector<std::string> & Hyperion::getTransformIds() const
 {
 	return _raw2ledTransform->getTransformIds();

libsrc/jsonserver/CMakeLists.txt

@@ -33,7 +33,7 @@ add_library(jsonserver
 	${JsonServer_RESOURCES_RCC}
 )
 
-qt5_use_modules(jsonserver Network)
+qt5_use_modules(jsonserver Network Gui)
 
 target_link_libraries(jsonserver
 	hyperion

libsrc/jsonserver/JsonClientConnection.cpp

@@ -22,6 +22,11 @@
 #include <QJsonDocument>
 #include <QVariantMap>
 #include <QDir>
+#include <QImage>
+#include <QBuffer>
+#include <QByteArray>
+#include <QIODevice>
+#include <QDateTime>
 
 // hyperion util includes
 #include <hyperion/ImageProcessorFactory.h>
@@ -50,6 +55,7 @@ JsonClientConnection::JsonClientConnection(QTcpSocket *socket)
 	, _log(Logger::getInstance("JSONCLIENTCONNECTION"))
 	, _forwarder_enabled(true)
 	, _streaming_logging_activated(false)
+	, _image_stream_timeout(0)
 {
 	// connect internal signals and slots
 	connect(_socket, SIGNAL(disconnected()), this, SLOT(socketClosed()));
@@ -58,6 +64,7 @@ JsonClientConnection::JsonClientConnection(QTcpSocket *socket)
 
 	_timer_ledcolors.setSingleShot(false);
 	connect(&_timer_ledcolors, SIGNAL(timeout()), this, SLOT(streamLedcolorsUpdate()));
+	_image_stream_mutex.unlock();
 }
 
 
@@ -1197,22 +1204,32 @@ void JsonClientConnection::handleLedColorsCommand(const QJsonObject& message, co
 {
 	// create result
 	QString subcommand = message["subcommand"].toString("");
-	_streaming_leds_reply["success"] = true;
-	_streaming_leds_reply["command"] = command;
-	_streaming_leds_reply["tan"] = tan;
 
 	if (subcommand == "ledstream-start")
 	{
+		_streaming_leds_reply["success"] = true;
 		_streaming_leds_reply["command"] = command+"-ledstream-update";
+		_streaming_leds_reply["tan"]     = tan;
 		_timer_ledcolors.start(125);
 	}
 	else if (subcommand == "ledstream-stop")
 	{
 		_timer_ledcolors.stop();
 	}
+	else if (subcommand == "imagestream-start")
+	{
+		_streaming_image_reply["success"] = true;
+		_streaming_image_reply["command"] = command+"-imagestream-update";
+		_streaming_image_reply["tan"]     = tan;
+		connect(_hyperion, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), this, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
+	}
+	else if (subcommand == "imagestream-stop")
+	{
+		disconnect(_hyperion, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), this, 0 );
+	}
 	else
 	{
-		sendErrorReply("unknown subcommand",command,tan);
+		sendErrorReply("unknown subcommand \""+subcommand+"\"",command,tan);
 		return;
 	}
 	
@@ -1482,5 +1499,28 @@ void JsonClientConnection::streamLedcolorsUpdate()
 
 	// send the result
 	sendMessage(_streaming_leds_reply);
-
 }
+
+void JsonClientConnection::setImage(int priority, const Image<ColorRgb> & image, int duration_ms)
+{
+	if ( (_image_stream_timeout+250) < QDateTime::currentMSecsSinceEpoch() && _image_stream_mutex.tryLock(0) )
+	{
+		_image_stream_timeout = QDateTime::currentMSecsSinceEpoch();
+
+		QImage jpgImage((const uint8_t *) image.memptr(), image.width(), image.height(), 3*image.width(), QImage::Format_RGB888);
+		QByteArray ba;
+		QBuffer buffer(&ba);
+		buffer.open(QIODevice::WriteOnly);
+		jpgImage.save(&buffer, "jpg");
+		
+		QJsonObject result;
+		result["image"] = "data:image/jpg;base64,"+QString(ba.toBase64());
+		_streaming_image_reply["result"] = result;
+		sendMessage(_streaming_image_reply);
+
+		_image_stream_mutex.unlock();
+	}
+}
+
+
+

libsrc/jsonserver/JsonClientConnection.h

@@ -6,6 +6,7 @@
 // Qt includes
 #include <QByteArray>
 #include <QTcpSocket>
+#include <QMutex>
 
 // Hyperion includes
 #include <hyperion/Hyperion.h>
@@ -116,6 +117,7 @@ public slots:
 	void componentStateChanged(const hyperion::Components component, bool enable);
 	void streamLedcolorsUpdate();
 	void incommingLogMessage(Logger::T_LOG_MESSAGE);
+	void setImage(int priority, const Image<ColorRgb> & image, int duration_ms);
 
 signals:
 	///
@@ -341,9 +343,18 @@ private:
 
 	// streaming buffers
 	QJsonObject _streaming_leds_reply;
+	QJsonObject _streaming_image_reply;
 	QJsonObject _streaming_logging_reply;
+
+	/// flag to determine state of log streaming
 	bool _streaming_logging_activated;
 
+	/// mutex to determine state of image streaming
+	QMutex _image_stream_mutex;
+
+	/// timeout for live video refresh
+	volatile qint64 _image_stream_timeout;
+	
 	// masks for fields in the basic header
 	static uint8_t const BHB0_OPCODE = 0x0F;
 	static uint8_t const BHB0_RSV3 = 0x10;

src/hyperiond/hyperiond.cpp

@@ -456,6 +456,7 @@ void HyperionDaemon::createGrabberDispmanx()
 	QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), _dispmanx, SLOT(setGrabbingMode(GrabbingMode)));
 	QObject::connect(_kodiVideoChecker, SIGNAL(videoMode(VideoMode)), _dispmanx, SLOT(setVideoMode(VideoMode)));
 	QObject::connect(_dispmanx, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)) );
+	QObject::connect(_dispmanx, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
 
 	_dispmanx->start();
 
@@ -474,6 +475,7 @@ void HyperionDaemon::createGrabberAmlogic()
 	QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), _amlGrabber, SLOT(setGrabbingMode(GrabbingMode)));
 	QObject::connect(_kodiVideoChecker, SIGNAL(videoMode(VideoMode)),       _amlGrabber, SLOT(setVideoMode(VideoMode)));
 	QObject::connect(_amlGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)) );
+	QObject::connect(_amlGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
 
 	_amlGrabber->start();
 	Info(_log, "AMLOGIC grabber created and started");
@@ -495,6 +497,7 @@ void HyperionDaemon::createGrabberX11(const QJsonObject & grabberConfig)
 	QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), _x11Grabber, SLOT(setGrabbingMode(GrabbingMode)));
 	QObject::connect(_kodiVideoChecker, SIGNAL(videoMode(VideoMode)),       _x11Grabber, SLOT(setVideoMode(VideoMode)));
 	QObject::connect(_x11Grabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)) );
+	QObject::connect(_x11Grabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
 
 	_x11Grabber->start();
 	Info(_log, "X11 grabber created and started");
@@ -515,6 +518,7 @@ void HyperionDaemon::createGrabberFramebuffer(const QJsonObject & grabberConfig)
 	QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), _fbGrabber, SLOT(setGrabbingMode(GrabbingMode)));
 	QObject::connect(_kodiVideoChecker, SIGNAL(videoMode(VideoMode)), _fbGrabber, SLOT(setVideoMode(VideoMode)));
 	QObject::connect(_fbGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)) );
+	QObject::connect(_fbGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
 
 	_fbGrabber->start();
 	Info(_log, "Framebuffer grabber created and started");
@@ -535,6 +539,7 @@ void HyperionDaemon::createGrabberOsx(const QJsonObject & grabberConfig)
 	QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), _osxGrabber, SLOT(setGrabbingMode(GrabbingMode)));
 	QObject::connect(_kodiVideoChecker, SIGNAL(videoMode(VideoMode)), _osxGrabber, SLOT(setVideoMode(VideoMode)));
 	QObject::connect(_osxGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)) );
+	QObject::connect(_osxGrabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)) );
 
 	_osxGrabber->start();
 	Info(_log, "OSX grabber created and started");
@@ -590,6 +595,7 @@ void HyperionDaemon::createGrabberV4L2()
 			Debug(_log, "V4L2 grabber created");
 
 			QObject::connect(grabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _protoServer, SLOT(sendImageToProtoSlaves(int, const Image<ColorRgb>&, const int)));
+			QObject::connect(grabber, SIGNAL(emitImage(int, const Image<ColorRgb>&, const int)), _hyperion, SLOT(setImage(int, const Image<ColorRgb>&, const int)));
 			if (grabberConfig["useKodiChecker"].toBool(false))
 			{
 				QObject::connect(_kodiVideoChecker, SIGNAL(grabbingMode(GrabbingMode)), grabber, SLOT(setGrabbingMode(GrabbingMode)));