Fix xcb deadlock while closing PIP decoder.

Close video decoder from inside the decoder thread,
otherwise xcb hangs in a lock.

Todo

@@ -54,7 +54,7 @@ libva:
     [drm:i915_hangcheck_elapsed] *ERROR* Hangcheck timer elapsed... GPU hung
     [drm:i915_wait_request] *ERROR* i915_wait_request returns -11 ...
     missing OSD support for 3d SBS / Top-Bottom streams, like VPDAU.
-    PIP support
+    PIP support / multistream handling
 
 libva: branch vaapi-ext / staging
     add support for vaapi-ext / staging
@@ -117,16 +117,14 @@ setup:
 unsorted:
     stoping vdr while plugin is suspended opens and closes a window.
     svdrp prim: support plugin names for device numbers.
-    hangup PipVideoStream -> Vdpau_get_format -> xcb -> poll
+    Workaround exists: hangup PipVideoStream -> Vdpau_get_format -> xcb -> poll
 	+ lock DecoderLockMutex
 
 future features (not planed for 1.0 - 1.5)
 
     video out with xv
     video out with opengl
-    video out with xvba
     software decoder for xv / opengl
-    multistream handling
     save and use auto-crop with channel zapping
 
     upmix stereo to AC-3 (supported by alsa plugin)

codec.c

@@ -181,6 +181,9 @@ static int Codec_get_buffer(AVCodecContext * video_ctx, AVFrame * frame)
 	//Debug(3, "codec: use surface %#010x\n", surface);
 
 	frame->type = FF_BUFFER_TYPE_USER;
+#if LIBAVCODEC_VERSION_INT <= AV_VERSION_INT(53,46,0)
+	frame->age = 256 * 256 * 256 * 64;
+#endif
 	// render
 	frame->data[0] = (void *)vrs;
 	frame->data[1] = NULL;
@@ -205,6 +208,9 @@ static int Codec_get_buffer(AVCodecContext * video_ctx, AVFrame * frame)
 	//Debug(3, "codec: use surface %#010x\n", surface);
 
 	frame->type = FF_BUFFER_TYPE_USER;
+#if LIBAVCODEC_VERSION_INT <= AV_VERSION_INT(53,46,0)
+	frame->age = 256 * 256 * 256 * 64;
+#endif
 	// vaapi needs both fields set
 	frame->data[0] = (void *)(size_t) surface;
 	frame->data[3] = (void *)(size_t) surface;

softhddev.c

@@ -1269,7 +1269,8 @@ struct __video_stream__
     volatile char Freezed;		///< stream freezed
 
     volatile char TrickSpeed;		///< current trick speed
-    volatile char ClearBuffers;		///< clear video buffers
+    volatile char Close;		///< command close video stream
+    volatile char ClearBuffers;		///< command clear video buffers
     volatile char ClearClose;		///< clear video buffers for close
 
     AVPacket PacketRb[VIDEO_PACKET_MAX];	///< PES packet ring buffer
@@ -1681,6 +1682,39 @@ static void FixPacketForFFMpeg(VideoDecoder * vdecoder, AVPacket * avpkt)
 
 #endif
 
+/**
+**	Close video stream.
+**
+**	@param stream	video stream
+**
+**	@note must be called from the video thread, othewise xcb has a
+**	deadlock.
+*/
+static void VideoStreamClose(VideoStream * stream)
+{
+    // FIXME: use this function to close the main video stream!
+    stream->SkipStream = 1;
+    if (stream->Decoder) {
+	VideoDecoder *decoder;
+
+	decoder = stream->Decoder;
+	// FIXME: this lock shouldn't be necessary now
+	pthread_mutex_lock(&stream->DecoderLockMutex);
+	stream->Decoder = NULL;		// lock read thread
+	pthread_mutex_unlock(&stream->DecoderLockMutex);
+	CodecVideoClose(decoder);
+	CodecVideoDelDecoder(decoder);
+    }
+    if (stream->HwDecoder) {
+	VideoDelHwDecoder(stream->HwDecoder);
+	stream->HwDecoder = NULL;
+	// FIXME: CodecVideoClose calls/uses hw decoder
+    }
+    VideoPacketExit(stream);
+
+    stream->NewStream = 1;
+}
+
 /**
 **	Poll PES packet ringbuffer.
 **
@@ -1700,6 +1734,11 @@ int VideoPollInput(VideoStream * stream)
 	return -1;
     }
 
+    if (stream->Close) {		// close stream request
+	VideoStreamClose(stream);
+	stream->Close = 0;
+	return 1;
+    }
     if (stream->ClearBuffers) {		// clear buffer request
 	atomic_set(&stream->PacketsFilled, 0);
 	stream->PacketRead = stream->PacketWrite;
@@ -1738,6 +1777,11 @@ int VideoDecodeInput(VideoStream * stream)
 	return -1;
     }
 
+    if (stream->Close) {		// close stream request
+	VideoStreamClose(stream);
+	stream->Close = 0;
+	return 1;
+    }
     if (stream->ClearBuffers) {		// clear buffer request
 	atomic_set(&stream->PacketsFilled, 0);
 	stream->PacketRead = stream->PacketWrite;
@@ -3254,10 +3298,15 @@ void PipStart(int x, int y, int width, int height, int pip_x, int pip_y,
 */
 void PipStop(void)
 {
+    int i;
+
     if (!MyVideoStream->HwDecoder) {	// video not running
 	return;
     }
 
+    ScaleVideo(0, 0, 0, 0);
+
+#if 0
     PipVideoStream->SkipStream = 1;	// lock write thread
     if (PipVideoStream->Decoder) {
 	VideoDecoder *decoder;
@@ -3277,8 +3326,13 @@ void PipStop(void)
     VideoPacketExit(PipVideoStream);
 
     PipVideoStream->NewStream = 1;
-
-    ScaleVideo(0, 0, 0, 0);
+#else
+    PipVideoStream->Close = 1;
+    for (i = 0; PipVideoStream->Close && i < 50; ++i) {
+	usleep(1 * 1000);
+    }
+    Info("[softhddev]%s: pip close %dms\n", __FUNCTION__, i);
+#endif
 }
 
 /**

video.c

@@ -2237,7 +2237,6 @@ static void VaapiExit(void)
     int i;
 
     // FIXME: more VA-API cleanups...
-    // FIXME: can hang with vdpau in pthread_rwlock_wrlock
 
     for (i = 0; i < VaapiDecoderN; ++i) {
 	if (VaapiDecoders[i]) {
@@ -8484,6 +8483,7 @@ static void VdpauDisplayHandlerThread(void)
 	} else {
 	    err = VideoPollInput(decoder->Stream);
 	}
+	// decoder can be invalid here
 	if (err) {
 	    // nothing buffered?
 	    if (err == -1 && decoder->Closing) {
@@ -9589,9 +9589,15 @@ VideoHwDecoder *VideoNewHwDecoder(VideoStream * stream)
 void VideoDelHwDecoder(VideoHwDecoder * hw_decoder)
 {
     if (hw_decoder) {
-	VideoThreadLock();
+#ifdef DEBUG
+	if (!pthread_equal(pthread_self(), VideoThread)) {
+	    Debug(3, "video: should only be called from inside the thread\n");
+	}
+#endif
+	// only called from inside the thread
+	//VideoThreadLock();
 	VideoUsedModule->DelHwDecoder(hw_decoder);
-	VideoThreadUnlock();
+	//VideoThreadUnlock();
     }
 }
 
@@ -10806,7 +10812,7 @@ void VideoInit(const char *display_name)
 	// FIXME: we need to retry connection
 	return;
     }
-    XInitThreads();
+    //XInitThreads();
     // Register error handler
     XSetIOErrorHandler(VideoIOErrorHandler);