Merge pull request #8266 from andrewvoznytsa/PR-qmlglsink-v2

Switch from builtin video sink to qmlglsink (v2)

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "libs/OpenSSL/android_openssl"]
 	path = libs/OpenSSL/android_openssl
 	url = https://github.com/Auterion/android_openssl
+[submodule "libs/gst-plugins-good"]
+	path = libs/gst-plugins-good
+	url = https://gitlab.freedesktop.org/andrew.voznytsa/gst-plugins-good.git

.travis.yml

@@ -16,12 +16,12 @@ matrix:
     include:
         - name: "Linux Installer"
           os: linux
-          dist: xenial
+          dist: bionic
           env: SPEC=linux-g++-64 CONFIG=installer
           sudo: required
         - name: "Linux Debug"
           os: linux
-          dist: xenial
+          dist: bionic
           env: SPEC=linux-g++-64 CONFIG=debug
           services: xvfb
           sudo: required

QGCInstaller.pri

@@ -45,7 +45,7 @@ installer {
     LinuxBuild {
         #-- TODO: This uses hardcoded paths. It should use $${DESTDIR}
         QMAKE_POST_LINK += && mkdir -p release/package
-        QMAKE_POST_LINK += && tar --warning=no-file-changed -cjf release/package/QGroundControl.tar.bz2 release --exclude='package' --transform 's/release/qgroundcontrol/'
+        QMAKE_POST_LINK += && tar -cj --exclude='package' -f release/package/QGroundControl.tar.bz2 release --transform 's/release/qgroundcontrol/'
     }
     AndroidBuild {
         QMAKE_POST_LINK += && mkdir -p $${DESTDIR}/package

gst-plugins-good @ 9d782fad

+Subproject commit 9d782fad9dc0384ba86ecae64511c193f6149f93

qgroundcontrol.pro

@@ -1374,18 +1374,13 @@ INCLUDEPATH += \
     src/VideoStreaming
 
 HEADERS += \
-    src/VideoStreaming/VideoItem.h \
     src/VideoStreaming/VideoReceiver.h \
     src/VideoStreaming/VideoStreaming.h \
-    src/VideoStreaming/VideoSurface.h \
-    src/VideoStreaming/VideoSurface_p.h \
     src/VideoStreaming/SubtitleWriter.h \
 
 SOURCES += \
-    src/VideoStreaming/VideoItem.cc \
     src/VideoStreaming/VideoReceiver.cc \
     src/VideoStreaming/VideoStreaming.cc \
-    src/VideoStreaming/VideoSurface.cc \
     src/VideoStreaming/SubtitleWriter.cc \
 
 contains (CONFIG, DISABLE_VIDEOSTREAMING) {

qmlglsink.pri

+# FIXME: will be fixed after converting qmlglsink into .pro#static lib
+QMAKE_CXXFLAGS_WARN_ON =
+
+LinuxBuild {
+    DEFINES += HAVE_QT_X11 HAVE_QT_EGLFS HAVE_QT_WAYLAND
+} else:MacBuild {
+    DEFINES += HAVE_QT_MAC
+} else:iOSBuild {
+    DEFINES += HAVE_QT_IOS
+} else:WindowsBuild {
+    DEFINES += HAVE_QT_WIN32
+    LIBS += opengl32.lib user32.lib
+} else:AndroidBuild {
+    DEFINES += HAVE_QT_ANDROID
+}
+
+SOURCES += \
+    libs/gst-plugins-good/ext/qt/gstplugin.cc \
+    libs/gst-plugins-good/ext/qt/gstqtglutility.cc \
+    libs/gst-plugins-good/ext/qt/gstqsgtexture.cc \
+    libs/gst-plugins-good/ext/qt/gstqtsink.cc \
+    libs/gst-plugins-good/ext/qt/gstqtsrc.cc \
+    libs/gst-plugins-good/ext/qt/qtwindow.cc \
+    libs/gst-plugins-good/ext/qt/qtitem.cc
+
+HEADERS += \
+    libs/gst-plugins-good/ext/qt/gstqsgtexture.h \
+    libs/gst-plugins-good/ext/qt/gstqtgl.h \
+    libs/gst-plugins-good/ext/qt/gstqtglutility.h \
+    libs/gst-plugins-good/ext/qt/gstqtsink.h \
+    libs/gst-plugins-good/ext/qt/gstqtsrc.h \
+    libs/gst-plugins-good/ext/qt/qtwindow.h \
+    libs/gst-plugins-good/ext/qt/qtitem.h

src/FlightDisplay/FlightDisplayViewVideo.qml

@@ -79,8 +79,8 @@ Item {
             id: videoBackgroundComponent
             QGCVideoBackground {
                 id:             videoContent
+                objectName:     "videoContent"
                 receiver:       _videoReceiver
-                display:        _videoReceiver && _videoReceiver.videoSurface
 
                 Connections {
                     target:         _videoReceiver
@@ -125,13 +125,13 @@ Item {
         Loader {
             // GStreamer is causing crashes on Lenovo laptop OpenGL Intel drivers. In order to workaround this
             // we don't load a QGCVideoBackground object when video is disabled. This prevents any video rendering
-            // code from running. Setting QGCVideoBackground.receiver/display = null does not work to prevent any
+            // code from running. Setting QGCVideoBackground.receiver = null does not work to prevent any
             // video OpenGL from being generated. Hence the Loader to completely remove it.
             height:             parent.getHeight()
             width:              parent.getWidth()
             anchors.centerIn:   parent
             visible:            _videoReceiver && _videoReceiver.videoRunning
-            sourceComponent:    videoDisabled ? null : videoBackgroundComponent
+            sourceComponent:    videoBackgroundComponent
 
             property bool videoDisabled: QGroundControl.settingsManager.videoSettings.videoSource.rawValue === QGroundControl.settingsManager.videoSettings.disabledVideoSource
         }
@@ -169,9 +169,9 @@ Item {
             }
             QGCVideoBackground {
                 id:             thermalVideo
+                objectName:     "thermalVideo"
                 anchors.fill:   parent
                 receiver:       QGroundControl.videoManager.thermalVideoReceiver
-                display:        QGroundControl.videoManager.thermalVideoReceiver ? QGroundControl.videoManager.thermalVideoReceiver.videoSurface : null
                 opacity:        _camera ? (_camera.thermalMode === QGCCameraControl.THERMAL_BLEND ? _camera.thermalOpacity / 100 : 1.0) : 0
             }
         }

src/FlightDisplay/VideoManager.cc

@@ -18,8 +18,6 @@
 #include <QCameraInfo>
 #endif
 
-#include <VideoItem.h>
-
 #include "ScreenToolsController.h"
 #include "VideoManager.h"
 #include "QGCToolbox.h"
@@ -57,7 +55,6 @@ VideoManager::setToolbox(QGCToolbox *toolbox)
    QQmlEngine::setObjectOwnership(this, QQmlEngine::CppOwnership);
    qmlRegisterUncreatableType<VideoManager> ("QGroundControl.VideoManager", 1, 0, "VideoManager", "Reference only");
    qmlRegisterUncreatableType<VideoReceiver>("QGroundControl",              1, 0, "VideoReceiver","Reference only");
-   qmlRegisterUncreatableType<VideoSurface> ("QGroundControl",              1, 0, "VideoSurface", "Reference only");
    _videoSettings = toolbox->settingsManager()->videoSettings();
    QString videoSource = _videoSettings->videoSource()->rawValue().toString();
    connect(_videoSettings->videoSource(),   &Fact::rawValueChanged, this, &VideoManager::_videoSourceChanged);
@@ -285,6 +282,113 @@ VideoManager::setfullScreen(bool f)
     emit fullScreenChanged();
 }
 
+//-----------------------------------------------------------------------------
+#if defined(QGC_GST_STREAMING)
+GstElement*
+VideoManager::_makeVideoSink(const QString& widgetName)
+{
+    GstElement* glupload        = nullptr;
+    GstElement* glcolorconvert  = nullptr;
+    GstElement* qmlglsink       = nullptr;
+    GstElement* bin             = nullptr;
+    GstElement* sink            = nullptr;
+
+    do {
+        QQuickItem* root = qgcApp()->mainRootWindow();
+
+        if (root == nullptr) {
+            qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. No root window";
+            break;
+        }
+
+        QQuickItem* widget = root->findChild<QQuickItem*>(widgetName);
+
+        if (widget == nullptr) {
+            qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. Widget \'" << widgetName << "\' not found";
+            break;
+        }
+
+        if ((glupload = gst_element_factory_make("glupload", nullptr)) == nullptr) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glupload')";
+            break;
+        }
+
+        if ((glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr)) == nullptr) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glcolorconvert')";
+            break;
+        }
+
+        if ((qmlglsink = gst_element_factory_make("qmlglsink", nullptr)) == nullptr) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('qmlglsink')";
+            break;
+        }
+
+        g_object_set(qmlglsink, "widget", widget, NULL);
+
+        if ((bin = gst_bin_new("videosink")) == nullptr) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_bin_new('videosink')";
+            break;
+        }
+
+        GstPad* pad;
+
+        if ((pad = gst_element_get_static_pad(glupload, "sink")) == nullptr) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_get_static_pad(glupload, 'sink')";
+            break;
+        }
+
+        gst_bin_add_many(GST_BIN(bin), glupload, glcolorconvert, qmlglsink, nullptr);
+        gboolean ret = gst_element_link_many(glupload, glcolorconvert, qmlglsink, nullptr);
+
+        qmlglsink = glcolorconvert = glupload = nullptr;
+
+        if (!ret) {
+            qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_link_many()";
+            break;
+        }
+
+        gst_element_add_pad(bin, gst_ghost_pad_new("sink", pad));
+        gst_object_unref(pad);
+        pad = nullptr;
+
+        sink = bin;
+        bin = nullptr;
+    } while(0);
+
+    if (bin != nullptr) {
+        gst_object_unref(bin);
+        bin = nullptr;
+    }
+
+    if (qmlglsink != nullptr) {
+        gst_object_unref(qmlglsink);
+        qmlglsink = nullptr;
+    }
+
+    if (glcolorconvert != nullptr) {
+        gst_object_unref(glcolorconvert);
+        glcolorconvert = nullptr;
+    }
+
+    if (glupload != nullptr) {
+        gst_object_unref(glupload);
+        glupload = nullptr;
+    }
+
+    return sink;
+}
+#endif
+
+//-----------------------------------------------------------------------------
+void
+VideoManager::_initVideo()
+{
+#if defined(QGC_GST_STREAMING)
+    _videoReceiver->setVideoSink(_makeVideoSink("videoContent"));
+    _thermalVideoReceiver->setVideoSink(_makeVideoSink("thermalVideo"));
+#endif
+}
+
 //-----------------------------------------------------------------------------
 void
 VideoManager::_updateSettings()

src/FlightDisplay/VideoManager.h

@@ -103,6 +103,11 @@ protected slots:
     void _connectionLostChanged     (bool connectionLost);
 
 protected:
+    friend class FinishVideoInitialization;
+#if defined(QGC_GST_STREAMING)
+    GstElement* _makeVideoSink      (const QString& widgetName);
+#endif
+    void _initVideo                 ();
     void _updateSettings            ();
 
 protected:

src/FlightMap/QGCVideoBackground.qml

@@ -14,13 +14,11 @@
  *   @author Gus Grubba <gus@auterion.com>
  */
 
-import QtQuick                          2.11
-import QtQuick.Controls                 2.4
-import QGroundControl.QgcQtGStreamer    1.0
+import QtQuick                                  2.11
+import QtQuick.Controls                         2.4
+import org.freedesktop.gstreamer.GLVideoItem    1.0
 
-VideoItem {
+GstGLVideoItem {
     id: videoBackground
-    property var display
     property var receiver
-    surface: display
 }

src/QGCApplication.cc

@@ -26,6 +26,7 @@
 #include <QStringListModel>
 #include <QRegularExpression>
 #include <QFontDatabase>
+#include <QQuickWindow>
 
 #ifdef QGC_ENABLE_BLUETOOTH
 #include <QBluetoothLocalDevice>
@@ -68,7 +69,6 @@
 #include "CoordinateVector.h"
 #include "PlanMasterController.h"
 #include "VideoManager.h"
-#include "VideoSurface.h"
 #include "VideoReceiver.h"
 #include "LogDownloadController.h"
 #if defined(QGC_ENABLE_MAVLINK_INSPECTOR)
@@ -128,6 +128,22 @@
 
 #include "QGCMapEngine.h"
 
+class FinishVideoInitialization : public QRunnable
+{
+public:
+  FinishVideoInitialization(VideoManager* manager)
+      : _manager(manager)
+  {}
+
+  void run () {
+      _manager->_initVideo();
+  }
+
+private:
+  VideoManager* _manager;
+};
+
+
 QGCApplication* QGCApplication::_app = nullptr;
 
 const char* QGCApplication::_deleteAllSettingsKey           = "DeleteAllSettingsNextBoot";
@@ -560,6 +576,13 @@ bool QGCApplication::_initForNormalAppBoot()
 
     _qmlAppEngine = toolbox()->corePlugin()->createRootWindow(this);
 
+    QQuickWindow* rootWindow = (QQuickWindow*)qgcApp()->mainRootWindow();
+
+    if (rootWindow) {
+        rootWindow->scheduleRenderJob (new FinishVideoInitialization (toolbox()->videoManager()),
+                QQuickWindow::BeforeSynchronizingStage);
+    }
+
     // Safe to show popup error messages now that main window is created
     UASMessageHandler* msgHandler = qgcApp()->toolbox()->uasMessageHandler();
     if (msgHandler) {

src/VideoStreaming/GLVideoItemStub.cc

+#include "GLVideoItemStub.h"
+
+GLVideoItemStub::GLVideoItemStub()
+{
+}
+
+GLVideoItemStub::~GLVideoItemStub()
+{
+}

src/VideoStreaming/GLVideoItemStub.h

+#pragma once
+
+#include <QtQuick/QQuickItem>
+
+class GLVideoItemStub : public QQuickItem
+{
+    Q_OBJECT
+
+public:
+    GLVideoItemStub();
+    ~GLVideoItemStub();
+
+protected:
+
+private:
+
+};

src/VideoStreaming/VideoItem.cc

-/****************************************************************************
- *
- * (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
- *
- * QGroundControl is licensed according to the terms in the file
- * COPYING.md in the root of the source code directory.
- *
- ****************************************************************************/
-
-
-/**
- * @file
- *   @brief QGC Video Item
- *   @author Gus Grubba <gus@auterion.com>
- */
-
-#include <QtCore/QPointer>
-#include <QtQuick/QSGNode>
-#include <QtQuick/QSGFlatColorMaterial>
-
-#include "VideoItem.h"
-#if defined(QGC_GST_STREAMING)
-#include "VideoSurface_p.h"
-#endif
-
-#if defined(QGC_GST_STREAMING)
-struct VideoItem::Private
-{
-    QPointer<VideoSurface> surface;
-    bool surfaceDirty;
-    QRectF targetArea;
-};
-#endif
-
-VideoItem::VideoItem(QQuickItem *parent)
-    : QQuickItem(parent)
-#if defined(QGC_GST_STREAMING)
-    , _data(new Private)
-#endif
-{
-#if defined(QGC_GST_STREAMING)
-    _data->surfaceDirty = true;
-    setFlag(QQuickItem::ItemHasContents, true);
-#endif
-}
-
-VideoItem::~VideoItem()
-{
-#if defined(QGC_GST_STREAMING)
-    setSurface(0);
-    delete _data;
-#endif
-}
-
-VideoSurface *VideoItem::surface() const
-{
-#if defined(QGC_GST_STREAMING)
-    return _data->surface.data();
-#else
-    return nullptr;
-#endif
-}
-
-void VideoItem::setSurface(VideoSurface *surface)
-{
-#if defined(QGC_GST_STREAMING)
-    if (_data->surface) {
-        _data->surface.data()->_data->items.remove(this);
-    }
-    _data->surface = surface;
-    _data->surfaceDirty = true;
-    if (_data->surface) {
-        _data->surface.data()->_data->items.insert(this);
-    }
-#else
-    Q_UNUSED(surface)
-#endif
-}
-
-#if defined(QGC_GST_STREAMING)
-QSGGeometry* VideoItem::_createDefaultGeometry(QRectF& rectBound)
-{
-	QSGGeometry *geometry = new QSGGeometry(QSGGeometry::defaultAttributes_Point2D(), 4);
-	geometry->vertexDataAsPoint2D()[0].set(rectBound.x(), rectBound.y());
-	geometry->vertexDataAsPoint2D()[1].set(rectBound.x(), rectBound.height());
-	geometry->vertexDataAsPoint2D()[2].set(rectBound.width(), rectBound.y());
-	geometry->vertexDataAsPoint2D()[3].set(rectBound.width(), rectBound.height());
-
-	return geometry;
-}
-
-QSGNode* VideoItem::updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData*)
-{
-    QRectF r = boundingRect();
-    QSGNode* newNode = nullptr;
-
-    if (_data->surfaceDirty) {
-        delete oldNode;
-        oldNode = nullptr;
-        _data->surfaceDirty = false;
-    }
-
-    if (!_data->surface || _data->surface.data()->_data->videoSink == nullptr) {
-        if (!oldNode) {
-            QSGFlatColorMaterial *material = new QSGFlatColorMaterial;
-            material->setColor(Qt::black);
-            QSGGeometryNode *node = new QSGGeometryNode;
-            node->setMaterial(material);
-            node->setFlag(QSGNode::OwnsMaterial);
-            node->setFlag(QSGNode::OwnsGeometry);
-            newNode = node;
-            _data->targetArea = QRectF(); //force geometry to be set
-        } else {
-            newNode = oldNode;
-        }
-        if (r != _data->targetArea) {
-            QSGGeometryNode *node = static_cast<QSGGeometryNode*>(newNode);
-			node->setGeometry(_createDefaultGeometry(r));
-            _data->targetArea = r;
-        }
-    } else {
-        g_signal_emit_by_name(_data->surface.data()->_data->videoSink, "update-node", (void*)oldNode, r.x(), r.y(), r.width(), r.height(), (void**)&newNode);
-    }
-
-	// Sometimes we can still end up here with no geometry when gstreamer fails to create it for whatever reason. If that happens it can
-	// cause crashes.
-	QSGGeometryNode *node = static_cast<QSGGeometryNode*>(newNode);
-	if (node->geometry() == nullptr) {
-		qDebug() << "Creating default geom";
-		node->setGeometry(_createDefaultGeometry(r));
-	}
-
-    return newNode;
-}
-#endif

src/VideoStreaming/VideoItem.h

-/****************************************************************************
- *
- * (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
- *
- * QGroundControl is licensed according to the terms in the file
- * COPYING.md in the root of the source code directory.
- *
- ****************************************************************************/
-
-
-/**
- * @file
- *   @brief QGC Video Item
- *   @author Gus Grubba <gus@auterion.com>
- */
-
-#pragma once
-
-#include <QtQuick/QQuickItem>
-#include "VideoSurface.h"
-
-class QSGGeometry;
-
-class VideoItem : public QQuickItem
-{
-    Q_OBJECT
-    Q_DISABLE_COPY(VideoItem)
-    Q_PROPERTY(VideoSurface* surface READ surface WRITE setSurface)
-
-public:
-    explicit VideoItem(QQuickItem *parent = 0);
-    virtual ~VideoItem();
-
-    VideoSurface *surface() const;
-    void setSurface(VideoSurface *surface);
-
-protected:
-#if defined(QGC_GST_STREAMING)
-    /*! Reimplemented from QQuickItem. */
-    virtual QSGNode* updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *updatePaintNodeData);
-#endif
-
-private:
-#if defined(QGC_GST_STREAMING)
-	QSGGeometry* _createDefaultGeometry(QRectF& rectBound);
-	
-    struct Private;
-    Private* const _data;
-#endif
-};
-

src/VideoStreaming/VideoReceiver.cc

@@ -64,24 +64,23 @@ VideoReceiver::VideoReceiver(QObject* parent)
     , _pipeline(nullptr)
     , _pipelineStopRec(nullptr)
     , _videoSink(nullptr)
+    , _lastFrameId(G_MAXUINT64)
+    , _lastFrameTime(0)
     , _restart_time_ms(1389)
     , _socket(nullptr)
     , _serverPresent(false)
     , _tcpTestInterval_ms(5000)
     , _udpReconnect_us(5000000)
 #endif
-    , _videoSurface(nullptr)
     , _videoRunning(false)
     , _showFullScreen(false)
     , _videoSettings(nullptr)
     , _hwDecoderName(nullptr)
     , _swDecoderName("avdec_h264")
 {
-    _videoSurface = new VideoSurface;
     _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings();
 #if defined(QGC_GST_STREAMING)
     setVideoDecoder(H264_SW);
-    _setVideoSink(_videoSurface->videoSink());
     _restart_timer.setSingleShot(true);
     connect(&_restart_timer, &QTimer::timeout, this, &VideoReceiver::_restart_timeout);
     _tcp_timer.setSingleShot(true);
@@ -98,32 +97,9 @@ VideoReceiver::~VideoReceiver()
 {
 #if defined(QGC_GST_STREAMING)
     stop();
-    if(_socket) {
-        delete _socket;
-        _socket = nullptr;
-    }
-    if (_videoSink) {
-        gst_object_unref(_videoSink);
-    }
+    setVideoSink(nullptr);
 #endif
-    if(_videoSurface)
-        delete _videoSurface;
-}
-
-#if defined(QGC_GST_STREAMING)
-void
-VideoReceiver::_setVideoSink(GstElement* sink)
-{
-    if (_videoSink) {
-        gst_object_unref(_videoSink);
-        _videoSink = nullptr;
-    }
-    if (sink) {
-        _videoSink = sink;
-        gst_object_ref_sink(_videoSink);
-    }
 }
-#endif
 
 //-----------------------------------------------------------------------------
 void
@@ -287,6 +263,9 @@ VideoReceiver::start()
         return;
     }
 
+    _lastFrameId = G_MAXUINT64;
+    _lastFrameTime = 0;
+
     bool running    = false;
     bool pipelineUp = false;
 
@@ -458,11 +437,21 @@ VideoReceiver::start()
 
         // If we failed before adding items to the pipeline, then clean up
         if (!pipelineUp) {
+            if (queue1 != nullptr) {
+                gst_object_unref(queue1);
+                queue1 = nullptr;
+            }
+
             if (decoder != nullptr) {
                 gst_object_unref(decoder);
                 decoder = nullptr;
             }
 
+            if (queue != nullptr) {
+                gst_object_unref(queue);
+                queue = nullptr;
+            }
+
             if (parser != nullptr) {
                 gst_object_unref(parser);
                 parser = nullptr;
@@ -480,13 +469,9 @@ VideoReceiver::start()
 
             if (_tee != nullptr) {
                 gst_object_unref(_tee);
-                dataSource = nullptr;
+                _tee = nullptr;
             }
 
-            if (queue != nullptr) {
-                gst_object_unref(queue);
-                dataSource = nullptr;
-            }
         }
 
         _running = false;
@@ -552,7 +537,6 @@ VideoReceiver::_shutdownPipeline() {
         bus = nullptr;
     }
     gst_element_set_state(_pipeline, GST_STATE_NULL);
-    gst_bin_remove(GST_BIN(_pipeline), _videoSink);
     gst_object_unref(_pipeline);
     _pipeline = nullptr;
     delete _sink;
@@ -712,6 +696,38 @@ VideoReceiver::setVideoDecoder(VideoEncoding encoding)
     }
 }
 
+//-----------------------------------------------------------------------------
+#if defined(QGC_GST_STREAMING)
+void
+VideoReceiver::setVideoSink(GstElement* videoSink)
+{
+    if(_pipeline != nullptr) {
+        qCDebug(VideoReceiverLog) << "Video receiver pipeline is active, video sink change is not possible";
+        return;
+    }
+
+    if (_videoSink != nullptr) {
+        gst_object_unref(_videoSink);
+        _videoSink = nullptr;
+    }
+
+    if (videoSink != nullptr) {
+        _videoSink = videoSink;
+        gst_object_ref(_videoSink);
+
+        GstPad* pad = gst_element_get_static_pad(_videoSink, "sink");
+
+        if (pad != nullptr) {
+            gst_pad_add_probe(pad, (GstPadProbeType)(GST_PAD_PROBE_TYPE_BUFFER), _videoSinkProbe, this, nullptr);
+            gst_object_unref(pad);
+            pad = nullptr;
+        } else {
+            qCDebug(VideoReceiverLog) << "Unable to find sink pad of video sink";
+        }
+    }
+}
+#endif
+
 //-----------------------------------------------------------------------------
 // When we finish our pipeline will look like this:
 //
@@ -923,6 +939,30 @@ VideoReceiver::_unlinkCallBack(GstPad* pad, GstPadProbeInfo* info, gpointer user
 }
 #endif
 
+//-----------------------------------------------------------------------------
+#if defined(QGC_GST_STREAMING)
+GstPadProbeReturn
+VideoReceiver::_videoSinkProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data)
+{
+    Q_UNUSED(pad);
+    if(info != nullptr && user_data != nullptr) {
+        VideoReceiver* pThis = static_cast<VideoReceiver*>(user_data);
+        pThis->_noteVideoSinkFrame();
+    }
+
+    return GST_PAD_PROBE_OK;
+}
+#endif
+
+//-----------------------------------------------------------------------------
+#if defined(QGC_GST_STREAMING)
+void
+VideoReceiver::_noteVideoSinkFrame()
+{
+    _lastFrameTime = QDateTime::currentSecsSinceEpoch();
+}
+#endif
+
 //-----------------------------------------------------------------------------
 #if defined(QGC_GST_STREAMING)
 GstPadProbeReturn
@@ -971,41 +1011,39 @@ void
 VideoReceiver::_updateTimer()
 {
 #if defined(QGC_GST_STREAMING)
-    if(_videoSurface) {
-        if(_stopping || _starting) {
-            return;
-        }
-        if(_streaming) {
-            if(!_videoRunning) {
-                _videoSurface->setLastFrame(0);
-                _videoRunning = true;
-                emit videoRunningChanged();
-            }
-        } else {
-            if(_videoRunning) {
-                _videoRunning = false;
-                emit videoRunningChanged();
-            }
+    if(_stopping || _starting) {
+        return;
+    }
+
+    if(_streaming) {
+        if(!_videoRunning) {
+            _videoRunning = true;
+            emit videoRunningChanged();
         }
+    } else {
         if(_videoRunning) {
-            uint32_t timeout = 1;
-            if(qgcApp()->toolbox() && qgcApp()->toolbox()->settingsManager()) {
-                timeout = _videoSettings->rtspTimeout()->rawValue().toUInt();
-            }
-            time_t elapsed = 0;
-            time_t lastFrame = _videoSurface->lastFrame();
-            if(lastFrame != 0) {
-                elapsed = time(nullptr) - _videoSurface->lastFrame();
-            }
-            if(elapsed > static_cast<time_t>(timeout) && _videoSurface) {
-                stop();
-                // We want to start it back again with _updateTimer
-                _stop = false;
-            }
-        } else {
-            if(!_stop && _running && !_uri.isEmpty() && _videoSettings->streamEnabled()->rawValue().toBool()) {
-                start();
-            }
+            _videoRunning = false;
+            emit videoRunningChanged();
+        }
+    }
+
+    if(_videoRunning) {
+        uint32_t timeout = 1;
+        if(qgcApp()->toolbox() && qgcApp()->toolbox()->settingsManager()) {
+            timeout = _videoSettings->rtspTimeout()->rawValue().toUInt();
+        }
+
+        const qint64 now = QDateTime::currentSecsSinceEpoch();
+
+        if(now - _lastFrameTime > timeout) {
+            stop();
+            // We want to start it back again with _updateTimer
+            _stop = false;
+        }
+    } else {
+		// FIXME: AV: if pipeline is _running but not _streaming for some time then we need to restart
+        if(!_stop && !_running && !_uri.isEmpty() && _videoSettings->streamEnabled()->rawValue().toBool()) {
+            start();
         }
     }
 #endif

src/VideoStreaming/VideoReceiver.h

@@ -20,8 +20,6 @@
 #include <QTimer>
 #include <QTcpSocket>
 
-#include "VideoSurface.h"
-
 #if defined(QGC_GST_STREAMING)
 #include <gst/gst.h>
 #endif
@@ -44,7 +42,6 @@ public:
 #if defined(QGC_GST_STREAMING)
     Q_PROPERTY(bool             recording           READ    recording           NOTIFY recordingChanged)
 #endif
-    Q_PROPERTY(VideoSurface*    videoSurface        READ    videoSurface        CONSTANT)
     Q_PROPERTY(bool             videoRunning        READ    videoRunning        NOTIFY  videoRunningChanged)
     Q_PROPERTY(QString          imageFile           READ    imageFile           NOTIFY  imageFileChanged)
     Q_PROPERTY(QString          videoFile           READ    videoFile           NOTIFY  videoFileChanged)
@@ -57,7 +54,6 @@ public:
     virtual bool            recording       () { return _recording; }
 #endif
 
-    virtual VideoSurface*   videoSurface    () { return _videoSurface; }
     virtual bool            videoRunning    () { return _videoRunning; }
     virtual QString         imageFile       () { return _imageFile; }
     virtual QString         videoFile       () { return _videoFile; }
@@ -68,6 +64,9 @@ public:
     virtual void        setShowFullScreen   (bool show) { _showFullScreen = show; emit showFullScreenChanged(); }
 
     void                  setVideoDecoder   (VideoEncoding encoding);
+#if defined(QGC_GST_STREAMING)
+    void                  setVideoSink      (GstElement* videoSink);
+#endif
 
 signals:
     void videoRunningChanged                ();
@@ -123,19 +122,23 @@ protected:
     Sink*               _sink;
     GstElement*         _tee;
 
+    void _noteVideoSinkFrame                            ();
+
     static gboolean             _onBusMessage           (GstBus* bus, GstMessage* message, gpointer user_data);
     static GstPadProbeReturn    _unlinkCallBack         (GstPad* pad, GstPadProbeInfo* info, gpointer user_data);
+    static GstPadProbeReturn    _videoSinkProbe         (GstPad* pad, GstPadProbeInfo* info, gpointer user_data);
     static GstPadProbeReturn    _keyframeWatch          (GstPad* pad, GstPadProbeInfo* info, gpointer user_data);
 
     virtual void                _detachRecordingBranch  (GstPadProbeInfo* info);
     virtual void                _shutdownRecordingBranch();
     virtual void                _shutdownPipeline       ();
     virtual void                _cleanupOldVideos       ();
-    virtual void                _setVideoSink           (GstElement* sink);
 
     GstElement*     _pipeline;
     GstElement*     _pipelineStopRec;
     GstElement*     _videoSink;
+    guint64         _lastFrameId;
+    qint64          _lastFrameTime;
 
     //-- Wait for Video Server to show up before starting
     QTimer          _frameTimer;
@@ -153,7 +156,6 @@ protected:
     QString         _uri;
     QString         _imageFile;
     QString         _videoFile;
-    VideoSurface*   _videoSurface;
     bool            _videoRunning;
     bool            _showFullScreen;
     VideoSettings*  _videoSettings;

src/VideoStreaming/VideoStreaming.cc

@@ -25,16 +25,14 @@
 #if defined(__ios__)
 #include "gst_ios_init.h"
 #endif
+#else
+#include "GLVideoItemStub.h"
 #endif
 
 #include "VideoStreaming.h"
-#include "VideoItem.h"
-#include "VideoSurface.h"
 
 #if defined(QGC_GST_STREAMING)
     G_BEGIN_DECLS
-    // Our own plugin
-    GST_PLUGIN_STATIC_DECLARE(QGC_VIDEOSINK_PLUGIN);
     // The static plugins we use
 #if defined(__mobile__)
     GST_PLUGIN_STATIC_DECLARE(coreelements);
@@ -47,10 +45,12 @@
     GST_PLUGIN_STATIC_DECLARE(rtpmanager);
     GST_PLUGIN_STATIC_DECLARE(isomp4);
     GST_PLUGIN_STATIC_DECLARE(matroska);
+    GST_PLUGIN_STATIC_DECLARE(opengl);
 #endif
 #if defined(__android__)
     GST_PLUGIN_STATIC_DECLARE(androidmedia);
 #endif
+    GST_PLUGIN_STATIC_DECLARE(qmlgl);
     G_END_DECLS
 #endif
 
@@ -148,8 +148,6 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu
             g_error_free(error);
         }
     #endif
-        // Our own plugin
-        GST_PLUGIN_STATIC_REGISTER(QGC_VIDEOSINK_PLUGIN);
         // The static plugins we use
     #if defined(__android__)
         GST_PLUGIN_STATIC_REGISTER(coreelements);
@@ -164,12 +162,33 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu
         GST_PLUGIN_STATIC_REGISTER(matroska);
         GST_PLUGIN_STATIC_REGISTER(androidmedia);
     #endif
+
+#if defined(__mobile__)
+        GST_PLUGIN_STATIC_REGISTER(opengl);
+#endif
+
+    /* the plugin must be loaded before loading the qml file to register the
+     * GstGLVideoItem qml item
+     * FIXME Add a QQmlExtensionPlugin into qmlglsink to register GstGLVideoItem
+     * with the QML engine, then remove this */
+    GstElement *sink = gst_element_factory_make("qmlglsink", nullptr);
+
+    if (sink == nullptr) {
+        GST_PLUGIN_STATIC_REGISTER(qmlgl);
+        sink = gst_element_factory_make("qmlglsink", nullptr);
+    }
+
+    if (sink != nullptr) {
+        gst_object_unref(sink);
+        sink = nullptr;
+    } else {
+        qCritical() << "unable to find qmlglsink - you need to build it yourself and add to GST_PLUGIN_PATH";
+    }
 #else
-    Q_UNUSED(argc);
-    Q_UNUSED(argv);
-    Q_UNUSED(logpath);
-    Q_UNUSED(debuglevel);
+    qmlRegisterType<GLVideoItemStub> ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem");
+    Q_UNUSED(argc)
+    Q_UNUSED(argv)
+    Q_UNUSED(logpath)
+    Q_UNUSED(debuglevel)
 #endif
-    qmlRegisterType<VideoItem>              ("QGroundControl.QgcQtGStreamer", 1, 0, "VideoItem");
-    qmlRegisterUncreatableType<VideoSurface>("QGroundControl.QgcQtGStreamer", 1, 0, "VideoSurface", QStringLiteral("VideoSurface from QML is not supported"));
 }

src/VideoStreaming/VideoStreaming.h

@@ -17,5 +17,3 @@
 #pragma once
 
 extern void initializeVideoStreaming    (int &argc, char *argv[], char* filename, char* debuglevel);
-
-

src/VideoStreaming/VideoStreaming.pri

@@ -12,9 +12,10 @@
 #
 
 LinuxBuild {
+    QT += x11extras waylandclient
     CONFIG += link_pkgconfig
     packagesExist(gstreamer-1.0) {
-        PKGCONFIG   += gstreamer-1.0  gstreamer-video-1.0
+        PKGCONFIG   += gstreamer-1.0  gstreamer-video-1.0 gstreamer-gl-1.0
         CONFIG      += VideoEnabled
     }
 } else:MacBuild {
@@ -39,10 +40,11 @@ LinuxBuild {
     exists($$GST_ROOT) {
         CONFIG      += VideoEnabled
 
-        LIBS        += -L$$GST_ROOT/lib -lgstreamer-1.0 -lgstvideo-1.0 -lgstbase-1.0
+        LIBS        += -L$$GST_ROOT/lib -lgstreamer-1.0 -lgstgl-1.0 -lgstvideo-1.0 -lgstbase-1.0
         LIBS        += -lglib-2.0 -lintl -lgobject-2.0
 
         INCLUDEPATH += \
+            $$GST_ROOT/include \
             $$GST_ROOT/include/gstreamer-1.0 \
             $$GST_ROOT/include/glib-2.0 \
             $$GST_ROOT/lib/gstreamer-1.0/include \
@@ -86,10 +88,12 @@ LinuxBuild {
             -lgstrtpmanager \
             -lgstisomp4 \
             -lgstmatroska \
-            -lgstandroidmedia
+            -lgstandroidmedia \
+            -lgstopengl
 
         # Rest of GStreamer dependencies
         LIBS += -L$$GST_ROOT/lib \
+            -lgraphene-1.0 -ljpeg -lpng16 \
             -lgstfft-1.0 -lm  \
             -lgstnet-1.0 -lgio-2.0 \
             -lgstphotography-1.0 -lgstgl-1.0 -lEGL \
@@ -113,68 +117,16 @@ VideoEnabled {
     message("Including support for video streaming")
 
     DEFINES += \
-        QGC_GST_STREAMING \
-        GST_PLUGIN_BUILD_STATIC \
-        QTGLVIDEOSINK_NAME=qt5glvideosink \
-        QGC_VIDEOSINK_PLUGIN=qt5videosink
-
-    INCLUDEPATH += \
-        $$PWD/gstqtvideosink \
-        $$PWD/gstqtvideosink/delegates \
-        $$PWD/gstqtvideosink/painters \
-        $$PWD/gstqtvideosink/utils \
-
-    #-- QtGstreamer (gutted to our needs)
-
-    HEADERS += \
-        $$PWD/gstqtvideosink/delegates/basedelegate.h \
-        $$PWD/gstqtvideosink/delegates/qtquick2videosinkdelegate.h \
-        $$PWD/gstqtvideosink/delegates/qtvideosinkdelegate.h \
-        $$PWD/gstqtvideosink/delegates/qwidgetvideosinkdelegate.h \
-        $$PWD/gstqtvideosink/gstqtglvideosink.h \
-        $$PWD/gstqtvideosink/gstqtglvideosinkbase.h \
-        $$PWD/gstqtvideosink/gstqtquick2videosink.h \
-        $$PWD/gstqtvideosink/gstqtvideosink.h \
-        $$PWD/gstqtvideosink/gstqtvideosinkbase.h \
-        $$PWD/gstqtvideosink/gstqtvideosinkmarshal.h \
-        $$PWD/gstqtvideosink/gstqtvideosinkplugin.h \
-        $$PWD/gstqtvideosink/gstqwidgetvideosink.h \
-        $$PWD/gstqtvideosink/painters/abstractsurfacepainter.h \
-        $$PWD/gstqtvideosink/painters/genericsurfacepainter.h \
-        $$PWD/gstqtvideosink/painters/openglsurfacepainter.h \
-        $$PWD/gstqtvideosink/painters/videomaterial.h \
-        $$PWD/gstqtvideosink/painters/videonode.h \
-        $$PWD/gstqtvideosink/utils/bufferformat.h \
-        $$PWD/gstqtvideosink/utils/utils.h \
-        $$PWD/gstqtvideosink/utils/glutils.h \
-
-    SOURCES += \
-        $$PWD/gstqtvideosink/delegates/basedelegate.cpp \
-        $$PWD/gstqtvideosink/delegates/qtquick2videosinkdelegate.cpp \
-        $$PWD/gstqtvideosink/delegates/qtvideosinkdelegate.cpp \
-        $$PWD/gstqtvideosink/delegates/qwidgetvideosinkdelegate.cpp \
-        $$PWD/gstqtvideosink/gstqtglvideosink.cpp \
-        $$PWD/gstqtvideosink/gstqtglvideosinkbase.cpp \
-        $$PWD/gstqtvideosink/gstqtvideosinkmarshal.c \
-        $$PWD/gstqtvideosink/gstqtquick2videosink.cpp \
-        $$PWD/gstqtvideosink/gstqtvideosink.cpp \
-        $$PWD/gstqtvideosink/gstqtvideosinkbase.cpp \
-        $$PWD/gstqtvideosink/gstqtvideosinkplugin.cpp \
-        $$PWD/gstqtvideosink/gstqwidgetvideosink.cpp \
-        $$PWD/gstqtvideosink/painters/genericsurfacepainter.cpp \
-        $$PWD/gstqtvideosink/painters/openglsurfacepainter.cpp \
-        $$PWD/gstqtvideosink/painters/videomaterial.cpp \
-        $$PWD/gstqtvideosink/painters/videonode.cpp \
-        $$PWD/gstqtvideosink/utils/bufferformat.cpp \
-        $$PWD/gstqtvideosink/utils/utils.cpp \
+        QGC_GST_STREAMING
 
     iOSBuild {
         OBJECTIVE_SOURCES += \
-            $$PWD/ios/gst_ios_init.m
+            $$PWD/iOS/gst_ios_init.m
         INCLUDEPATH += \
-            $$PWD/ios
+            $$PWD/iOS
     }
 
+    include($$PWD/../../qmlglsink.pri)
 } else {
     LinuxBuild|MacBuild|iOSBuild|WindowsBuild|AndroidBuild {
         message("Skipping support for video streaming (GStreamer libraries not installed)")
@@ -182,5 +134,10 @@ VideoEnabled {
     } else {
         message("Skipping support for video streaming (Unsupported platform)")
     }
-}
 
+    SOURCES += \
+        $$PWD/GLVideoItemStub.cc
+
+    HEADERS += \
+        $$PWD/GLVideoItemStub.h
+}