Commit 700fab73 authored by Andrew Voznytsa's avatar Andrew Voznytsa

Switch from builtin video sink to qmlglsink

parent e36a8fbc
......@@ -1374,18 +1374,13 @@ INCLUDEPATH += \
src/VideoStreaming
HEADERS += \
src/VideoStreaming/VideoItem.h \
src/VideoStreaming/VideoReceiver.h \
src/VideoStreaming/VideoStreaming.h \
src/VideoStreaming/VideoSurface.h \
src/VideoStreaming/VideoSurface_p.h \
src/VideoStreaming/SubtitleWriter.h \
SOURCES += \
src/VideoStreaming/VideoItem.cc \
src/VideoStreaming/VideoReceiver.cc \
src/VideoStreaming/VideoStreaming.cc \
src/VideoStreaming/VideoSurface.cc \
src/VideoStreaming/SubtitleWriter.cc \
contains (CONFIG, DISABLE_VIDEOSTREAMING) {
......
......@@ -79,8 +79,8 @@ Item {
id: videoBackgroundComponent
QGCVideoBackground {
id: videoContent
objectName: "videoContent"
receiver: _videoReceiver
display: _videoReceiver && _videoReceiver.videoSurface
Connections {
target: _videoReceiver
......@@ -125,7 +125,7 @@ Item {
Loader {
// GStreamer is causing crashes on Lenovo laptop OpenGL Intel drivers. In order to workaround this
// we don't load a QGCVideoBackground object when video is disabled. This prevents any video rendering
// code from running. Setting QGCVideoBackground.receiver/display = null does not work to prevent any
// code from running. Setting QGCVideoBackground.receiver = null does not work to prevent any
// video OpenGL from being generated. Hence the Loader to completely remove it.
height: parent.getHeight()
width: parent.getWidth()
......@@ -169,9 +169,9 @@ Item {
}
QGCVideoBackground {
id: thermalVideo
objectName: "thermalVideo"
anchors.fill: parent
receiver: QGroundControl.videoManager.thermalVideoReceiver
display: QGroundControl.videoManager.thermalVideoReceiver ? QGroundControl.videoManager.thermalVideoReceiver.videoSurface : null
opacity: _camera ? (_camera.thermalMode === QGCCameraControl.THERMAL_BLEND ? _camera.thermalOpacity / 100 : 1.0) : 0
}
}
......
......@@ -18,8 +18,6 @@
#include <QCameraInfo>
#endif
#include <VideoItem.h>
#include "ScreenToolsController.h"
#include "VideoManager.h"
#include "QGCToolbox.h"
......@@ -57,7 +55,6 @@ VideoManager::setToolbox(QGCToolbox *toolbox)
QQmlEngine::setObjectOwnership(this, QQmlEngine::CppOwnership);
qmlRegisterUncreatableType<VideoManager> ("QGroundControl.VideoManager", 1, 0, "VideoManager", "Reference only");
qmlRegisterUncreatableType<VideoReceiver>("QGroundControl", 1, 0, "VideoReceiver","Reference only");
qmlRegisterUncreatableType<VideoSurface> ("QGroundControl", 1, 0, "VideoSurface", "Reference only");
_videoSettings = toolbox->settingsManager()->videoSettings();
QString videoSource = _videoSettings->videoSource()->rawValue().toString();
connect(_videoSettings->videoSource(), &Fact::rawValueChanged, this, &VideoManager::_videoSourceChanged);
......@@ -285,6 +282,113 @@ VideoManager::setfullScreen(bool f)
emit fullScreenChanged();
}
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
GstElement*
VideoManager::_makeVideoSink(const QString& widgetName)
{
GstElement* glupload = nullptr;
GstElement* glcolorconvert = nullptr;
GstElement* qmlglsink = nullptr;
GstElement* bin = nullptr;
GstElement* sink = nullptr;
do {
QQuickItem* root = qgcApp()->mainRootWindow();
if (root == nullptr) {
qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. No root window";
break;
}
QQuickItem* widget = root->findChild<QQuickItem*>(widgetName);
if (widget == nullptr) {
qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. Widget \'" << widgetName << "\' not found";
break;
}
if ((glupload = gst_element_factory_make("glupload", nullptr)) == nullptr) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glupload')";
break;
}
if ((glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr)) == nullptr) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glcolorconvert')";
break;
}
if ((qmlglsink = gst_element_factory_make("qmlglsink", nullptr)) == nullptr) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('qmlglsink')";
break;
}
g_object_set(qmlglsink, "widget", widget, NULL);
if ((bin = gst_bin_new("videosink")) == nullptr) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_bin_new('videosink')";
break;
}
GstPad* pad;
if ((pad = gst_element_get_static_pad(glupload, "sink")) == nullptr) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_get_static_pad(glupload, 'sink')";
break;
}
gst_bin_add_many(GST_BIN(bin), glupload, glcolorconvert, qmlglsink, nullptr);
gboolean ret = gst_element_link_many(glupload, glcolorconvert, qmlglsink, nullptr);
qmlglsink = glcolorconvert = glupload = nullptr;
if (!ret) {
qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_link_many()";
break;
}
gst_element_add_pad(bin, gst_ghost_pad_new("sink", pad));
gst_object_unref(pad);
pad = nullptr;
sink = bin;
bin = nullptr;
} while(0);
if (bin != nullptr) {
gst_object_unref(bin);
bin = nullptr;
}
if (qmlglsink != nullptr) {
gst_object_unref(qmlglsink);
qmlglsink = nullptr;
}
if (glcolorconvert != nullptr) {
gst_object_unref(glcolorconvert);
glcolorconvert = nullptr;
}
if (glupload != nullptr) {
gst_object_unref(glupload);
glupload = nullptr;
}
return sink;
}
#endif
//-----------------------------------------------------------------------------
void
VideoManager::_initVideo()
{
#if defined(QGC_GST_STREAMING)
_videoReceiver->setVideoSink(_makeVideoSink("videoContent"));
_thermalVideoReceiver->setVideoSink(_makeVideoSink("thermalVideo"));
#endif
}
//-----------------------------------------------------------------------------
void
VideoManager::_updateSettings()
......
......@@ -103,6 +103,11 @@ protected slots:
void _connectionLostChanged (bool connectionLost);
protected:
friend class FinishVideoInitialization;
#if defined(QGC_GST_STREAMING)
GstElement* _makeVideoSink (const QString& widgetName);
#endif
void _initVideo ();
void _updateSettings ();
protected:
......
......@@ -14,13 +14,11 @@
* @author Gus Grubba <gus@auterion.com>
*/
import QtQuick 2.11
import QtQuick.Controls 2.4
import QGroundControl.QgcQtGStreamer 1.0
import QtQuick 2.11
import QtQuick.Controls 2.4
import org.freedesktop.gstreamer.GLVideoItem 1.0
VideoItem {
GstGLVideoItem {
id: videoBackground
property var display
property var receiver
surface: display
}
......@@ -26,6 +26,7 @@
#include <QStringListModel>
#include <QRegularExpression>
#include <QFontDatabase>
#include <QQuickWindow>
#ifdef QGC_ENABLE_BLUETOOTH
#include <QBluetoothLocalDevice>
......@@ -68,7 +69,6 @@
#include "CoordinateVector.h"
#include "PlanMasterController.h"
#include "VideoManager.h"
#include "VideoSurface.h"
#include "VideoReceiver.h"
#include "LogDownloadController.h"
#if defined(QGC_ENABLE_MAVLINK_INSPECTOR)
......@@ -128,6 +128,22 @@
#include "QGCMapEngine.h"
class FinishVideoInitialization : public QRunnable
{
public:
FinishVideoInitialization(VideoManager* manager)
: _manager(manager)
{}
void run () {
_manager->_initVideo();
}
private:
VideoManager* _manager;
};
QGCApplication* QGCApplication::_app = nullptr;
const char* QGCApplication::_deleteAllSettingsKey = "DeleteAllSettingsNextBoot";
......@@ -560,6 +576,13 @@ bool QGCApplication::_initForNormalAppBoot()
_qmlAppEngine = toolbox()->corePlugin()->createRootWindow(this);
QQuickWindow* rootWindow = (QQuickWindow*)qgcApp()->mainRootWindow();
if (rootWindow) {
rootWindow->scheduleRenderJob (new FinishVideoInitialization (toolbox()->videoManager()),
QQuickWindow::BeforeSynchronizingStage);
}
// Safe to show popup error messages now that main window is created
UASMessageHandler* msgHandler = qgcApp()->toolbox()->uasMessageHandler();
if (msgHandler) {
......
/****************************************************************************
*
* (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
*
* QGroundControl is licensed according to the terms in the file
* COPYING.md in the root of the source code directory.
*
****************************************************************************/
/**
* @file
* @brief QGC Video Item
* @author Gus Grubba <gus@auterion.com>
*/
#include <QtCore/QPointer>
#include <QtQuick/QSGNode>
#include <QtQuick/QSGFlatColorMaterial>
#include "VideoItem.h"
#if defined(QGC_GST_STREAMING)
#include "VideoSurface_p.h"
#endif
#if defined(QGC_GST_STREAMING)
struct VideoItem::Private
{
QPointer<VideoSurface> surface;
bool surfaceDirty;
QRectF targetArea;
};
#endif
VideoItem::VideoItem(QQuickItem *parent)
: QQuickItem(parent)
#if defined(QGC_GST_STREAMING)
, _data(new Private)
#endif
{
#if defined(QGC_GST_STREAMING)
_data->surfaceDirty = true;
setFlag(QQuickItem::ItemHasContents, true);
#endif
}
VideoItem::~VideoItem()
{
#if defined(QGC_GST_STREAMING)
setSurface(0);
delete _data;
#endif
}
VideoSurface *VideoItem::surface() const
{
#if defined(QGC_GST_STREAMING)
return _data->surface.data();
#else
return nullptr;
#endif
}
void VideoItem::setSurface(VideoSurface *surface)
{
#if defined(QGC_GST_STREAMING)
if (_data->surface) {
_data->surface.data()->_data->items.remove(this);
}
_data->surface = surface;
_data->surfaceDirty = true;
if (_data->surface) {
_data->surface.data()->_data->items.insert(this);
}
#else
Q_UNUSED(surface)
#endif
}
#if defined(QGC_GST_STREAMING)
QSGGeometry* VideoItem::_createDefaultGeometry(QRectF& rectBound)
{
QSGGeometry *geometry = new QSGGeometry(QSGGeometry::defaultAttributes_Point2D(), 4);
geometry->vertexDataAsPoint2D()[0].set(rectBound.x(), rectBound.y());
geometry->vertexDataAsPoint2D()[1].set(rectBound.x(), rectBound.height());
geometry->vertexDataAsPoint2D()[2].set(rectBound.width(), rectBound.y());
geometry->vertexDataAsPoint2D()[3].set(rectBound.width(), rectBound.height());
return geometry;
}
QSGNode* VideoItem::updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData*)
{
QRectF r = boundingRect();
QSGNode* newNode = nullptr;
if (_data->surfaceDirty) {
delete oldNode;
oldNode = nullptr;
_data->surfaceDirty = false;
}
if (!_data->surface || _data->surface.data()->_data->videoSink == nullptr) {
if (!oldNode) {
QSGFlatColorMaterial *material = new QSGFlatColorMaterial;
material->setColor(Qt::black);
QSGGeometryNode *node = new QSGGeometryNode;
node->setMaterial(material);
node->setFlag(QSGNode::OwnsMaterial);
node->setFlag(QSGNode::OwnsGeometry);
newNode = node;
_data->targetArea = QRectF(); //force geometry to be set
} else {
newNode = oldNode;
}
if (r != _data->targetArea) {
QSGGeometryNode *node = static_cast<QSGGeometryNode*>(newNode);
node->setGeometry(_createDefaultGeometry(r));
_data->targetArea = r;
}
} else {
g_signal_emit_by_name(_data->surface.data()->_data->videoSink, "update-node", (void*)oldNode, r.x(), r.y(), r.width(), r.height(), (void**)&newNode);
}
// Sometimes we can still end up here with no geometry when gstreamer fails to create it for whatever reason. If that happens it can
// cause crashes.
QSGGeometryNode *node = static_cast<QSGGeometryNode*>(newNode);
if (node->geometry() == nullptr) {
qDebug() << "Creating default geom";
node->setGeometry(_createDefaultGeometry(r));
}
return newNode;
}
#endif
/****************************************************************************
*
* (c) 2009-2020 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
*
* QGroundControl is licensed according to the terms in the file
* COPYING.md in the root of the source code directory.
*
****************************************************************************/
/**
* @file
* @brief QGC Video Item
* @author Gus Grubba <gus@auterion.com>
*/
#pragma once
#include <QtQuick/QQuickItem>
#include "VideoSurface.h"
class QSGGeometry;
class VideoItem : public QQuickItem
{
Q_OBJECT
Q_DISABLE_COPY(VideoItem)
Q_PROPERTY(VideoSurface* surface READ surface WRITE setSurface)
public:
explicit VideoItem(QQuickItem *parent = 0);
virtual ~VideoItem();
VideoSurface *surface() const;
void setSurface(VideoSurface *surface);
protected:
#if defined(QGC_GST_STREAMING)
/*! Reimplemented from QQuickItem. */
virtual QSGNode* updatePaintNode(QSGNode *oldNode, UpdatePaintNodeData *updatePaintNodeData);
#endif
private:
#if defined(QGC_GST_STREAMING)
QSGGeometry* _createDefaultGeometry(QRectF& rectBound);
struct Private;
Private* const _data;
#endif
};
......@@ -64,24 +64,23 @@ VideoReceiver::VideoReceiver(QObject* parent)
, _pipeline(nullptr)
, _pipelineStopRec(nullptr)
, _videoSink(nullptr)
, _lastFrameId(G_MAXUINT64)
, _lastFrameTime(0)
, _restart_time_ms(1389)
, _socket(nullptr)
, _serverPresent(false)
, _tcpTestInterval_ms(5000)
, _udpReconnect_us(5000000)
#endif
, _videoSurface(nullptr)
, _videoRunning(false)
, _showFullScreen(false)
, _videoSettings(nullptr)
, _hwDecoderName(nullptr)
, _swDecoderName("avdec_h264")
{
_videoSurface = new VideoSurface;
_videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings();
#if defined(QGC_GST_STREAMING)
setVideoDecoder(H264_SW);
_setVideoSink(_videoSurface->videoSink());
_restart_timer.setSingleShot(true);
connect(&_restart_timer, &QTimer::timeout, this, &VideoReceiver::_restart_timeout);
_tcp_timer.setSingleShot(true);
......@@ -98,32 +97,9 @@ VideoReceiver::~VideoReceiver()
{
#if defined(QGC_GST_STREAMING)
stop();
if(_socket) {
delete _socket;
_socket = nullptr;
}
if (_videoSink) {
gst_object_unref(_videoSink);
}
setVideoSink(nullptr);
#endif
if(_videoSurface)
delete _videoSurface;
}
#if defined(QGC_GST_STREAMING)
void
VideoReceiver::_setVideoSink(GstElement* sink)
{
if (_videoSink) {
gst_object_unref(_videoSink);
_videoSink = nullptr;
}
if (sink) {
_videoSink = sink;
gst_object_ref_sink(_videoSink);
}
}
#endif
//-----------------------------------------------------------------------------
void
......@@ -287,6 +263,9 @@ VideoReceiver::start()
return;
}
_lastFrameId = G_MAXUINT64;
_lastFrameTime = 0;
bool running = false;
bool pipelineUp = false;
......@@ -458,11 +437,21 @@ VideoReceiver::start()
// If we failed before adding items to the pipeline, then clean up
if (!pipelineUp) {
if (queue1 != nullptr) {
gst_object_unref(queue1);
queue1 = nullptr;
}
if (decoder != nullptr) {
gst_object_unref(decoder);
decoder = nullptr;
}
if (queue != nullptr) {
gst_object_unref(queue);
queue = nullptr;
}
if (parser != nullptr) {
gst_object_unref(parser);
parser = nullptr;
......@@ -480,13 +469,9 @@ VideoReceiver::start()
if (_tee != nullptr) {
gst_object_unref(_tee);
dataSource = nullptr;
_tee = nullptr;
}
if (queue != nullptr) {
gst_object_unref(queue);
dataSource = nullptr;
}
}
_running = false;
......@@ -552,7 +537,6 @@ VideoReceiver::_shutdownPipeline() {
bus = nullptr;
}
gst_element_set_state(_pipeline, GST_STATE_NULL);
gst_bin_remove(GST_BIN(_pipeline), _videoSink);
gst_object_unref(_pipeline);
_pipeline = nullptr;
delete _sink;
......@@ -712,6 +696,38 @@ VideoReceiver::setVideoDecoder(VideoEncoding encoding)
}
}
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
void
VideoReceiver::setVideoSink(GstElement* videoSink)
{
if(_pipeline != nullptr) {
qCDebug(VideoReceiverLog) << "Video receiver pipeline is active, video sink change is not possible";
return;
}
if (_videoSink != nullptr) {
gst_object_unref(_videoSink);
_videoSink = nullptr;
}
if (videoSink != nullptr) {
_videoSink = videoSink;
gst_object_ref(_videoSink);
GstPad* pad = gst_element_get_static_pad(_videoSink, "sink");
if (pad != nullptr) {
gst_pad_add_probe(pad, (GstPadProbeType)(GST_PAD_PROBE_TYPE_BUFFER), _videoSinkProbe, this, nullptr);
gst_object_unref(pad);
pad = nullptr;
} else {
qCDebug(VideoReceiverLog) << "Unable to find sink pad of video sink";
}
}
}
#endif
//-----------------------------------------------------------------------------
// When we finish our pipeline will look like this:
//
......@@ -923,6 +939,30 @@ VideoReceiver::_unlinkCallBack(GstPad* pad, GstPadProbeInfo* info, gpointer user
}
#endif
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
GstPadProbeReturn
VideoReceiver::_videoSinkProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data)
{
Q_UNUSED(pad);
if(info != nullptr && user_data != nullptr) {
VideoReceiver* pThis = static_cast<VideoReceiver*>(user_data);
pThis->_noteVideoSinkFrame();
}
return GST_PAD_PROBE_OK;
}
#endif
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
void
VideoReceiver::_noteVideoSinkFrame()
{
_lastFrameTime = time(nullptr);
}
#endif
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
GstPadProbeReturn
......@@ -971,41 +1011,39 @@ void
VideoReceiver::_updateTimer()
{
#if defined(QGC_GST_STREAMING)
if(_videoSurface) {
if(_stopping || _starting) {
return;
}
if(_streaming) {
if(!_videoRunning) {
_videoSurface->setLastFrame(0);
_videoRunning = true;
emit videoRunningChanged();
}
} else {
if(_videoRunning) {
_videoRunning = false;
emit videoRunningChanged();
}
if(_stopping || _starting) {
return;
}
if(_streaming) {
if(!_videoRunning) {
_videoRunning = true;
emit videoRunningChanged();
}
} else {
if(_videoRunning) {
uint32_t timeout = 1;
if(qgcApp()->toolbox() && qgcApp()->toolbox()->settingsManager()) {
timeout = _videoSettings->rtspTimeout()->rawValue().toUInt();
}
time_t elapsed = 0;
time_t lastFrame = _videoSurface->lastFrame();
if(lastFrame != 0) {
elapsed = time(nullptr) - _videoSurface->lastFrame();
}
if(elapsed > static_cast<time_t>(timeout) && _videoSurface) {
stop();
// We want to start it back again with _updateTimer
_stop = false;
}
} else {
if(!_stop && _running && !_uri.isEmpty() && _videoSettings->streamEnabled()->rawValue().toBool()) {
start();
}
_videoRunning = false;
emit videoRunningChanged();
}
}
if(_videoRunning) {
uint32_t timeout = 1;
if(qgcApp()->toolbox() && qgcApp()->toolbox()->settingsManager()) {
timeout = _videoSettings->rtspTimeout()->rawValue().toUInt();
}
const time_t now = time(nullptr);
if(now - _lastFrameTime > timeout) {
stop();
// We want to start it back again with _updateTimer
_stop = false;
}
} else {
// FIXME: AV: if pipeline is _running but not _streaming for some time then we need to restart
if(!_stop && !_running && !_uri.isEmpty() && _videoSettings->streamEnabled()->rawValue().toBool()) {
start();
}
}
#endif
......
......@@ -20,8 +20,6 @@
#include <QTimer>
#include <QTcpSocket>