VideoReceiver.cc 32.4 KB
Newer Older
1 2
/****************************************************************************
 *
3
 *   (c) 2009-2019 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
4 5 6 7 8
 *
 * QGroundControl is licensed according to the terms in the file
 * COPYING.md in the root of the source code directory.
 *
 ****************************************************************************/
Gus Grubba's avatar
Gus Grubba committed
9 10 11 12 13 14 15 16 17


/**
 * @file
 *   @brief QGC Video Receiver
 *   @author Gus Grubba <mavlink@grubba.com>
 */

#include "VideoReceiver.h"
18 19
#include "SettingsManager.h"
#include "QGCApplication.h"
20
#include "VideoManager.h"
21 22 23
#ifdef QGC_GST_TAISYNC_ENABLED
#include "TaisyncHandler.h"
#endif
Gus Grubba's avatar
Gus Grubba committed
24
#include <QDebug>
25
#include <QUrl>
26 27
#include <QDir>
#include <QDateTime>
28
#include <QSysInfo>
29

30 31
QGC_LOGGING_CATEGORY(VideoReceiverLog, "VideoReceiverLog")

32 33
#if defined(QGC_GST_STREAMING)

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
static const char* kVideoExtensions[] =
{
    "mkv",
    "mov",
    "mp4"
};

static const char* kVideoMuxes[] =
{
    "matroskamux",
    "qtmux",
    "mp4mux"
};

#define NUM_MUXES (sizeof(kVideoMuxes) / sizeof(char*))

50 51 52
#endif


Gus Grubba's avatar
Gus Grubba committed
53 54
VideoReceiver::VideoReceiver(QObject* parent)
    : QObject(parent)
55
#if defined(QGC_GST_STREAMING)
56
    , _running(false)
57
    , _recording(false)
58
    , _streaming(false)
59 60
    , _starting(false)
    , _stopping(false)
61
    , _stop(true)
Gus Grubba's avatar
Gus Grubba committed
62 63 64 65 66
    , _sink(nullptr)
    , _tee(nullptr)
    , _pipeline(nullptr)
    , _pipelineStopRec(nullptr)
    , _videoSink(nullptr)
67
    , _restart_time_ms(1389)
68
    , _udpReconnect_us(5000000)
69
#endif
Gus Grubba's avatar
Gus Grubba committed
70
    , _videoSurface(nullptr)
71 72
    , _videoRunning(false)
    , _showFullScreen(false)
Gus Grubba's avatar
Gus Grubba committed
73
    , _videoSettings(nullptr)
74 75
    , _hwDecoderName(nullptr)
    , _swDecoderName("avdec_h264")
Gus Grubba's avatar
Gus Grubba committed
76
{
77
    _videoSurface = new VideoSurface;
Gus Grubba's avatar
Gus Grubba committed
78
    _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings();
79
#if defined(QGC_GST_STREAMING)
80
    setVideoDecoder(H264_SW);
81
    _setVideoSink(_videoSurface->videoSink());
82 83
    _restart_timer.setSingleShot(true);
    connect(&_restart_timer, &QTimer::timeout, this, &VideoReceiver::_restart_timeout);
84 85 86
    connect(this, &VideoReceiver::msgErrorReceived, this, &VideoReceiver::_handleError);
    connect(this, &VideoReceiver::msgEOSReceived, this, &VideoReceiver::_handleEOS);
    connect(this, &VideoReceiver::msgStateChangedReceived, this, &VideoReceiver::_handleStateChanged);
87 88
    connect(&_frameTimer, &QTimer::timeout, this, &VideoReceiver::_updateTimer);
    _frameTimer.start(1000);
89
#endif
Gus Grubba's avatar
Gus Grubba committed
90 91 92 93
}

VideoReceiver::~VideoReceiver()
{
94
#if defined(QGC_GST_STREAMING)
95
    stop();
96 97 98
    if (_videoSink) {
        gst_object_unref(_videoSink);
    }
99
#endif
100 101
    if(_videoSurface)
        delete _videoSurface;
Gus Grubba's avatar
Gus Grubba committed
102 103
}

104
#if defined(QGC_GST_STREAMING)
105 106
void
VideoReceiver::_setVideoSink(GstElement* sink)
Gus Grubba's avatar
Gus Grubba committed
107 108 109
{
    if (_videoSink) {
        gst_object_unref(_videoSink);
Gus Grubba's avatar
Gus Grubba committed
110
        _videoSink = nullptr;
Gus Grubba's avatar
Gus Grubba committed
111 112 113 114 115 116
    }
    if (sink) {
        _videoSink = sink;
        gst_object_ref_sink(_videoSink);
    }
}
117
#endif
Gus Grubba's avatar
Gus Grubba committed
118

119 120 121 122 123 124 125 126 127
//-----------------------------------------------------------------------------
void
VideoReceiver::grabImage(QString imageFile)
{
    _imageFile = imageFile;
    emit imageFileChanged();
}

//-----------------------------------------------------------------------------
128
#if defined(QGC_GST_STREAMING)
129 130
static void
newPadCB(GstElement* element, GstPad* pad, gpointer data)
131
{
132
    gchar* name = gst_pad_get_name(pad);
133
    //g_print("A new pad %s was created\n", name);
134 135
    GstCaps* p_caps = gst_pad_get_pad_template_caps (pad);
    gchar* description = gst_caps_to_string(p_caps);
136
    qCDebug(VideoReceiverLog) << p_caps << ", " << description;
137
    g_free(description);
138 139
    GstElement* sink = GST_ELEMENT(data);
    if(gst_element_link_pads(element, name, sink, "sink") == false)
140 141 142 143
        qCritical() << "newPadCB : failed to link elements\n";
    g_free(name);
}

144 145
//-----------------------------------------------------------------------------
void
146
VideoReceiver::_restart_timeout()
147
{
148
    qgcApp()->toolbox()->videoManager()->restartVideo();
149
}
150
#endif
151

152
//-----------------------------------------------------------------------------
153 154 155 156 157 158 159 160
// When we finish our pipeline will look like this:
//
//                                   +-->queue-->decoder-->_videosink
//                                   |
//    datasource-->demux-->parser-->tee
//                                   ^
//                                   |
//                                   +-Here we will later link elements for recording
161 162
void
VideoReceiver::start()
Gus Grubba's avatar
Gus Grubba committed
163
{
164 165 166 167
    if (_uri.isEmpty()) {
        return;
    }
    qCDebug(VideoReceiverLog) << "start():" << _uri;
Gus Grubba's avatar
Gus Grubba committed
168
    if(qgcApp()->runningUnitTests()) {
169 170
        return;
    }
171 172
    if(!_videoSettings->streamEnabled()->rawValue().toBool() ||
       !_videoSettings->streamConfigured()) {
173 174 175
        qCDebug(VideoReceiverLog) << "start() but not enabled/configured";
        return;
    }
176

177
#if defined(QGC_GST_STREAMING)
178
    _stop = false;
179

180
#if defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__))
181
    //-- Taisync on iOS or Android sends a raw h.264 stream
182 183 184 185
    bool isTaisyncUSB = qgcApp()->toolbox()->videoManager()->isTaisync();
#else
    bool isTaisyncUSB = false;
#endif
Gus Grubba's avatar
Gus Grubba committed
186 187
    bool isUdp264   = _uri.contains("udp://")  && !isTaisyncUSB;
    bool isUdp265   = _uri.contains("udp265://")  && !isTaisyncUSB;
188 189
    bool isTCP      = _uri.contains("tcp://")  && !isTaisyncUSB;
    bool isMPEGTS   = _uri.contains("mpegts://")  && !isTaisyncUSB;
190 191

    if (!isTaisyncUSB && _uri.isEmpty()) {
Gus Grubba's avatar
Gus Grubba committed
192 193 194
        qCritical() << "VideoReceiver::start() failed because URI is not specified";
        return;
    }
Gus Grubba's avatar
Gus Grubba committed
195
    if (_videoSink == nullptr) {
Gus Grubba's avatar
Gus Grubba committed
196 197 198
        qCritical() << "VideoReceiver::start() failed because video sink is not set";
        return;
    }
199 200 201 202
    if(_running) {
        qCDebug(VideoReceiverLog) << "Already running!";
        return;
    }
203 204 205 206 207
    if (isUdp264) {
        setVideoDecoder(H264_HW);
    } else if (isUdp265) {
        setVideoDecoder(H265_HW);
    }
Gus Grubba's avatar
Gus Grubba committed
208

209
    _starting = true;
210

211
    bool running    = false;
212
    bool pipelineUp = false;
Gus Grubba's avatar
Gus Grubba committed
213

Gus Grubba's avatar
Gus Grubba committed
214 215 216 217 218 219 220
    GstElement*     dataSource  = nullptr;
    GstCaps*        caps        = nullptr;
    GstElement*     demux       = nullptr;
    GstElement*     parser      = nullptr;
    GstElement*     queue       = nullptr;
    GstElement*     decoder     = nullptr;
    GstElement*     queue1      = nullptr;
221

Gus Grubba's avatar
Gus Grubba committed
222
    do {
Gus Grubba's avatar
Gus Grubba committed
223
        if ((_pipeline = gst_pipeline_new("receiver")) == nullptr) {
224
            qCritical() << "VideoReceiver::start() failed. Error with gst_pipeline_new()";
Gus Grubba's avatar
Gus Grubba committed
225 226 227
            break;
        }

Gus Grubba's avatar
Gus Grubba committed
228
        if(isUdp264 || isUdp265 || isMPEGTS || isTaisyncUSB) {
229
            dataSource = gst_element_factory_make("udpsrc", "udp-source");
230 231
        } else if(isTCP) {
            dataSource = gst_element_factory_make("tcpclientsrc", "tcpclient-source");
232 233
        } else {
            dataSource = gst_element_factory_make("rtspsrc", "rtsp-source");
Gus Grubba's avatar
Gus Grubba committed
234 235
        }

236 237
        if (!dataSource) {
            qCritical() << "VideoReceiver::start() failed. Error with data source for gst_element_factory_make()";
Gus Grubba's avatar
Gus Grubba committed
238 239 240
            break;
        }

Gus Grubba's avatar
Gus Grubba committed
241
        if(isUdp264) {
Gus Grubba's avatar
Gus Grubba committed
242
            if ((caps = gst_caps_from_string("application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264")) == nullptr) {
243 244 245
                qCritical() << "VideoReceiver::start() failed. Error with gst_caps_from_string()";
                break;
            }
246
            g_object_set(static_cast<gpointer>(dataSource), "uri", qPrintable(_uri), "caps", caps, nullptr);
Gus Grubba's avatar
Gus Grubba committed
247
        } else if(isUdp265) {
248 249 250 251 252
            if ((caps = gst_caps_from_string("application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H265")) == nullptr) {
                qCritical() << "VideoReceiver::start() failed. Error with gst_caps_from_string()";
                break;
            }
            g_object_set(static_cast<gpointer>(dataSource), "uri", qPrintable(_uri.replace("udp265", "udp")), "caps", caps, nullptr);
253
#if  defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__))
254 255 256 257 258
        } else if(isTaisyncUSB) {
            QString uri = QString("0.0.0.0:%1").arg(TAISYNC_VIDEO_UDP_PORT);
            qCDebug(VideoReceiverLog) << "Taisync URI:" << uri;
            g_object_set(static_cast<gpointer>(dataSource), "port", TAISYNC_VIDEO_UDP_PORT, nullptr);
#endif
259 260
        } else if(isTCP) {
            QUrl url(_uri);
261
            g_object_set(static_cast<gpointer>(dataSource), "host", qPrintable(url.host()), "port", url.port(), nullptr );
262 263 264
        } else if(isMPEGTS) {
            QUrl url(_uri);
            g_object_set(static_cast<gpointer>(dataSource), "port", url.port(), nullptr);
265
        } else {
266
            g_object_set(static_cast<gpointer>(dataSource), "location", qPrintable(_uri), "latency", 17, "udp-reconnect", 1, "timeout", _udpReconnect_us, NULL);
267
        }
Gus Grubba's avatar
Gus Grubba committed
268

269 270
        if (isTCP || isMPEGTS) {
            if ((demux = gst_element_factory_make("tsdemux", "mpeg-ts-demuxer")) == nullptr) {
271 272 273 274
                qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('tsdemux')";
                break;
            }
        } else {
275
            if(!isTaisyncUSB) {
276 277
                if ((demux = gst_element_factory_make(_depayName, "rtp-depacketizer")) == nullptr) {
                   qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _depayName << "')";
278 279
                    break;
                }
280
            }
281 282
        }

283 284
        if ((parser = gst_element_factory_make(_parserName, "parser")) == nullptr) {
            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _parserName << "')";
Gus Grubba's avatar
Gus Grubba committed
285 286 287
            break;
        }

Gus Grubba's avatar
Gus Grubba committed
288
        if((_tee = gst_element_factory_make("tee", nullptr)) == nullptr)  {
289 290 291
            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('tee')";
            break;
        }
Gus Grubba's avatar
Gus Grubba committed
292

Gus Grubba's avatar
Gus Grubba committed
293
        if((queue = gst_element_factory_make("queue", nullptr)) == nullptr)  {
294 295
            // TODO: We may want to add queue2 max-size-buffers=1 to get lower latency
            //       We should compare gstreamer scripts to QGroundControl to determine the need
296
            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('queue')";
297 298
            break;
        }
299

300 301 302 303 304 305
        if (!_hwDecoderName || (decoder = gst_element_factory_make(_hwDecoderName, "decoder")) == nullptr) {
            qWarning() << "VideoReceiver::start() hardware decoding not available " << ((_hwDecoderName) ? _hwDecoderName : "");
            if ((decoder = gst_element_factory_make(_swDecoderName, "decoder")) == nullptr) {
                qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _swDecoderName << "')";
                break;
            }
306 307
        }

Gus Grubba's avatar
Gus Grubba committed
308
        if ((queue1 = gst_element_factory_make("queue", nullptr)) == nullptr) {
309 310 311 312
            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('queue') [1]";
            break;
        }

313 314 315 316 317
        if(isTaisyncUSB) {
            gst_bin_add_many(GST_BIN(_pipeline), dataSource, parser, _tee, queue, decoder, queue1, _videoSink, nullptr);
        } else {
            gst_bin_add_many(GST_BIN(_pipeline), dataSource, demux, parser, _tee, queue, decoder, queue1, _videoSink, nullptr);
        }
318
        pipelineUp = true;
319

Gus Grubba's avatar
Gus Grubba committed
320
        if(isUdp264 || isUdp265) {
321
            // Link the pipeline in front of the tee
Gus Grubba's avatar
Gus Grubba committed
322
            if(!gst_element_link_many(dataSource, demux, parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
323 324 325
                qCritical() << "Unable to link UDP elements.";
                break;
            }
326 327 328 329 330 331
        } else if(isTaisyncUSB) {
            // Link the pipeline in front of the tee
            if(!gst_element_link_many(dataSource, parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
                qCritical() << "Unable to link Taisync USB elements.";
                break;
            }
332
        } else if (isTCP || isMPEGTS) {
333
            if(!gst_element_link(dataSource, demux)) {
334
                qCritical() << "Unable to link TCP/MPEG-TS dataSource to Demux.";
335 336
                break;
            }
Gus Grubba's avatar
Gus Grubba committed
337
            if(!gst_element_link_many(parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
338
                qCritical() << "Unable to link TCP/MPEG-TS pipline to parser.";
339 340
                break;
            }
341
            g_signal_connect(demux, "pad-added", G_CALLBACK(newPadCB), parser);
342
        } else {
343
            g_signal_connect(dataSource, "pad-added", G_CALLBACK(newPadCB), demux);
Gus Grubba's avatar
Gus Grubba committed
344
            if(!gst_element_link_many(demux, parser, _tee, queue, decoder, _videoSink, nullptr)) {
345
                qCritical() << "Unable to link RTSP elements.";
346 347
                break;
            }
348 349
        }

Gus Grubba's avatar
Gus Grubba committed
350
        dataSource = demux = parser = queue = decoder = queue1 = nullptr;
Gus Grubba's avatar
Gus Grubba committed
351

Gus Grubba's avatar
Gus Grubba committed
352
        GstBus* bus = nullptr;
Gus Grubba's avatar
Gus Grubba committed
353

Gus Grubba's avatar
Gus Grubba committed
354
        if ((bus = gst_pipeline_get_bus(GST_PIPELINE(_pipeline))) != nullptr) {
355 356 357
            gst_bus_enable_sync_message_emission(bus);
            g_signal_connect(bus, "sync-message", G_CALLBACK(_onBusMessage), this);
            gst_object_unref(bus);
Gus Grubba's avatar
Gus Grubba committed
358
            bus = nullptr;
359
        }
Gus Grubba's avatar
Gus Grubba committed
360

361
        GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(_pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-paused");
Gus Grubba's avatar
Gus Grubba committed
362 363 364 365
        running = gst_element_set_state(_pipeline, GST_STATE_PLAYING) != GST_STATE_CHANGE_FAILURE;

    } while(0);

Gus Grubba's avatar
Gus Grubba committed
366
    if (caps != nullptr) {
Gus Grubba's avatar
Gus Grubba committed
367
        gst_caps_unref(caps);
Gus Grubba's avatar
Gus Grubba committed
368
        caps = nullptr;
Gus Grubba's avatar
Gus Grubba committed
369 370 371 372 373
    }

    if (!running) {
        qCritical() << "VideoReceiver::start() failed";

374
        // In newer versions, the pipeline will clean up all references that are added to it
Gus Grubba's avatar
Gus Grubba committed
375
        if (_pipeline != nullptr) {
376
            gst_object_unref(_pipeline);
Gus Grubba's avatar
Gus Grubba committed
377
            _pipeline = nullptr;
Gus Grubba's avatar
Gus Grubba committed
378 379
        }

380 381
        // If we failed before adding items to the pipeline, then clean up
        if (!pipelineUp) {
Gus Grubba's avatar
Gus Grubba committed
382
            if (decoder != nullptr) {
383
                gst_object_unref(decoder);
Gus Grubba's avatar
Gus Grubba committed
384
                decoder = nullptr;
385
            }
Gus Grubba's avatar
Gus Grubba committed
386

Gus Grubba's avatar
Gus Grubba committed
387
            if (parser != nullptr) {
388
                gst_object_unref(parser);
Gus Grubba's avatar
Gus Grubba committed
389
                parser = nullptr;
390
            }
Gus Grubba's avatar
Gus Grubba committed
391

Gus Grubba's avatar
Gus Grubba committed
392
            if (demux != nullptr) {
393
                gst_object_unref(demux);
Gus Grubba's avatar
Gus Grubba committed
394
                demux = nullptr;
395
            }
Gus Grubba's avatar
Gus Grubba committed
396

Gus Grubba's avatar
Gus Grubba committed
397
            if (dataSource != nullptr) {
398
                gst_object_unref(dataSource);
Gus Grubba's avatar
Gus Grubba committed
399
                dataSource = nullptr;
400
            }
401

Gus Grubba's avatar
Gus Grubba committed
402
            if (_tee != nullptr) {
403
                gst_object_unref(_tee);
Gus Grubba's avatar
Gus Grubba committed
404
                dataSource = nullptr;
405
            }
406

Gus Grubba's avatar
Gus Grubba committed
407
            if (queue != nullptr) {
408
                gst_object_unref(queue);
Gus Grubba's avatar
Gus Grubba committed
409
                dataSource = nullptr;
410
            }
Gus Grubba's avatar
Gus Grubba committed
411
        }
412 413 414

        _running = false;
    } else {
415
        GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(_pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-playing");
416 417
        _running = true;
        qCDebug(VideoReceiverLog) << "Running";
Gus Grubba's avatar
Gus Grubba committed
418
    }
419
    _starting = false;
420
#endif
Gus Grubba's avatar
Gus Grubba committed
421 422
}

423 424 425
//-----------------------------------------------------------------------------
void
VideoReceiver::stop()
Gus Grubba's avatar
Gus Grubba committed
426
{
427
    if(qgcApp() && qgcApp()->runningUnitTests()) {
428 429
        return;
    }
430
#if defined(QGC_GST_STREAMING)
431
    _stop = true;
432
    qCDebug(VideoReceiverLog) << "stop()";
433 434
    if(!_streaming) {
        _shutdownPipeline();
Gus Grubba's avatar
Gus Grubba committed
435
    } else if (_pipeline != nullptr && !_stopping) {
436 437 438 439
        qCDebug(VideoReceiverLog) << "Stopping _pipeline";
        gst_element_send_event(_pipeline, gst_event_new_eos());
        _stopping = true;
        GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(_pipeline));
440
        GstMessage* message = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, (GstMessageType)(GST_MESSAGE_EOS|GST_MESSAGE_ERROR));
441
        gst_object_unref(bus);
442 443 444 445 446 447
        if(GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) {
            _shutdownPipeline();
            qCritical() << "Error stopping pipeline!";
        } else if(GST_MESSAGE_TYPE(message) == GST_MESSAGE_EOS) {
            _handleEOS();
        }
448
        gst_message_unref(message);
Gus Grubba's avatar
Gus Grubba committed
449
    }
450
#endif
Gus Grubba's avatar
Gus Grubba committed
451 452
}

453 454 455
//-----------------------------------------------------------------------------
void
VideoReceiver::setUri(const QString & uri)
Gus Grubba's avatar
Gus Grubba committed
456 457 458 459
{
    _uri = uri;
}

460
//-----------------------------------------------------------------------------
461
#if defined(QGC_GST_STREAMING)
462 463
void
VideoReceiver::_shutdownPipeline() {
464 465 466 467
    if(!_pipeline) {
        qCDebug(VideoReceiverLog) << "No pipeline";
        return;
    }
Gus Grubba's avatar
Gus Grubba committed
468 469
    GstBus* bus = nullptr;
    if ((bus = gst_pipeline_get_bus(GST_PIPELINE(_pipeline))) != nullptr) {
470 471
        gst_bus_disable_sync_message_emission(bus);
        gst_object_unref(bus);
Gus Grubba's avatar
Gus Grubba committed
472
        bus = nullptr;
473 474 475 476
    }
    gst_element_set_state(_pipeline, GST_STATE_NULL);
    gst_bin_remove(GST_BIN(_pipeline), _videoSink);
    gst_object_unref(_pipeline);
Gus Grubba's avatar
Gus Grubba committed
477
    _pipeline = nullptr;
478
    delete _sink;
Gus Grubba's avatar
Gus Grubba committed
479
    _sink = nullptr;
480 481 482 483 484 485
    _streaming = false;
    _recording = false;
    _stopping = false;
    _running = false;
    emit recordingChanged();
}
486
#endif
487

488
//-----------------------------------------------------------------------------
489
#if defined(QGC_GST_STREAMING)
490 491
void
VideoReceiver::_handleError() {
492
    qCDebug(VideoReceiverLog) << "Gstreamer error!";
493 494
    // If there was an error we switch to software decoding only
    _tryWithHardwareDecoding = false;
495
    stop();
496
    _restart_timer.start(_restart_time_ms);
497 498 499
}
#endif

500
//-----------------------------------------------------------------------------
501
#if defined(QGC_GST_STREAMING)
502 503
void
VideoReceiver::_handleEOS() {
504 505
    if(_stopping) {
        _shutdownPipeline();
506
        qCDebug(VideoReceiverLog) << "Stopped";
507 508 509
    } else if(_recording && _sink->removing) {
        _shutdownRecordingBranch();
    } else {
510
        qWarning() << "VideoReceiver: Unexpected EOS!";
511
        _handleError();
Gus Grubba's avatar
Gus Grubba committed
512 513
    }
}
514
#endif
Gus Grubba's avatar
Gus Grubba committed
515

516
//-----------------------------------------------------------------------------
517
#if defined(QGC_GST_STREAMING)
518 519 520 521
void
VideoReceiver::_handleStateChanged() {
    if(_pipeline) {
        _streaming = GST_STATE(_pipeline) == GST_STATE_PLAYING;
522
        //qCDebug(VideoReceiverLog) << "State changed, _streaming:" << _streaming;
523
    }
524 525 526
}
#endif

527
//-----------------------------------------------------------------------------
528
#if defined(QGC_GST_STREAMING)
529 530
gboolean
VideoReceiver::_onBusMessage(GstBus* bus, GstMessage* msg, gpointer data)
Gus Grubba's avatar
Gus Grubba committed
531 532
{
    Q_UNUSED(bus)
Gus Grubba's avatar
Gus Grubba committed
533
    Q_ASSERT(msg != nullptr && data != nullptr);
Gus Grubba's avatar
Gus Grubba committed
534
    VideoReceiver* pThis = (VideoReceiver*)data;
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556

    switch(GST_MESSAGE_TYPE(msg)) {
    case(GST_MESSAGE_ERROR): {
        gchar* debug;
        GError* error;
        gst_message_parse_error(msg, &error, &debug);
        g_free(debug);
        qCritical() << error->message;
        g_error_free(error);
        pThis->msgErrorReceived();
    }
        break;
    case(GST_MESSAGE_EOS):
        pThis->msgEOSReceived();
        break;
    case(GST_MESSAGE_STATE_CHANGED):
        pThis->msgStateChangedReceived();
        break;
    default:
        break;
    }

Gus Grubba's avatar
Gus Grubba committed
557 558
    return TRUE;
}
559
#endif
560

561
//-----------------------------------------------------------------------------
562
#if defined(QGC_GST_STREAMING)
563 564
void
VideoReceiver::_cleanupOldVideos()
565
{
566
    //-- Only perform cleanup if storage limit is enabled
567
    if(_videoSettings->enableStorageLimit()->rawValue().toBool()) {
568 569 570 571 572 573 574 575
        QString savePath = qgcApp()->toolbox()->settingsManager()->appSettings()->videoSavePath();
        QDir videoDir = QDir(savePath);
        videoDir.setFilter(QDir::Files | QDir::Readable | QDir::NoSymLinks | QDir::Writable);
        videoDir.setSorting(QDir::Time);
        //-- All the movie extensions we support
        QStringList nameFilters;
        for(uint32_t i = 0; i < NUM_MUXES; i++) {
            nameFilters << QString("*.") + QString(kVideoExtensions[i]);
576
        }
577 578 579 580 581 582
        videoDir.setNameFilters(nameFilters);
        //-- get the list of videos stored
        QFileInfoList vidList = videoDir.entryInfoList();
        if(!vidList.isEmpty()) {
            uint64_t total   = 0;
            //-- Settings are stored using MB
583
            uint64_t maxSize = (_videoSettings->maxVideoSize()->rawValue().toUInt() * 1024 * 1024);
584 585 586 587 588 589 590 591 592 593 594 595
            //-- Compute total used storage
            for(int i = 0; i < vidList.size(); i++) {
                total += vidList[i].size();
            }
            //-- Remove old movies until max size is satisfied.
            while(total >= maxSize && !vidList.isEmpty()) {
                total -= vidList.last().size();
                qCDebug(VideoReceiverLog) << "Removing old video file:" << vidList.last().filePath();
                QFile file (vidList.last().filePath());
                file.remove();
                vidList.removeLast();
            }
596 597 598
        }
    }
}
599
#endif
600

601 602 603 604
//-----------------------------------------------------------------------------
void
VideoReceiver::setVideoDecoder(VideoEncoding encoding)
{
605 606 607 608 609 610 611 612
    /*
    #if defined(Q_OS_MAC)
        _hwDecoderName = "vtdec";
    #else
        _hwDecoderName = "vaapidecode";
    #endif
    */

613
    if (encoding == H265_HW || encoding == H265_SW) {
614
        _depayName  = "rtph265depay";
615
        _parserName = "h265parse";
616 617 618 619
#if defined(__android__)
        _hwDecoderName = "amcviddec-omxgooglehevcdecoder";
#endif
        _swDecoderName = "avdec_h265";
620
    } else {
621
        _depayName  = "rtph264depay";
622
        _parserName = "h264parse";
623 624 625 626
#if defined(__android__)
        _hwDecoderName = "amcviddec-omxgoogleh264decoder";
#endif
        _swDecoderName = "avdec_h264";
627 628
    }

629 630 631 632
    if (!_tryWithHardwareDecoding) {
        _hwDecoderName = nullptr;
    }
}
633

634
//-----------------------------------------------------------------------------
635 636 637 638 639 640 641 642 643 644 645
// When we finish our pipeline will look like this:
//
//                                   +-->queue-->decoder-->_videosink
//                                   |
//    datasource-->demux-->parser-->tee
//                                   |
//                                   |    +--------------_sink-------------------+
//                                   |    |                                      |
//   we are adding these elements->  +->teepad-->queue-->matroskamux-->_filesink |
//                                        |                                      |
//                                        +--------------------------------------+
646
void
647
VideoReceiver::startRecording(const QString &videoFile)
648
{
649
#if defined(QGC_GST_STREAMING)
650

651 652
    qCDebug(VideoReceiverLog) << "startRecording()";
    // exit immediately if we are already recording
Gus Grubba's avatar
Gus Grubba committed
653
    if(_pipeline == nullptr || _recording) {
654 655 656 657
        qCDebug(VideoReceiverLog) << "Already recording!";
        return;
    }

658
    uint32_t muxIdx = _videoSettings->recordingFormat()->rawValue().toUInt();
659 660 661 662 663 664 665 666
    if(muxIdx >= NUM_MUXES) {
        qgcApp()->showMessage(tr("Invalid video format defined."));
        return;
    }

    //-- Disk usage maintenance
    _cleanupOldVideos();

667
    _sink           = new Sink();
668
    _sink->teepad   = gst_element_get_request_pad(_tee, "src_%u");
Gus Grubba's avatar
Gus Grubba committed
669
    _sink->queue    = gst_element_factory_make("queue", nullptr);
670
    _sink->parse    = gst_element_factory_make(_parserName, nullptr);
Gus Grubba's avatar
Gus Grubba committed
671 672
    _sink->mux      = gst_element_factory_make(kVideoMuxes[muxIdx], nullptr);
    _sink->filesink = gst_element_factory_make("filesink", nullptr);
673 674
    _sink->removing = false;

675
    if(!_sink->teepad || !_sink->queue || !_sink->mux || !_sink->filesink || !_sink->parse) {
676 677 678 679
        qCritical() << "VideoReceiver::startRecording() failed to make _sink elements";
        return;
    }

680 681 682 683 684 685 686 687 688 689 690
    if(videoFile.isEmpty()) {
        QString savePath = qgcApp()->toolbox()->settingsManager()->appSettings()->videoSavePath();
        if(savePath.isEmpty()) {
            qgcApp()->showMessage(tr("Unabled to record video. Video save path must be specified in Settings."));
            return;
        }
        _videoFile = savePath + "/" + QDateTime::currentDateTime().toString("yyyy-MM-dd_hh.mm.ss") + "." + kVideoExtensions[muxIdx];
    } else {
        _videoFile = videoFile;
    }
    emit videoFileChanged();
691

692
    g_object_set(static_cast<gpointer>(_sink->filesink), "location", qPrintable(_videoFile), nullptr);
693
    qCDebug(VideoReceiverLog) << "New video file:" << _videoFile;
694 695

    gst_object_ref(_sink->queue);
696
    gst_object_ref(_sink->parse);
697 698 699
    gst_object_ref(_sink->mux);
    gst_object_ref(_sink->filesink);

700 701
    gst_bin_add_many(GST_BIN(_pipeline), _sink->queue, _sink->parse, _sink->mux, nullptr);
    gst_element_link_many(_sink->queue, _sink->parse, _sink->mux, nullptr);
702 703

    gst_element_sync_state_with_parent(_sink->queue);
704
    gst_element_sync_state_with_parent(_sink->parse);
705 706
    gst_element_sync_state_with_parent(_sink->mux);

707 708 709
    // Install a probe on the recording branch to drop buffers until we hit our first keyframe
    // When we hit our first keyframe, we can offset the timestamps appropriately according to the first keyframe time
    // This will ensure the first frame is a keyframe at t=0, and decoding can begin immediately on playback
710 711
    // Once we have this valid frame, we attach the filesink.
    // Attaching it here would cause the filesink to fail to preroll and to stall the pipeline for a few seconds.
712
    GstPad* probepad = gst_element_get_static_pad(_sink->queue, "src");
Gus Grubba's avatar
Gus Grubba committed
713
    gst_pad_add_probe(probepad, (GstPadProbeType)(GST_PAD_PROBE_TYPE_BUFFER /* | GST_PAD_PROBE_TYPE_BLOCK */), _keyframeWatch, this, nullptr); // to drop the buffer or to block the buffer?
714 715 716
    gst_object_unref(probepad);

    // Link the recording branch to the pipeline
717 718 719 720
    GstPad* sinkpad = gst_element_get_static_pad(_sink->queue, "sink");
    gst_pad_link(_sink->teepad, sinkpad);
    gst_object_unref(sinkpad);

721 722
    GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(_pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline-recording");

723 724 725
    _recording = true;
    emit recordingChanged();
    qCDebug(VideoReceiverLog) << "Recording started";
DonLakeFlyer's avatar
DonLakeFlyer committed
726 727
#else
    Q_UNUSED(videoFile)
728 729 730
#endif
}

731 732 733
//-----------------------------------------------------------------------------
void
VideoReceiver::stopRecording(void)
734
{
735
#if defined(QGC_GST_STREAMING)
736 737
    qCDebug(VideoReceiverLog) << "stopRecording()";
    // exit immediately if we are not recording
Gus Grubba's avatar
Gus Grubba committed
738
    if(_pipeline == nullptr || !_recording) {
739 740 741 742
        qCDebug(VideoReceiverLog) << "Not recording!";
        return;
    }
    // Wait for data block before unlinking
Gus Grubba's avatar
Gus Grubba committed
743
    gst_pad_add_probe(_sink->teepad, GST_PAD_PROBE_TYPE_IDLE, _unlinkCallBack, this, nullptr);
744 745 746
#endif
}

747
//-----------------------------------------------------------------------------
748 749 750 751 752 753
// This is only installed on the transient _pipelineStopRec in order
// to finalize a video file. It is not used for the main _pipeline.
// -EOS has appeared on the bus of the temporary pipeline
// -At this point all of the recoring elements have been flushed, and the video file has been finalized
// -Now we can remove the temporary pipeline and its elements
#if defined(QGC_GST_STREAMING)
754 755
void
VideoReceiver::_shutdownRecordingBranch()
756 757
{
    gst_bin_remove(GST_BIN(_pipelineStopRec), _sink->queue);
758
    gst_bin_remove(GST_BIN(_pipelineStopRec), _sink->parse);
759 760 761 762 763
    gst_bin_remove(GST_BIN(_pipelineStopRec), _sink->mux);
    gst_bin_remove(GST_BIN(_pipelineStopRec), _sink->filesink);

    gst_element_set_state(_pipelineStopRec, GST_STATE_NULL);
    gst_object_unref(_pipelineStopRec);
Gus Grubba's avatar
Gus Grubba committed
764
    _pipelineStopRec = nullptr;
765

766 767 768 769
    gst_element_set_state(_sink->filesink,  GST_STATE_NULL);
    gst_element_set_state(_sink->parse,     GST_STATE_NULL);
    gst_element_set_state(_sink->mux,       GST_STATE_NULL);
    gst_element_set_state(_sink->queue,     GST_STATE_NULL);
770 771

    gst_object_unref(_sink->queue);
772
    gst_object_unref(_sink->parse);
773 774 775 776
    gst_object_unref(_sink->mux);
    gst_object_unref(_sink->filesink);

    delete _sink;
Gus Grubba's avatar
Gus Grubba committed
777
    _sink = nullptr;
778
    _recording = false;
779

780 781 782 783 784
    emit recordingChanged();
    qCDebug(VideoReceiverLog) << "Recording Stopped";
}
#endif

785
//-----------------------------------------------------------------------------
786 787 788 789 790
// -Unlink the recording branch from the tee in the main _pipeline
// -Create a second temporary pipeline, and place the recording branch elements into that pipeline
// -Setup watch and handler for EOS event on the temporary pipeline's bus
// -Send an EOS event at the beginning of that pipeline
#if defined(QGC_GST_STREAMING)
791 792
void
VideoReceiver::_detachRecordingBranch(GstPadProbeInfo* info)
793 794 795 796
{
    Q_UNUSED(info)

    // Also unlinks and unrefs
Gus Grubba's avatar
Gus Grubba committed
797
    gst_bin_remove_many(GST_BIN(_pipeline), _sink->queue, _sink->parse, _sink->mux, _sink->filesink, nullptr);
798 799 800 801 802 803 804 805 806

    // Give tee its pad back
    gst_element_release_request_pad(_tee, _sink->teepad);
    gst_object_unref(_sink->teepad);

    // Create temporary pipeline
    _pipelineStopRec = gst_pipeline_new("pipeStopRec");

    // Put our elements from the recording branch into the temporary pipeline
Gus Grubba's avatar
Gus Grubba committed
807 808
    gst_bin_add_many(GST_BIN(_pipelineStopRec), _sink->queue, _sink->parse, _sink->mux, _sink->filesink, nullptr);
    gst_element_link_many(_sink->queue, _sink->parse, _sink->mux, _sink->filesink, nullptr);
809

810 811 812
    // Add handler for EOS event
    GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(_pipelineStopRec));
    gst_bus_enable_sync_message_emission(bus);
813
    g_signal_connect(bus, "sync-message", G_CALLBACK(_onBusMessage), this);
814
    gst_object_unref(bus);
815 816 817 818 819 820 821 822 823 824 825 826 827

    if(gst_element_set_state(_pipelineStopRec, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
        qCDebug(VideoReceiverLog) << "problem starting _pipelineStopRec";
    }

    // Send EOS at the beginning of the pipeline
    GstPad* sinkpad = gst_element_get_static_pad(_sink->queue, "sink");
    gst_pad_send_event(sinkpad, gst_event_new_eos());
    gst_object_unref(sinkpad);
    qCDebug(VideoReceiverLog) << "Recording branch unlinked";
}
#endif

828
//-----------------------------------------------------------------------------
829
#if defined(QGC_GST_STREAMING)
830 831
GstPadProbeReturn
VideoReceiver::_unlinkCallBack(GstPad* pad, GstPadProbeInfo* info, gpointer user_data)
832 833
{
    Q_UNUSED(pad);
Gus Grubba's avatar
Gus Grubba committed
834 835
    if(info != nullptr && user_data != nullptr) {
        VideoReceiver* pThis = static_cast<VideoReceiver*>(user_data);
836 837 838 839 840
        // We will only act once
        if(g_atomic_int_compare_and_exchange(&pThis->_sink->removing, FALSE, TRUE)) {
            pThis->_detachRecordingBranch(info);
        }
    }
841 842 843
    return GST_PAD_PROBE_REMOVE;
}
#endif
844

845 846 847 848 849 850
//-----------------------------------------------------------------------------
#if defined(QGC_GST_STREAMING)
GstPadProbeReturn
VideoReceiver::_keyframeWatch(GstPad* pad, GstPadProbeInfo* info, gpointer user_data)
{
    Q_UNUSED(pad);
Gus Grubba's avatar
Gus Grubba committed
851
    if(info != nullptr && user_data != nullptr) {
852 853 854 855
        GstBuffer* buf = gst_pad_probe_info_get_buffer(info);
        if(GST_BUFFER_FLAG_IS_SET(buf, GST_BUFFER_FLAG_DELTA_UNIT)) { // wait for a keyframe
            return GST_PAD_PROBE_DROP;
        } else {
Gus Grubba's avatar
Gus Grubba committed
856
            VideoReceiver* pThis = static_cast<VideoReceiver*>(user_data);
857
            // reset the clock
Jacob Walser's avatar
Jacob Walser committed
858
            GstClock* clock = gst_pipeline_get_clock(GST_PIPELINE(pThis->_pipeline));
859 860 861 862 863
            GstClockTime time = gst_clock_get_time(clock);
            gst_object_unref(clock);
            gst_element_set_base_time(pThis->_pipeline, time); // offset pipeline timestamps to start at zero again
            buf->dts = 0; // The offset will not apply to this current buffer, our first frame, timestamp is zero
            buf->pts = 0;
864 865 866 867 868 869

            // Add the filesink once we have a valid I-frame
            gst_bin_add_many(GST_BIN(pThis->_pipeline), pThis->_sink->filesink, nullptr);
            gst_element_link_many(pThis->_sink->mux, pThis->_sink->filesink, nullptr);
            gst_element_sync_state_with_parent(pThis->_sink->filesink);

870
            qCDebug(VideoReceiverLog) << "Got keyframe, stop dropping buffers";
871
            pThis->gotFirstRecordingKeyFrame();
872 873 874 875 876 877 878
        }
    }

    return GST_PAD_PROBE_REMOVE;
}
#endif

879 880 881 882 883 884
//-----------------------------------------------------------------------------
void
VideoReceiver::_updateTimer()
{
#if defined(QGC_GST_STREAMING)
    if(_videoSurface) {
885
        if(_stopping || _starting) {
886 887
            return;
        }
888
        if(_streaming) {
889 890 891 892 893 894 895 896 897 898 899 900
            if(!_videoRunning) {
                _videoSurface->setLastFrame(0);
                _videoRunning = true;
                emit videoRunningChanged();
            }
        } else {
            if(_videoRunning) {
                _videoRunning = false;
                emit videoRunningChanged();
            }
        }
        if(_videoRunning) {
901 902
            uint32_t timeout = 1;
            if(qgcApp()->toolbox() && qgcApp()->toolbox()->settingsManager()) {
903
                timeout = _videoSettings->rtspTimeout()->rawValue().toUInt();
904
            }
905 906 907
            time_t elapsed = 0;
            time_t lastFrame = _videoSurface->lastFrame();
            if(lastFrame != 0) {
Gus Grubba's avatar
Gus Grubba committed
908
                elapsed = time(nullptr) - _videoSurface->lastFrame();
909
            }
Gus Grubba's avatar
Gus Grubba committed
910
            if(elapsed > static_cast<time_t>(timeout) && _videoSurface) {
911
                stop();
912 913
                // We want to start it back again with _updateTimer
                _stop = false;
914 915
            }
        } else {
916
            if(!_stop && _running && !_uri.isEmpty() && _videoSettings->streamEnabled()->rawValue().toBool()) {
917 918 919 920 921 922 923
                start();
            }
        }
    }
#endif
}