Switch to decodebin and use decoder with highest rank; Use GL context provided...

Switch to decodebin and use decoder with highest rank; Use GL context provided by qmlglsink; Fix JNI code for AndroidMedia initialization

android.pri

@@ -16,7 +16,11 @@ OTHER_FILES += \
     $$PWD/android/src/com/hoho/android/usbserial/driver/UsbSerialRuntimeException.java \
     $$PWD/android/src/org/mavlink/qgroundcontrol/QGCActivity.java \
     $$PWD/android/src/org/mavlink/qgroundcontrol/UsbIoManager.java \
-    $$PWD/android/src/org/mavlink/qgroundcontrol/TaiSync.java
+    $$PWD/android/src/org/mavlink/qgroundcontrol/TaiSync.java \
+    $$PWD/android/src/org/freedesktop/gstreamer/androidmedia/GstAhcCallback.java \
+    $$PWD/android/src/org/freedesktop/gstreamer/androidmedia/GstAhsCallback.java \
+    $$PWD/android/src/org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java
+
 
 DISTFILES += \
     $$PWD/android/gradle/wrapper/gradle-wrapper.jar \

android/src/org/freedesktop/gstreamer/androidmedia/GstAhcCallback.java

+/*
+ * Copyright (C) 2012, Collabora Ltd.
+ *   Author: Youness Alaoui
+ *
+ * Copyright (C) 2015, Collabora Ltd.
+ *   Author: Justin Kim <justin.kim@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+package org.freedesktop.gstreamer.androidmedia;
+
+import android.hardware.Camera;
+
+public class GstAhcCallback implements Camera.PreviewCallback,
+                                       Camera.ErrorCallback,
+                                       Camera.AutoFocusCallback {
+    public long mUserData;
+    public long mCallback;
+
+    public static native void gst_ah_camera_on_preview_frame(byte[] data, Camera camera,
+                                                             long callback, long user_data);
+    public static native void gst_ah_camera_on_error(int error, Camera camera,
+                                                     long callback, long user_data);
+    public static native void gst_ah_camera_on_auto_focus(boolean success, Camera camera,
+                                                             long callback, long user_data);
+
+    public GstAhcCallback(long callback, long user_data) {
+        mCallback = callback;
+        mUserData = user_data;
+    }
+
+    @Override
+    public void onPreviewFrame(byte[] data, Camera camera) {
+        gst_ah_camera_on_preview_frame(data, camera, mCallback, mUserData);
+    }
+
+    @Override
+    public void onError(int error, Camera camera) {
+        gst_ah_camera_on_error(error, camera, mCallback, mUserData);
+    }
+
+    @Override
+    public void onAutoFocus(boolean success, Camera camera) {
+        gst_ah_camera_on_auto_focus(success, camera, mCallback, mUserData);
+    }
+}

android/src/org/freedesktop/gstreamer/androidmedia/GstAhsCallback.java

+/*
+ * Copyright (C) 2016 SurroundIO
+ *   Author: Martin Kelly <martin@surround.io>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+package org.freedesktop.gstreamer.androidmedia;
+
+import android.hardware.Sensor;
+import android.hardware.SensorEvent;
+import android.hardware.SensorEventListener;
+
+public class GstAhsCallback implements SensorEventListener {
+    public long mUserData;
+    public long mSensorCallback;
+    public long mAccuracyCallback;
+
+    public static native void gst_ah_sensor_on_sensor_changed(SensorEvent event,
+                                                              long callback, long user_data);
+    public static native void gst_ah_sensor_on_accuracy_changed(Sensor sensor, int accuracy,
+                                                                long callback, long user_data);
+
+    public GstAhsCallback(long sensor_callback,
+        long accuracy_callback, long user_data) {
+        mSensorCallback = sensor_callback;
+        mAccuracyCallback = accuracy_callback;
+        mUserData = user_data;
+    }
+
+    @Override
+    public void onSensorChanged(SensorEvent event) {
+      gst_ah_sensor_on_sensor_changed(event, mSensorCallback, mUserData);
+    }
+
+    @Override
+    public void onAccuracyChanged(Sensor sensor, int accuracy) {
+      gst_ah_sensor_on_accuracy_changed(sensor, accuracy,
+          mAccuracyCallback, mUserData);
+    }
+}

android/src/org/freedesktop/gstreamer/androidmedia/GstAmcOnFrameAvailableListener.java

+/*
+ * Copyright (C) 2015, Collabora Ltd.
+ *   Author: Matthieu Bouron <matthieu.bouron@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ *
+ */
+
+package org.freedesktop.gstreamer.androidmedia;
+
+import android.graphics.SurfaceTexture;
+import android.graphics.SurfaceTexture.OnFrameAvailableListener;
+
+public class GstAmcOnFrameAvailableListener implements OnFrameAvailableListener
+{
+    private long context = 0;
+
+    public synchronized void onFrameAvailable (SurfaceTexture surfaceTexture) {
+        native_onFrameAvailable(context, surfaceTexture);
+    }
+
+    public synchronized long getContext () {
+        return context;
+    }
+
+    public synchronized void setContext (long c) {
+        context = c;
+    }
+
+    private native void native_onFrameAvailable (long context, SurfaceTexture surfaceTexture);
+}

android/src/org/mavlink/qgroundcontrol/QGCActivity.java

@@ -183,7 +183,7 @@ public class QGCActivity extends QtActivity
     public static native void qgcLogDebug(String message);
     public static native void qgcLogWarning(String message);
 
-    private static native void nativeInit();
+    public native void nativeInit();
 
     // QGCActivity singleton
     public QGCActivity()
@@ -198,6 +198,7 @@ public class QGCActivity extends QtActivity
     public void onCreate(Bundle savedInstanceState)
     {
         super.onCreate(savedInstanceState);
+        nativeInit();
         PowerManager pm = (PowerManager)_instance.getSystemService(Context.POWER_SERVICE);
         _wakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "QGroundControl");
         if(_wakeLock != null) {
@@ -743,10 +744,5 @@ public class QGCActivity extends QtActivity
             }
         }).start();
     }
-
-    public void jniOnLoad()
-    {
-        nativeInit();
-    }
 }
 

src/FlightDisplay/VideoManager.cc

@@ -282,8 +282,44 @@ VideoManager::setfullScreen(bool f)
 
 //-----------------------------------------------------------------------------
 #if defined(QGC_GST_STREAMING)
+gboolean
+VideoManager::_videoSinkQuery(GstPad* pad, GstObject* parent, GstQuery* query)
+{
+    GstElement* element;
+
+    switch (GST_QUERY_TYPE(query)) {
+    case GST_QUERY_CAPS:
+        element = gst_bin_get_by_name(GST_BIN(parent), "glupload");
+        break;
+    case GST_QUERY_CONTEXT:
+        element = gst_bin_get_by_name(GST_BIN(parent), "qmlglsink");
+        break;
+    default:
+        return gst_pad_query_default (pad, parent, query);
+    }
+
+    if (element == nullptr) {
+        qWarning() << "VideoManager::_videoSinkQuery(): No element found";
+        return FALSE;
+    }
+
+    GstPad* sinkpad = gst_element_get_static_pad(element, "sink");
+
+    if (sinkpad == nullptr) {
+        qWarning() << "VideoManager::_videoSinkQuery(): No sink pad found";
+        return FALSE;
+    }
+
+    const gboolean ret = gst_pad_query(sinkpad, query);
+
+    gst_object_unref(sinkpad);
+    sinkpad = nullptr;
+
+    return ret;
+}
+
 GstElement*
-VideoManager::_makeVideoSink(const QString& widgetName)
+VideoManager::_makeVideoSink(gpointer widget)
 {
     GstElement* glupload        = nullptr;
     GstElement* glcolorconvert  = nullptr;
@@ -292,37 +328,30 @@ VideoManager::_makeVideoSink(const QString& widgetName)
     GstElement* sink            = nullptr;
 
     do {
-        QQuickItem* root = qgcApp()->mainRootWindow();
-
-        if (root == nullptr) {
-            qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. No root window";
-            break;
-        }
-
-        QQuickItem* widget = root->findChild<QQuickItem*>(widgetName);
-
-        if (widget == nullptr) {
-            qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. Widget \'" << widgetName << "\' not found";
-            break;
-        }
-
-        if ((glupload = gst_element_factory_make("glupload", nullptr)) == nullptr) {
+        if ((glupload = gst_element_factory_make("glupload", "glupload")) == nullptr) {
             qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glupload')";
             break;
         }
 
-        if ((glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr)) == nullptr) {
+        if ((glcolorconvert = gst_element_factory_make("glcolorconvert", "glcolorconvert")) == nullptr) {
             qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('glcolorconvert')";
             break;
         }
 
-        if ((qmlglsink = gst_element_factory_make("qmlglsink", nullptr)) == nullptr) {
+        if ((qmlglsink = gst_element_factory_make("qmlglsink", "qmlglsink")) == nullptr) {
             qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_element_factory_make('qmlglsink')";
             break;
         }
 
         g_object_set(qmlglsink, "widget", widget, NULL);
 
+// FIXME: AV: temporally disable any sort of QoS due to unknow (yet) timing issues
+#if defined(__android__) || defined(__ios__)
+        //g_object_set(qmlglsink, "max-lateness", -1, NULL);
+        g_object_set(qmlglsink, "qos", FALSE, NULL);
+        g_object_set(qmlglsink, "sync", FALSE, NULL);
+#endif
+
         if ((bin = gst_bin_new("videosink")) == nullptr) {
             qCritical() << "VideoManager::_makeVideoSink() failed. Error with gst_bin_new('videosink')";
             break;
@@ -336,6 +365,7 @@ VideoManager::_makeVideoSink(const QString& widgetName)
         }
 
         gst_bin_add_many(GST_BIN(bin), glupload, glcolorconvert, qmlglsink, nullptr);
+
         gboolean ret = gst_element_link_many(glupload, glcolorconvert, qmlglsink, nullptr);
 
         qmlglsink = glcolorconvert = glupload = nullptr;
@@ -345,7 +375,12 @@ VideoManager::_makeVideoSink(const QString& widgetName)
             break;
         }
 
-        gst_element_add_pad(bin, gst_ghost_pad_new("sink", pad));
+        GstPad* ghostpad = gst_ghost_pad_new("sink", pad);
+
+        gst_pad_set_query_function(ghostpad, _videoSinkQuery);
+
+        gst_element_add_pad(bin, ghostpad);
+
         gst_object_unref(pad);
         pad = nullptr;
 
@@ -382,8 +417,28 @@ void
 VideoManager::_initVideo()
 {
 #if defined(QGC_GST_STREAMING)
-    _videoReceiver->setVideoSink(_makeVideoSink("videoContent"));
-    _thermalVideoReceiver->setVideoSink(_makeVideoSink("thermalVideo"));
+    QQuickItem* root = qgcApp()->mainRootWindow();
+
+    if (root == nullptr) {
+        qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. No root window";
+        return;
+    }
+
+    QQuickItem* widget = root->findChild<QQuickItem*>("videoContent");
+
+    if (widget != nullptr) {
+        _videoReceiver->setVideoSink(_makeVideoSink(widget));
+    } else {
+        qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. 'videoContent' widget not found";
+    }
+
+    widget = root->findChild<QQuickItem*>("thermalVideo");
+
+    if (widget != nullptr) {
+        _thermalVideoReceiver->setVideoSink(_makeVideoSink(widget));
+    } else {
+        qCDebug(VideoManagerLog) << "VideoManager::_makeVideoSink() failed. 'thermalVideo' widget not found";
+    }
 #endif
 }
 

src/FlightDisplay/VideoManager.h

@@ -105,7 +105,8 @@ protected slots:
 protected:
     friend class FinishVideoInitialization;
 #if defined(QGC_GST_STREAMING)
-    GstElement* _makeVideoSink      (const QString& widgetName);
+    static gboolean _videoSinkQuery (GstPad* pad, GstObject* parent, GstQuery* query);
+    GstElement*     _makeVideoSink  (gpointer widget);
 #endif
     void _initVideo                 ();
     void _updateSettings            ();

src/VideoStreaming/VideoReceiver.cc

@@ -75,8 +75,6 @@ VideoReceiver::VideoReceiver(QObject* parent)
     , _videoRunning(false)
     , _showFullScreen(false)
     , _videoSettings(nullptr)
-    , _hwDecoderName(nullptr)
-    , _swDecoderName("avdec_h264")
 {
     _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings();
 #if defined(QGC_GST_STREAMING)
@@ -126,6 +124,77 @@ newPadCB(GstElement* element, GstPad* pad, gpointer data)
     g_free(name);
 }
 
+static gboolean
+autoplugQueryCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
+{
+    GstElement* glupload = (GstElement* )data;
+
+    GstPad* sinkpad = gst_element_get_static_pad(glupload, "sink");
+
+    if (!sinkpad) {
+        qCritical() << "autoplugQueryCaps(): No sink pad found";
+        return FALSE;
+    }
+
+    GstCaps* filter;
+
+    gst_query_parse_caps(query, &filter);
+
+    GstCaps* sinkcaps = gst_pad_query_caps(sinkpad, filter);
+
+    gst_query_set_caps_result(query, sinkcaps);
+
+    const gboolean ret = !gst_caps_is_empty(sinkcaps);
+
+    gst_caps_unref(sinkcaps);
+    sinkcaps = nullptr;
+
+    gst_object_unref(sinkpad);
+    sinkpad = nullptr;
+
+    return ret;
+}
+
+static gboolean
+autoplugQueryContext(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
+{
+    GstElement* glsink = (GstElement* )data;
+
+    GstPad* sinkpad = gst_element_get_static_pad(glsink, "sink");
+
+    if (!sinkpad){
+        qCritical() << "autoplugQueryContext(): No sink pad found";
+        return FALSE;
+    }
+
+    const gboolean ret = gst_pad_query(sinkpad, query);
+
+    gst_object_unref(sinkpad);
+    sinkpad = nullptr;
+
+    return ret;
+}
+
+static gboolean
+autoplugQueryCB(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
+{
+    gboolean ret;
+
+    switch (GST_QUERY_TYPE (query)) {
+    case GST_QUERY_CAPS:
+        ret = autoplugQueryCaps(bin, pad, element, query, data);
+        break;
+    case GST_QUERY_CONTEXT:
+        ret = autoplugQueryContext(bin, pad, element, query, data);
+        break;
+    default:
+        ret = FALSE;
+        break;
+    }
+
+    return ret;
+}
+
 //-----------------------------------------------------------------------------
 void
 VideoReceiver::_restart_timeout()
@@ -274,7 +343,6 @@ VideoReceiver::start()
     GstElement*     parser      = nullptr;
     GstElement*     queue       = nullptr;
     GstElement*     decoder     = nullptr;
-    GstElement*     queue1      = nullptr;
 
     do {
         if ((_pipeline = gst_pipeline_new("receiver")) == nullptr) {
@@ -354,35 +422,27 @@ VideoReceiver::start()
             break;
         }
 
-        if (!_hwDecoderName || (decoder = gst_element_factory_make(_hwDecoderName, "decoder")) == nullptr) {
-            qWarning() << "VideoReceiver::start() hardware decoding not available " << ((_hwDecoderName) ? _hwDecoderName : "");
-            if ((decoder = gst_element_factory_make(_swDecoderName, "decoder")) == nullptr) {
-                qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _swDecoderName << "')";
-                break;
-            }
-        }
-
-        if ((queue1 = gst_element_factory_make("queue", nullptr)) == nullptr) {
-            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('queue') [1]";
+        if ((decoder = gst_element_factory_make("decodebin", "decoder")) == nullptr) {
+            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('decodebin')";
             break;
         }
 
         if(isTaisyncUSB) {
-            gst_bin_add_many(GST_BIN(_pipeline), dataSource, parser, _tee, queue, decoder, queue1, _videoSink, nullptr);
+            gst_bin_add_many(GST_BIN(_pipeline), dataSource, parser, _tee, queue, decoder, _videoSink, nullptr);
         } else {
-            gst_bin_add_many(GST_BIN(_pipeline), dataSource, demux, parser, _tee, queue, decoder, queue1, _videoSink, nullptr);
+            gst_bin_add_many(GST_BIN(_pipeline), dataSource, demux, parser, _tee, queue, decoder, _videoSink, nullptr);
         }
         pipelineUp = true;
 
         if(isUdp264 || isUdp265) {
             // Link the pipeline in front of the tee
-            if(!gst_element_link_many(dataSource, demux, parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
+            if(!gst_element_link_many(dataSource, demux, parser, _tee, queue, decoder, nullptr)) {
                 qCritical() << "Unable to link UDP elements.";
                 break;
             }
         } else if(isTaisyncUSB) {
             // Link the pipeline in front of the tee
-            if(!gst_element_link_many(dataSource, parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
+            if(!gst_element_link_many(dataSource, parser, _tee, queue, decoder, nullptr)) {
                 qCritical() << "Unable to link Taisync USB elements.";
                 break;
             }
@@ -391,20 +451,23 @@ VideoReceiver::start()
                 qCritical() << "Unable to link TCP/MPEG-TS dataSource to Demux.";
                 break;
             }
-            if(!gst_element_link_many(parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) {
+            if(!gst_element_link_many(parser, _tee, queue, decoder, nullptr)) {
                 qCritical() << "Unable to link TCP/MPEG-TS pipline to parser.";
                 break;
             }
             g_signal_connect(demux, "pad-added", G_CALLBACK(newPadCB), parser);
         } else {
             g_signal_connect(dataSource, "pad-added", G_CALLBACK(newPadCB), demux);
-            if(!gst_element_link_many(demux, parser, _tee, queue, decoder, _videoSink, nullptr)) {
+            if(!gst_element_link_many(demux, parser, _tee, queue, decoder, nullptr)) {
                 qCritical() << "Unable to link RTSP elements.";
                 break;
             }
         }
 
-        dataSource = demux = parser = queue = decoder = queue1 = nullptr;
+        g_signal_connect(decoder, "pad-added", G_CALLBACK(newPadCB), _videoSink);
+        g_signal_connect(decoder, "autoplug-query", G_CALLBACK(autoplugQueryCB), _videoSink);
+
+        dataSource = demux = parser = queue = decoder = nullptr;
 
         GstBus* bus = nullptr;
 
@@ -436,11 +499,6 @@ VideoReceiver::start()
 
         // If we failed before adding items to the pipeline, then clean up
         if (!pipelineUp) {
-            if (queue1 != nullptr) {
-                gst_object_unref(queue1);
-                queue1 = nullptr;
-            }
-
             if (decoder != nullptr) {
                 gst_object_unref(decoder);
                 decoder = nullptr;
@@ -554,8 +612,6 @@ VideoReceiver::_shutdownPipeline() {
 void
 VideoReceiver::_handleError() {
     qCDebug(VideoReceiverLog) << "Gstreamer error!";
-    // If there was an error we switch to software decoding only
-    _tryWithHardwareDecoding = false;
     stop();
     _restart_timer.start(_restart_time_ms);
 }
@@ -666,32 +722,12 @@ VideoReceiver::_cleanupOldVideos()
 void
 VideoReceiver::setVideoDecoder(VideoEncoding encoding)
 {
-    /*
-    #if defined(Q_OS_MAC)
-        _hwDecoderName = "vtdec";
-    #else
-        _hwDecoderName = "vaapidecode";
-    #endif
-    */
-
     if (encoding == H265_HW || encoding == H265_SW) {
         _depayName  = "rtph265depay";
         _parserName = "h265parse";
-#if defined(__android__)
-        _hwDecoderName = "amcviddec-omxgooglehevcdecoder";
-#endif
-        _swDecoderName = "avdec_h265";
     } else {
         _depayName  = "rtph264depay";
         _parserName = "h264parse";
-#if defined(__android__)
-        _hwDecoderName = "amcviddec-omxgoogleh264decoder";
-#endif
-        _swDecoderName = "avdec_h264";
-    }
-
-    if (!_tryWithHardwareDecoding) {
-        _hwDecoderName = nullptr;
     }
 }
 

src/VideoStreaming/VideoReceiver.h

@@ -161,8 +161,5 @@ protected:
     VideoSettings*  _videoSettings;
     const char*     _depayName;
     const char*     _parserName;
-    bool            _tryWithHardwareDecoding = true;
-    const char*     _hwDecoderName;
-    const char*     _swDecoderName;
 };
 

src/VideoStreaming/VideoStreaming.cc

@@ -19,10 +19,23 @@
 
 #if defined(QGC_GST_STREAMING)
 #include <gst/gst.h>
-#ifdef __android__
-//#define ANDDROID_GST_DEBUG
-#endif
-#if defined(__ios__)
+#if defined(__android__)
+#include <android/log.h>
+
+static void gst_android_log(GstDebugCategory * category,
+                            GstDebugLevel      level,
+                            const gchar      * file,
+                            const gchar      * function,
+                            gint               line,
+                            GObject          * object,
+                            GstDebugMessage  * message,
+                            gpointer           data)
+{
+    if (level <= gst_debug_category_get_threshold (category)) {
+        __android_log_print(ANDROID_LOG_ERROR, "GST", "%s, %s: %s", file, function, gst_debug_message_get(message));
+    }
+}
+#elif defined(__ios__)
 #include "gst_ios_init.h"
 #endif
 #else
@@ -34,8 +47,9 @@
 #if defined(QGC_GST_STREAMING)
     G_BEGIN_DECLS
     // The static plugins we use
-#if defined(__mobile__)
+#if defined(__android__) || defined(__ios__)
     GST_PLUGIN_STATIC_DECLARE(coreelements);
+    GST_PLUGIN_STATIC_DECLARE(playback);
     GST_PLUGIN_STATIC_DECLARE(libav);
     GST_PLUGIN_STATIC_DECLARE(rtp);
     GST_PLUGIN_STATIC_DECLARE(rtsp);
@@ -46,9 +60,11 @@
     GST_PLUGIN_STATIC_DECLARE(isomp4);
     GST_PLUGIN_STATIC_DECLARE(matroska);
     GST_PLUGIN_STATIC_DECLARE(opengl);
-#endif
 #if defined(__android__)
     GST_PLUGIN_STATIC_DECLARE(androidmedia);
+#elif defined(__ios__)
+    GST_PLUGIN_STATIC_DECLARE(applemedia);
+#endif
 #endif
     GST_PLUGIN_STATIC_DECLARE(qmlgl);
     G_END_DECLS
@@ -64,49 +80,6 @@ static void qgcputenv(const QString& key, const QString& root, const QString& pa
 #endif
 #endif
 
-#ifdef ANDDROID_GST_DEBUG
-// Redirects stdio and stderr to logcat
-#include <unistd.h>
-#include <pthread.h>
-#include <android/log.h>
-
-static int pfd[2];
-static pthread_t thr;
-static const char *tag = "myapp";
-
-static void *thread_func(void*)
-{
-    ssize_t rdsz;
-    char buf[128];
-    while((rdsz = read(pfd[0], buf, sizeof buf - 1)) > 0) {
-        if(buf[rdsz - 1] == '\n') --rdsz;
-        buf[rdsz] = 0;  /* add null-terminator */
-        __android_log_write(ANDROID_LOG_DEBUG, tag, buf);
-    }
-    return 0;
-}
-
-int start_logger(const char *app_name)
-{
-    tag = app_name;
-
-    /* make stdout line-buffered and stderr unbuffered */
-    setvbuf(stdout, 0, _IOLBF, 0);
-    setvbuf(stderr, 0, _IONBF, 0);
-
-    /* create the pipe and redirect stdout and stderr */
-    pipe(pfd);
-    dup2(pfd[1], 1);
-    dup2(pfd[1], 2);
-
-    /* spawn the logging thread */
-    if(pthread_create(&thr, 0, thread_func, 0) == -1)
-        return -1;
-    pthread_detach(thr);
-    return 0;
-}
-#endif
-
 void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debuglevel)
 {
 #if defined(QGC_GST_STREAMING)
@@ -126,45 +99,56 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu
         qgcputenv("GST_PLUGIN_PATH", currentDir, "/gstreamer-plugins");
     #endif
 
-    // Initialize GStreamer
-    #if defined(__ios__)
-        //-- iOS specific initialization
-        gst_ios_init();
-    #else
-        //-- Generic initialization
-        if (qgetenv("GST_DEBUG").isEmpty() && logpath) {
-            QString gstDebugFile = QString("%1/%2").arg(logpath).arg("gstreamer-log.txt");
-            qDebug() << "GStreamer debug output:" << gstDebugFile;
-            if (debuglevel) {
-                qputenv("GST_DEBUG", debuglevel);
-            }
-            qputenv("GST_DEBUG_NO_COLOR", "1");
-            qputenv("GST_DEBUG_FILE", gstDebugFile.toUtf8());
-            qputenv("GST_DEBUG_DUMP_DOT_DIR", logpath);
-        }
-        GError* error = nullptr;
-        if (!gst_init_check(&argc, &argv, &error)) {
-            qCritical() << "gst_init_check() failed: " << error->message;
-            g_error_free(error);
+    //-- Generic initialization
+    if (qgetenv("GST_DEBUG").isEmpty() && logpath) {
+        QString gstDebugFile = QString("%1/%2").arg(logpath).arg("gstreamer-log.txt");
+        qDebug() << "GStreamer debug output:" << gstDebugFile;
+        if (debuglevel) {
+            qputenv("GST_DEBUG", debuglevel);
         }
-    #endif
-        // The static plugins we use
-    #if defined(__android__)
-        GST_PLUGIN_STATIC_REGISTER(coreelements);
-        GST_PLUGIN_STATIC_REGISTER(libav);
-        GST_PLUGIN_STATIC_REGISTER(rtp);
-        GST_PLUGIN_STATIC_REGISTER(rtsp);
-        GST_PLUGIN_STATIC_REGISTER(udp);
-        GST_PLUGIN_STATIC_REGISTER(videoparsersbad);
-        GST_PLUGIN_STATIC_REGISTER(x264);
-        GST_PLUGIN_STATIC_REGISTER(rtpmanager);
-        GST_PLUGIN_STATIC_REGISTER(isomp4);
-        GST_PLUGIN_STATIC_REGISTER(matroska);
-        GST_PLUGIN_STATIC_REGISTER(androidmedia);
-    #endif
+        qputenv("GST_DEBUG_NO_COLOR", "1");
+        qputenv("GST_DEBUG_FILE", gstDebugFile.toUtf8());
+        qputenv("GST_DEBUG_DUMP_DOT_DIR", logpath);
+    }
+
+    // Initialize GStreamer
+#if defined(__android__)
+    gst_debug_add_log_function(gst_android_log, nullptr, nullptr);
+#elif defined(__ios__)
+    //-- iOS specific initialization
+    gst_ios_pre_init();
+#endif
 
-#if defined(__mobile__)
-        GST_PLUGIN_STATIC_REGISTER(opengl);
+    GError* error = nullptr;
+    if (!gst_init_check(&argc, &argv, &error)) {
+        qCritical() << "gst_init_check() failed: " << error->message;
+        g_error_free(error);
+    }
+
+    // The static plugins we use
+#if defined(__android__) || defined(__ios__)
+    GST_PLUGIN_STATIC_REGISTER(coreelements);
+    GST_PLUGIN_STATIC_REGISTER(playback);
+    GST_PLUGIN_STATIC_REGISTER(libav);
+    GST_PLUGIN_STATIC_REGISTER(rtp);
+    GST_PLUGIN_STATIC_REGISTER(rtsp);
+    GST_PLUGIN_STATIC_REGISTER(udp);
+    GST_PLUGIN_STATIC_REGISTER(videoparsersbad);
+    GST_PLUGIN_STATIC_REGISTER(x264);
+    GST_PLUGIN_STATIC_REGISTER(rtpmanager);
+    GST_PLUGIN_STATIC_REGISTER(isomp4);
+    GST_PLUGIN_STATIC_REGISTER(matroska);
+    GST_PLUGIN_STATIC_REGISTER(opengl);
+
+#if defined(__android__)
+    GST_PLUGIN_STATIC_REGISTER(androidmedia);
+#elif defined(__ios__)
+    GST_PLUGIN_STATIC_REGISTER(applemedia);
+#endif
+#endif
+
+#if defined(__ios__)
+    gst_ios_post_init();
 #endif
 
     /* the plugin must be loaded before loading the qml file to register the
@@ -185,7 +169,7 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu
         qCritical() << "unable to find qmlglsink - you need to build it yourself and add to GST_PLUGIN_PATH";
     }
 #else
-    qmlRegisterType<GLVideoItemStub> ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem");
+    qmlRegisterType<GLVideoItemStub>("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem");
     Q_UNUSED(argc)
     Q_UNUSED(argv)
     Q_UNUSED(logpath)

src/VideoStreaming/VideoStreaming.pri

@@ -32,7 +32,7 @@ LinuxBuild {
     exists($$GST_ROOT) {
         CONFIG      += VideoEnabled
         INCLUDEPATH += $$GST_ROOT/Headers
-        LIBS        += -F$$(HOME)/Library/Developer/GStreamer/iPhone.sdk -framework GStreamer -liconv -lresolv
+        LIBS        += -F$$(HOME)/Library/Developer/GStreamer/iPhone.sdk -framework GStreamer -framework AVFoundation -framework CoreMedia -framework CoreVideo -framework VideoToolbox -liconv -lresolv
     }
 } else:WindowsBuild {
     #- gstreamer installed by default under c:/gstreamer
@@ -78,6 +78,7 @@ LinuxBuild {
         LIBS += -L$$GST_ROOT/lib/gstreamer-1.0 \
             -lgstvideo-1.0 \
             -lgstcoreelements \
+            -lgstplayback \
             -lgstudp \
             -lgstrtp \
             -lgstrtsp \

src/VideoStreaming/iOS/gst_ios_init.h

-#ifndef __GST_IOS_INIT_H__
-#define __GST_IOS_INIT_H__
+#pragma once
 
 #include <gst/gst.h>
 
 G_BEGIN_DECLS
 
-#define GST_G_IO_MODULE_DECLARE(name) \
-extern void G_PASTE(g_io_module_, G_PASTE(name, _load_static)) (void)
-
-#define GST_G_IO_MODULE_LOAD(name) \
-G_PASTE(g_io_module_, G_PASTE(name, _load_static)) ()
-
-/* Uncomment each line to enable the plugin categories that your application needs.
- * You can also enable individual plugins. See gst_ios_init.c to see their names
- */
-
-//#define GST_IOS_PLUGINS_GES
-#define GST_IOS_PLUGINS_CORE
-//#define GST_IOS_PLUGINS_CAPTURE
-//#define GST_IOS_PLUGINS_CODECS_RESTRICTED
-//#define GST_IOS_PLUGINS_ENCODING
-//#define GST_IOS_PLUGINS_CODECS_GPL
-//#define GST_IOS_PLUGINS_NET_RESTRICTED
-//#define GST_IOS_PLUGINS_SYS
-//#define GST_IOS_PLUGINS_VIS
-//#define GST_IOS_PLUGINS_PLAYBACK
-//#define GST_IOS_PLUGINS_EFFECTS
-//#define GST_IOS_PLUGINS_CODECS
-#define GST_IOS_PLUGINS_NET
-
-//-- Specific to QGC
-#define GST_IOS_PLUGIN_X264
-#define GST_IOS_PLUGIN_LIBAV
-#define GST_IOS_PLUGIN_ISOMP4
-#define GST_IOS_PLUGIN_VIDEOPARSERSBAD
-#define GST_IOS_PLUGIN_MATROSKA
-
-//#define GST_IOS_GIO_MODULE_GNUTLS
-
-void gst_ios_init (void);
+void gst_ios_pre_init(void);
+void gst_ios_post_init(void);
 
 G_END_DECLS
-
-#endif

src/main.cc

@@ -138,7 +138,7 @@ static const char kJniClassName[] {"org/mavlink/qgroundcontrol/QGCActivity"};
 void setNativeMethods(void)
 {
     JNINativeMethod javaMethods[] {
-        {"nativeInit", "(Landroid/content/Context;)V", reinterpret_cast<void *>(gst_android_init)}
+        {"nativeInit", "()V", reinterpret_cast<void *>(gst_android_init)}
     };
 
     QAndroidJniEnvironment jniEnv;
@@ -176,12 +176,8 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved)
     if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
         return -1;
     }
-    setNativeMethods();
 
-    QAndroidJniObject resultL = QAndroidJniObject::callStaticObjectMethod(
-        kJniClassName,
-        "jniOnLoad",
-        "();");
+    setNativeMethods();
 
 #if defined(QGC_GST_STREAMING)
     // Tell the androidmedia plugin about the Java VM
@@ -191,7 +187,9 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved)
  #if !defined(NO_SERIAL_LINK)
     QSerialPort::setNativeMethods();
  #endif
+
     JoystickAndroid::setNativeMethods();
+
 #if defined(QGC_ENABLE_PAIRING)
     PairingManager::setNativeMethods();
 #endif