From 0f2705d2fe8705b6d898fad3f4dcedae20ad77b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Fran=C4=8De=C5=A1kin?= Date: Sun, 9 Jun 2019 22:54:17 +0200 Subject: [PATCH] User hardware video decoding on Android --- .../mavlink/qgroundcontrol/QGCActivity.java | 7 ++ src/VideoStreaming/VideoReceiver.cc | 41 ++++++-- src/VideoStreaming/VideoReceiver.h | 4 +- src/VideoStreaming/VideoStreaming.cc | 4 + src/VideoStreaming/VideoStreaming.pri | 2 + src/main.cc | 94 ++++++++++++++++++- 6 files changed, 143 insertions(+), 9 deletions(-) diff --git a/android/src/org/mavlink/qgroundcontrol/QGCActivity.java b/android/src/org/mavlink/qgroundcontrol/QGCActivity.java index 6b10fd965..88ff1d5e0 100644 --- a/android/src/org/mavlink/qgroundcontrol/QGCActivity.java +++ b/android/src/org/mavlink/qgroundcontrol/QGCActivity.java @@ -184,6 +184,8 @@ public class QGCActivity extends QtActivity public static native void qgcLogDebug(String message); public static native void qgcLogWarning(String message); + private static native void nativeInit(); + // QGCActivity singleton public QGCActivity() { @@ -745,5 +747,10 @@ public class QGCActivity extends QtActivity } }).start(); } + + public void jniOnLoad() + { + nativeInit(); + } } diff --git a/src/VideoStreaming/VideoReceiver.cc b/src/VideoStreaming/VideoReceiver.cc index f6c2500f4..6fc9c3184 100644 --- a/src/VideoStreaming/VideoReceiver.cc +++ b/src/VideoStreaming/VideoReceiver.cc @@ -71,6 +71,8 @@ VideoReceiver::VideoReceiver(QObject* parent) , _videoRunning(false) , _showFullScreen(false) , _videoSettings(nullptr) + , _hwDecoderName(nullptr) + , _swDecoderName("avdec_h264") { _videoSurface = new VideoSurface; _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings(); @@ -159,6 +161,10 @@ VideoReceiver::_restart_timeout() void VideoReceiver::start() { + if (_uri.isEmpty()) { + return; + } + qCDebug(VideoReceiverLog) << "start():" << _uri; if(qgcApp()->runningUnitTests()) { return; } @@ -170,7 +176,6 @@ VideoReceiver::start() #if defined(QGC_GST_STREAMING) _stop = false; - qCDebug(VideoReceiverLog) << "start():" << _uri; #if defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__)) //-- Taisync on iOS or Android sends a raw h.264 stream @@ -280,9 +285,12 @@ VideoReceiver::start() break; } - if ((decoder = gst_element_factory_make(_decoderName, "decoder")) == nullptr) { - qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _decoderName << "')"; - break; + if (!_hwDecoderName || (decoder = gst_element_factory_make(_hwDecoderName, "decoder")) == nullptr) { + qWarning() << "VideoReceiver::start() hardware decoding not available " << ((_hwDecoderName) ? _hwDecoderName : ""); + if ((decoder = gst_element_factory_make(_swDecoderName, "decoder")) == nullptr) { + qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _swDecoderName << "')"; + break; + } } if ((queue1 = gst_element_factory_make("queue", nullptr)) == nullptr) { @@ -470,6 +478,8 @@ VideoReceiver::_shutdownPipeline() { void VideoReceiver::_handleError() { qCDebug(VideoReceiverLog) << "Gstreamer error!"; + // If there was an error we switch to software decoding only + _tryWithHardwareDecoding = false; stop(); _restart_timer.start(_restart_time_ms); } @@ -580,17 +590,34 @@ VideoReceiver::_cleanupOldVideos() void VideoReceiver::setVideoDecoder(VideoEncoding encoding) { + /* + #if defined(Q_OS_MAC) + _hwDecoderName = "vtdec"; + #else + _hwDecoderName = "vaapidecode"; + #endif + */ + if (encoding == H265_HW || encoding == H265_SW) { _depayName = "rtph265depay"; _parserName = "h265parse"; - _decoderName = "avdec_h265"; +#if defined(__android__) + _hwDecoderName = "amcviddec-omxgooglehevcdecoder"; +#endif + _swDecoderName = "avdec_h265"; } else { _depayName = "rtph264depay"; _parserName = "h264parse"; - _decoderName = "avdec_h264"; +#if defined(__android__) + _hwDecoderName = "amcviddec-omxgoogleh264decoder"; +#endif + _swDecoderName = "avdec_h264"; } -} + if (!_tryWithHardwareDecoding) { + _hwDecoderName = nullptr; + } +} //----------------------------------------------------------------------------- // When we finish our pipeline will look like this: // diff --git a/src/VideoStreaming/VideoReceiver.h b/src/VideoStreaming/VideoReceiver.h index 7f3b34021..882e6e97d 100644 --- a/src/VideoStreaming/VideoReceiver.h +++ b/src/VideoStreaming/VideoReceiver.h @@ -152,6 +152,8 @@ protected: VideoSettings* _videoSettings; const char* _depayName; const char* _parserName; - const char* _decoderName; + bool _tryWithHardwareDecoding = true; + const char* _hwDecoderName; + const char* _swDecoderName; }; diff --git a/src/VideoStreaming/VideoStreaming.cc b/src/VideoStreaming/VideoStreaming.cc index cd02747e7..82f807b36 100644 --- a/src/VideoStreaming/VideoStreaming.cc +++ b/src/VideoStreaming/VideoStreaming.cc @@ -47,6 +47,9 @@ GST_PLUGIN_STATIC_DECLARE(rtpmanager); GST_PLUGIN_STATIC_DECLARE(isomp4); GST_PLUGIN_STATIC_DECLARE(matroska); +#endif +#if defined(__android__) + GST_PLUGIN_STATIC_DECLARE(androidmedia); #endif G_END_DECLS #endif @@ -159,6 +162,7 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu GST_PLUGIN_STATIC_REGISTER(rtpmanager); GST_PLUGIN_STATIC_REGISTER(isomp4); GST_PLUGIN_STATIC_REGISTER(matroska); + GST_PLUGIN_STATIC_REGISTER(androidmedia); #endif #else Q_UNUSED(argc); diff --git a/src/VideoStreaming/VideoStreaming.pri b/src/VideoStreaming/VideoStreaming.pri index f4265181b..aef28dc5b 100644 --- a/src/VideoStreaming/VideoStreaming.pri +++ b/src/VideoStreaming/VideoStreaming.pri @@ -97,11 +97,13 @@ LinuxBuild { -lgstrtpmanager \ -lgstisomp4 \ -lgstmatroska \ + -lgstandroidmedia # Rest of GStreamer dependencies LIBS += -L$$GST_ROOT/lib \ -lgstfft-1.0 -lm \ -lgstnet-1.0 -lgio-2.0 \ + -lgstphotography-1.0 -lgstgl-1.0 -lEGL \ -lgstaudio-1.0 -lgstcodecparsers-1.0 -lgstbase-1.0 \ -lgstreamer-1.0 -lgstrtp-1.0 -lgstpbutils-1.0 -lgstrtsp-1.0 -lgsttag-1.0 \ -lgstvideo-1.0 -lavformat -lavcodec -lavutil -lx264 -lavfilter -lswresample \ diff --git a/src/main.cc b/src/main.cc index c65c97176..b3c4473e7 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,6 @@ /**************************************************************************** * - * (c) 2009-2016 QGROUNDCONTROL PROJECT + * (c) 2009-2019 QGROUNDCONTROL PROJECT * * QGroundControl is licensed according to the terms in the file * COPYING.md in the root of the source code directory. @@ -85,6 +85,84 @@ int WindowsCrtReportHook(int reportType, char* message, int* returnValue) #include "qserialport.h" #endif +static jobject _class_loader = nullptr; +static jobject _context = nullptr; + +//----------------------------------------------------------------------------- +extern "C" { + void gst_amc_jni_set_java_vm(JavaVM *java_vm); + + jobject gst_android_get_application_class_loader(void) + { + return _class_loader; + } +} + +//----------------------------------------------------------------------------- +static void +gst_android_init(JNIEnv* env, jobject context) +{ + jobject class_loader = nullptr; + + jclass context_cls = env->GetObjectClass(context); + if (!context_cls) { + return; + } + + jmethodID get_class_loader_id = env->GetMethodID(context_cls, "getClassLoader", "()Ljava/lang/ClassLoader;"); + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return; + } + + class_loader = env->CallObjectMethod(context, get_class_loader_id); + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return; + } + + _context = env->NewGlobalRef(context); + _class_loader = env->NewGlobalRef (class_loader); +} + +//----------------------------------------------------------------------------- +static const char kJniClassName[] {"org/mavlink/qgroundcontrol/QGCActivity"}; + +void setNativeMethods(void) +{ + JNINativeMethod javaMethods[] { + {"nativeInit", "(Landroid/content/Context;)V", reinterpret_cast(gst_android_init)} + }; + + QAndroidJniEnvironment jniEnv; + if (jniEnv->ExceptionCheck()) { + jniEnv->ExceptionDescribe(); + jniEnv->ExceptionClear(); + } + + jclass objectClass = jniEnv->FindClass(kJniClassName); + if(!objectClass) { + qWarning() << "Couldn't find class:" << kJniClassName; + return; + } + + jint val = jniEnv->RegisterNatives(objectClass, javaMethods, sizeof(javaMethods) / sizeof(javaMethods[0])); + + if (val < 0) { + qWarning() << "Error registering methods: " << val; + } else { + qDebug() << "Main Native Functions Registered"; + } + + if (jniEnv->ExceptionCheck()) { + jniEnv->ExceptionDescribe(); + jniEnv->ExceptionClear(); + } +} + +//----------------------------------------------------------------------------- jint JNI_OnLoad(JavaVM* vm, void* reserved) { Q_UNUSED(reserved); @@ -93,6 +171,18 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { return -1; } + setNativeMethods(); + + QAndroidJniObject resultL = QAndroidJniObject::callStaticObjectMethod( + kJniClassName, + "jniOnLoad", + "();"); + +#if defined(QGC_GST_STREAMING) + // Tell the androidmedia plugin about the Java VM + gst_amc_jni_set_java_vm(vm); +#endif + #if !defined(NO_SERIAL_LINK) QSerialPort::setNativeMethods(); #endif @@ -102,6 +192,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) } #endif +//----------------------------------------------------------------------------- #ifdef __android__ #include bool checkAndroidWritePermission() { @@ -117,6 +208,7 @@ bool checkAndroidWritePermission() { } #endif +//----------------------------------------------------------------------------- /** * @brief Starts the application * -- 2.22.0