From b6f76e2a8e60668dffcc9b4dc43dbea04e2d9d16 Mon Sep 17 00:00:00 2001 From: Nate Weibley Date: Fri, 24 Jun 2016 20:32:08 -0400 Subject: [PATCH] Ccache test for #3614 (#3631) * travis-ci try ccache on OSX and android * travis-ci ccache stats * try again * Attempt to prepend ccache to qmake command * Test enabling ccache for ios too * Deploy ccache from S3 per @dagar * Revert "Test enabling ccache for ios too" It was a nice thought. This reverts commit 7e7aeb367696ee9c8abc5cc8122dcd35196d96bb. * Try to optimally set the number of make jobs * Fix job estimation and disable ccache on android * Android: reenable ccache and disable parallel jobs * Check if android build passes with 4 workers * Try to parallelize xcodebuild too * Give mac builders an extra worker and pretty up the ios build output * Add make output filtering with correct exit code, no sudo or ccache on ios build * Try wrapping ccache to enable use on xcodebuild * Make ios ccache helpers executable * Turn back on pretty printing for IOS * Show some ccache stats on IOS build to see if we're getting hits * Change android build OS type to linux * Require trusty for android build * Require sudo for android so we get trusty * Maybe this will function properly on edge? * Try to use stripped binary bzips * Screw it! Run android on precise again * Back to the original android build configuration. I give up --- .travis.yml | 75 +++++++++++++++++++++++++++---------------- QGCCommon.pri | 22 ++++++++++--- tools/iosccachecc.sh | 2 ++ tools/iosccachecxx.sh | 2 ++ 4 files changed, 70 insertions(+), 31 deletions(-) create mode 100755 tools/iosccachecc.sh create mode 100755 tools/iosccachecxx.sh diff --git a/.travis.yml b/.travis.yml index c65096313..1a2bde044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ language: cpp env: global: - SHADOW_BUILD_DIR=/tmp/shadow_build_dir + - JOBS=2 # ANDROID_STOREPASS - secure: RGovyUnMw3fp/bHZi058JvANT1rYmNqrsuSYew0cIgirO6YbMHr/rsjwCm1FTYpBl8s1zgr+u2b8ftYnfnCz2YT+Aip4NWrVYpVU0FEmfytGILrnUS0pjlt8m7fU9AKR1ElOSll7yw7e1kftynN39Q321etvwbLZcXon6zz0suE= @@ -30,6 +31,7 @@ matrix: - os: osx osx_image: xcode7 env: SPEC=ios + sudo: false - os: android language: android env: SPEC=android-g++ CONFIG=installer @@ -71,8 +73,7 @@ addons: - wget cache: - directories: - - $HOME/.ccache + ccache: true before_install: @@ -92,7 +93,7 @@ before_install: cp ${TRAVIS_BUILD_DIR}/test/qtlogging.ini ~/Library/Preferences/QtProject/ ; fi - - if [ "${TRAVIS_OS_NAME}" = "android" ]; then + - if [ "${SPEC}" = "android-g++" ]; then wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/gstreamer-1.0-android-armv7-1.5.2.tar.bz2 && mkdir -p ${TRAVIS_BUILD_DIR}/gstreamer-1.0-android-armv7-1.5.2 && tar jxf gstreamer-1.0-android-armv7-1.5.2.tar.bz2 -C ${TRAVIS_BUILD_DIR}/gstreamer-1.0-android-armv7-1.5.2 @@ -101,8 +102,8 @@ before_install: install: - if [[ "${TRAVIS_OS_NAME}" = "linux" && "${CONFIG}" != "doxygen" ]]; then - wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-linux.tar.bz2 && - tar jxf Qt5.5.1-linux.tar.bz2 -C /tmp && + wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-linux-min.tar.bz2 && + tar jxf Qt5.5.1-linux-min.tar.bz2 -C /tmp && export PATH=/tmp/Qt/5.5/gcc_64/bin:$PATH && export CXX="g++-4.8" && export CC="gcc-4.8" && @@ -115,8 +116,8 @@ install: wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/gstreamer-1.0-1.5.2-x86_64.pkg && sudo installer -verboseR -pkg gstreamer-1.0-devel-1.5.2-x86_64.pkg -target / && sudo installer -verboseR -pkg gstreamer-1.0-1.5.2-x86_64.pkg -target / && - wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-mac-clang.tar.bz2 && - tar jxf Qt5.5.1-mac-clang.tar.bz2 -C /tmp && + wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-mac-clang-min.tar.bz2 && + tar jxf Qt5.5.1-mac-clang-min.tar.bz2 -C /tmp && wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/osx-gstreamer.tar.bz2 && sudo tar jxf osx-gstreamer.tar.bz2 -C /Library/Frameworks && export QT_DIR=Qt5.5-mac-clang/5.5/clang_64 && @@ -125,40 +126,53 @@ install: export QML2_IMPORT_PATH=/tmp/$QT_DIR/qml ; fi + - if [[ "${TRAVIS_OS_NAME}" = "osx" ]]; then + wget https://s3.amazonaws.com/px4-travis/toolchain/macos/ccache && + sudo mv ccache /usr/local/bin && + chmod +x /usr/local/bin/ccache && + export CCACHE_CPP2=1 + ; + fi - if [[ "${TRAVIS_OS_NAME}" = "osx" && "${SPEC}" = "ios" ]]; then - wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-ios.tar.bz2 && - tar jxf Qt5.5.1-ios.tar.bz2 -C /tmp && - export PATH=/tmp/ios/bin:$PATH + wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-ios-min.tar.bz2 && + tar jxf Qt5.5.1-ios-min.tar.bz2 -C /tmp && + export PATH=/tmp/ios/bin:$PATH && + export IOS_CCACHE_CC=`/usr/bin/xcrun -sdk iphoneos -find clang` && + export IOS_CCACHE_CXX=`/usr/bin/xcrun -sdk iphoneos -find clang++` ; fi - - if [ "${TRAVIS_OS_NAME}" = "android" ]; then + - if [ "${SPEC}" = "android-g++" ]; then wget http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin && chmod +x android-ndk-r10e-linux-x86_64.bin && ./android-ndk-r10e-linux-x86_64.bin > /dev/null && export PATH=`pwd`/android-ndk-r10e:$PATH && export ANDROID_NDK_ROOT=`pwd`/android-ndk-r10e && export ANDROID_SDK_ROOT=/usr/local/android-sdk && - wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-linux.tar.bz2 && - tar jxf Qt5.5.1-linux.tar.bz2 -C /tmp && + wget https://s3-us-west-2.amazonaws.com/qgroundcontrol/dependencies/Qt5.5.1-linux-min.tar.bz2 && + tar jxf Qt5.5.1-linux-min.tar.bz2 -C /tmp && export PATH=/tmp/Qt/5.5/android_armv7/bin:$PATH + export USE_CCACHE=1 && + export NDK_CCACHE=`which ccache` ; fi before_script: # setup ccache - - mkdir -p ~/bin - - ln -s /usr/bin/ccache ~/bin/g++ - - ln -s /usr/bin/ccache ~/bin/g++-4.8 - - ln -s /usr/bin/ccache ~/bin/gcc - - ln -s /usr/bin/ccache ~/bin/gcc-4.8 - - export PATH=~/bin:$PATH - wget http://px4-travis.s3.amazonaws.com/Firmware/master/parameters.xml -O src/FirmwarePlugin/PX4/PX4ParameterFactMetaData.xml - wget http://px4-travis.s3.amazonaws.com/Firmware/master/airframes.xml -O src/AutoPilotPlugins/PX4/AirframeFactMetaData.xml - - if [[ "${TRAVIS_OS_NAME}" = "android" && "${CONFIG}" = "installer" && -z ${ANDROID_STOREPASS} ]]; then + - if [[ "${SPEC}" = "android-g++" && "${CONFIG}" = "installer" && -z ${ANDROID_STOREPASS} ]]; then export CONFIG=release ; fi + - if [ "${TRAVIS_OS_NAME}" != "osx" ]; then + export JOBS=$((`cat /proc/cpuinfo | grep -c ^processor`+1)) + ; + fi + - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then + export JOBS=$((`sysctl -n hw.ncpu`+1)) + ; + fi - if [[ "${CONFIG}" != "doxygen" && "${SPEC}" != "ios" ]]; then mkdir ${SHADOW_BUILD_DIR} && cd ${SHADOW_BUILD_DIR} && @@ -173,19 +187,26 @@ before_script: fi script: - - if [ "${TRAVIS_OS_NAME}" = "android" ]; then + - if [ "${SPEC}" = "android-g++" ]; then cd ${TRAVIS_BUILD_DIR} && - ./tools/update_android_version.sh + ./tools/update_android_version.sh && + export JOBS=4 ; fi - if [[ "${CONFIG}" != "doxygen" && "${SPEC}" != "ios" ]]; then - cd ${SHADOW_BUILD_DIR} && - make -j4 + ccache -M 1GB && + ccache -z && + cd ${SHADOW_BUILD_DIR} && echo Building with $JOBS workers && + make -j$JOBS | sed 's/${TRAVIS_BUILD_DIR}/-/' && $(exit ${PIPESTATUS[0]}) && + ccache -s ; fi - if [ "${SPEC}" = "ios" ]; then - cd ${SHADOW_BUILD_DIR} && - xcodebuild -configuration Release CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO + ccache -M 1GB && + ccache -z && + cd ${SHADOW_BUILD_DIR} && echo Building with $JOBS workers && + xcodebuild -IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$JOBS -configuration Release CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | xcpretty -c && $(exit ${PIPESTATUS[0]}) && + ccache -s ; fi - if [ "${CONFIG}" = "doxygen" ]; then @@ -213,7 +234,7 @@ after_success: else export GOOGLE_PLAY_TRACK=none; fi - - if [[ "${TRAVIS_OS_NAME}" = "android" && "${TRAVIS_PULL_REQUEST}" = "false" && "${GOOGLE_PLAY_TRACK}" != "none" ]]; then + - if [[ "${SPEC}" = "android-g++" && "${TRAVIS_PULL_REQUEST}" = "false" && "${GOOGLE_PLAY_TRACK}" != "none" ]]; then pip install --user google-api-python-client PyOpenSSL && cd ${TRAVIS_BUILD_DIR} && openssl aes-256-cbc -K $encrypted_25db6eb7c3fd_key -iv $encrypted_25db6eb7c3fd_iv -in ${TRAVIS_BUILD_DIR}/android/Google_Play_Android_Developer-4432a3c4f5d1.json.enc -out android/Google_Play_Android_Developer-4432a3c4f5d1.json -d && diff --git a/QGCCommon.pri b/QGCCommon.pri index 67f4e25ad..5caef6271 100644 --- a/QGCCommon.pri +++ b/QGCCommon.pri @@ -21,10 +21,10 @@ linux { message("Linux build") CONFIG += LinuxBuild DEFINES += __STDC_LIMIT_MACROS - linux-clang { - message("Linux clang") - QMAKE_CXXFLAGS += -Qunused-arguments -fcolor-diagnostics - } + linux-clang { + message("Linux clang") + QMAKE_CXXFLAGS += -Qunused-arguments -fcolor-diagnostics + } } else : linux-rasp-pi2-g++ { message("Linux R-Pi2 build") CONFIG += LinuxBuild @@ -78,6 +78,20 @@ equals(QT_MAJOR_VERSION, 5) | greaterThan(QT_MINOR_VERSION, 5) { error("Unsupported build platform, only Linux, Windows, Android and Mac (Mac OS and iOS) are supported") } +# Enable ccache where we can +linux|macx|ios { + system(which ccache) { + message("Found ccache, enabling") + !ios { + QMAKE_CXX = ccache $$QMAKE_CXX + QMAKE_CC = ccache $$QMAKE_CC + } else { + QMAKE_CXX = $$PWD/tools/iosccachecc.sh + QMAKE_CC = $$PWD/tools/iosccachecxx.sh + } + } +} + MobileBuild { DEFINES += __mobile__ } diff --git a/tools/iosccachecc.sh b/tools/iosccachecc.sh new file mode 100755 index 000000000..01c52e0d3 --- /dev/null +++ b/tools/iosccachecc.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +ccache $IOS_CCACHE_CC "$@" \ No newline at end of file diff --git a/tools/iosccachecxx.sh b/tools/iosccachecxx.sh new file mode 100755 index 000000000..e9c075a2f --- /dev/null +++ b/tools/iosccachecxx.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +ccache $IOS_CCACHE_CXX "$@" \ No newline at end of file -- 2.22.0