From 6fcd775d4892758f5c7b3a37844ce94b54ec9ec2 Mon Sep 17 00:00:00 2001 From: TriplEight Date: Fri, 12 Apr 2019 18:51:01 +0200 Subject: [PATCH] CI improvements (#10579) * check if it was the file * collect tog files * some debug * some more debug * some even more debug * upload all the logs * change cache max size * test with max cache size overflow * verbose for all cargo jobs * lower max cache size * Rust nightly and beta test runs reintroduced * artifacts:when: always, coz on_success is default * sccache full log * normal artifacts name, big cache size * add file to test the compilation again * another way of getting artifacts * per-job cache * data race tests * is it needed to clean the working folder? * return to common cache dir * do not pass logs between jobs * typo * avoid artifacts conflict * colored logs * always colored logs * proper stages * more proper stages skip-ci * more proper stages [skip-ci] * shouldn't skip ci --- .gitlab-ci.yml | 108 +++++++++++++++++------- parity-clib/examples/cpp/CMakeLists.txt | 4 +- scripts/gitlab/build-linux.sh | 12 +-- scripts/gitlab/build-windows.sh | 10 +-- scripts/gitlab/test-linux.sh | 7 +- scripts/gitlab/validate-chainspecs.sh | 2 +- 6 files changed, 98 insertions(+), 45 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 436b8a156..12c65aeff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,40 +31,75 @@ variables: paths: - artifacts/ +.collect_logs: &collect_logs + artifacts: + name: "${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" + when: always + expire_in: 7 days + paths: + - artifacts/ + .docker-cache-status: &docker-cache-status variables: CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}" + dependencies: [] before_script: - - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_error.log RUST_LOG=sccache::server=debug sccache --start-server + - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log + RUST_LOG=sccache::server=debug + SCCACHE_CACHE_SIZE=50G + SCCACHE_DIR=/ci-cache/parity-ethereum/sccache/ + sccache --start-server - sccache -s after_script: - - echo "All crate-types:" - - grep 'parse_arguments.*--crate-type' sccache_error.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c - - echo "Non-cacheable reasons:" - - grep CannotCache sccache_error.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c + # sccache debug info + - if test -e sccache_debug.log; + then + echo "All crate-types:"; + grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c; + echo "Non-cacheable reasons:"; + grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c; + else + echo "No logs from sccache"; + exit 0; + fi + # collect log files + - mkdir -p ./artifacts + - find . -name "*.log" + - find . -name "*.log" | xargs tar --append -f ./artifacts/logs_"${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}".tar tags: - linux-docker +.build-on-linux: &build-on-linux + stage: build + <<: *docker-cache-status + <<: *collect_artifacts + script: + - scripts/gitlab/build-linux.sh + - sccache -s + cargo-check 0 3: stage: test <<: *docker-cache-status + <<: *collect_logs script: - - time cargo check --target $CARGO_TARGET --locked --no-default-features + - time cargo check --target $CARGO_TARGET --locked --no-default-features --verbose --color=always - sccache -s cargo-check 1 3: stage: test <<: *docker-cache-status + <<: *collect_logs script: - - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features + - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose --color=always - sccache -s cargo-check 2 3: stage: test <<: *docker-cache-status + <<: *collect_logs script: - - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" + - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always - sccache -s cargo-audit: @@ -72,11 +107,11 @@ cargo-audit: <<: *docker-cache-status script: - cargo audit - - sccache -s validate-chainspecs: stage: test <<: *docker-cache-status + <<: *collect_logs script: - ./scripts/gitlab/validate-chainspecs.sh - sccache -s @@ -84,6 +119,7 @@ validate-chainspecs: test-cpp: stage: build <<: *docker-cache-status + <<: *collect_logs script: - ./scripts/gitlab/test-cpp.sh - sccache -s @@ -91,56 +127,68 @@ test-cpp: test-linux: stage: build <<: *docker-cache-status + <<: *collect_logs script: - - ./scripts/gitlab/test-linux.sh + - ./scripts/gitlab/test-linux.sh stable - sccache -s -build-android: - stage: build - image: parity/rust-parity-ethereum-android-build:stretch - variables: - CARGO_TARGET: armv7-linux-androideabi - <<: *docker-cache-status - <<: *collect_artifacts - script: - - scripts/gitlab/build-linux.sh - tags: - - linux-docker - -build-linux: &build-linux +test-linux-beta: stage: build only: *releaseable_branches <<: *docker-cache-status - <<: *collect_artifacts + <<: *collect_logs script: - - scripts/gitlab/build-linux.sh + - ./scripts/gitlab/test-linux.sh beta - sccache -s +test-linux-nightly: + stage: build + only: *releaseable_branches + <<: *docker-cache-status + <<: *collect_logs + script: + - ./scripts/gitlab/test-linux.sh nightly + - sccache -s + allow_failure: true + +build-android: + <<: *build-on-linux + image: parity/rust-parity-ethereum-android-build:stretch + variables: + CARGO_TARGET: armv7-linux-androideabi + +build-linux: + <<: *build-on-linux + only: *releaseable_branches + build-linux-i386: - <<: *build-linux + <<: *build-on-linux + only: *releaseable_branches image: parity/rust-parity-ethereum-build:i386 variables: CARGO_TARGET: i686-unknown-linux-gnu build-linux-arm64: - <<: *build-linux + <<: *build-on-linux + only: *releaseable_branches image: parity/rust-parity-ethereum-build:arm64 variables: CARGO_TARGET: aarch64-unknown-linux-gnu build-linux-armhf: - <<: *build-linux + <<: *build-on-linux + only: *releaseable_branches image: parity/rust-parity-ethereum-build:armhf variables: CARGO_TARGET: armv7-unknown-linux-gnueabihf build-darwin: stage: build - only: *releaseable_branches <<: *collect_artifacts + only: *releaseable_branches variables: CARGO_TARGET: x86_64-apple-darwin - CARGO_HOME: "${CI_PROJECT_DIR}/.cargo" + CARGO_HOME: "${CI_PROJECT_DIR}/.cargo" CC: gcc CXX: g++ script: diff --git a/parity-clib/examples/cpp/CMakeLists.txt b/parity-clib/examples/cpp/CMakeLists.txt index 8cc6aef8f..28c6241d4 100644 --- a/parity-clib/examples/cpp/CMakeLists.txt +++ b/parity-clib/examples/cpp/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) include(ExternalProject) include_directories("${CMAKE_SOURCE_DIR}/../..") -set (CMAKE_CXX_STANDARD 11) # Enfore C++11 +set (CMAKE_CXX_STANDARD 11) # Enforce C++11 add_executable(parity-example main.cpp) ExternalProject_Add( @@ -9,7 +9,7 @@ ExternalProject_Add( DOWNLOAD_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND "" - COMMAND cargo build -p parity-clib # Note: use --release in a real project + COMMAND cargo build -p parity-clib --verbose --color=always # Note: use --release in a real project BINARY_DIR "${CMAKE_SOURCE_DIR}/../../../target" INSTALL_COMMAND "" LOG_BUILD ON) diff --git a/scripts/gitlab/build-linux.sh b/scripts/gitlab/build-linux.sh index f0697080c..ebd65cd9b 100755 --- a/scripts/gitlab/build-linux.sh +++ b/scripts/gitlab/build-linux.sh @@ -18,13 +18,13 @@ cat .cargo/config echo "_____ Building target: "$CARGO_TARGET" _____" if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] then - time cargo build --target $CARGO_TARGET --release -p parity-clib --features final + time cargo build --target $CARGO_TARGET --verbose --color=always --release -p parity-clib --features final else - time cargo build --target $CARGO_TARGET --release --features final - time cargo build --target $CARGO_TARGET --release -p evmbin - time cargo build --target $CARGO_TARGET --release -p ethstore-cli - time cargo build --target $CARGO_TARGET --release -p ethkey-cli - time cargo build --target $CARGO_TARGET --release -p whisper-cli + time cargo build --target $CARGO_TARGET --verbose --color=always --release --features final + time cargo build --target $CARGO_TARGET --verbose --color=always --release -p evmbin + time cargo build --target $CARGO_TARGET --verbose --color=always --release -p ethstore-cli + time cargo build --target $CARGO_TARGET --verbose --color=always --release -p ethkey-cli + time cargo build --target $CARGO_TARGET --verbose --color=always --release -p whisper-cli fi echo "_____ Post-processing binaries _____" diff --git a/scripts/gitlab/build-windows.sh b/scripts/gitlab/build-windows.sh index 7ddf4453e..76332124d 100755 --- a/scripts/gitlab/build-windows.sh +++ b/scripts/gitlab/build-windows.sh @@ -14,11 +14,11 @@ echo "RUSTC_WRAPPER: " $RUSTC_WRAPPER echo "SCCACHE_DIR: " $SCCACHE_DIR echo "_____ Building target: "$CARGO_TARGET" _____" -time cargo build --target $CARGO_TARGET --release --features final -time cargo build --target $CARGO_TARGET --release -p evmbin -time cargo build --target $CARGO_TARGET --release -p ethstore-cli -time cargo build --target $CARGO_TARGET --release -p ethkey-cli -time cargo build --target $CARGO_TARGET --release -p whisper-cli +time cargo build --target $CARGO_TARGET --verbose --release --features final +time cargo build --target $CARGO_TARGET --verbose --release -p evmbin +time cargo build --target $CARGO_TARGET --verbose --release -p ethstore-cli +time cargo build --target $CARGO_TARGET --verbose --release -p ethkey-cli +time cargo build --target $CARGO_TARGET --verbose --release -p whisper-cli echo "__________Sign binaries__________" scripts/gitlab/sign-win.cmd $keyfile $certpass target/$CARGO_TARGET/release/parity.exe diff --git a/scripts/gitlab/test-linux.sh b/scripts/gitlab/test-linux.sh index 6a98d2f7b..2854508bb 100755 --- a/scripts/gitlab/test-linux.sh +++ b/scripts/gitlab/test-linux.sh @@ -1,4 +1,6 @@ #!/bin/bash +# ARGUMENT $1 Rust flavor to run test with (stable/beta/nightly) + echo "________Running test-linux.sh________" set -e # fail on any error set -u # treat unset variables as error @@ -8,5 +10,8 @@ OPTIONS="--release" #use nproc `linux only THREADS=$(nproc) +rustup default $1 +rustup show + echo "________Running Parity Full Test Suite________" -time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET -- --test-threads $THREADS +time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET --verbose --color=always -- --test-threads $THREADS diff --git a/scripts/gitlab/validate-chainspecs.sh b/scripts/gitlab/validate-chainspecs.sh index 9b7ef39e7..58391e131 100755 --- a/scripts/gitlab/validate-chainspecs.sh +++ b/scripts/gitlab/validate-chainspecs.sh @@ -6,7 +6,7 @@ echo "________Running validate_chainspecs.sh________" ERR=0 echo "________Validate chainspecs________" -time cargo build --release -p chainspec +time cargo build --release -p chainspec --verbose --color=always for spec in ethcore/res/*.json; do if ! ./target/release/chainspec "$spec"; then ERR=1; fi