grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

CI improvements (#10579)

Browse Source 
* check if it was the file

* collect tog files

* some debug

* some more debug

* some even more debug

* upload all the logs

* change cache max size

* test with max cache size overflow

* verbose for all cargo jobs

* lower max cache size

* Rust nightly and beta test runs reintroduced

* artifacts:when: always, coz on_success is default

* sccache full log

* normal artifacts name, big cache size

* add file to test the compilation again

* another way of getting artifacts

* per-job cache

* data race tests

* is it needed to clean the working folder?

* return to common cache dir

* do not pass logs between jobs

* typo

* avoid artifacts conflict

* colored logs

* always colored logs

* proper stages

* more proper stages skip-ci

* more proper stages [skip-ci]

* shouldn't skip ci
This commit is contained in:
TriplEight 2019-04-12 18:51:01 +02:00 committed by GitHub
parent 944bf6a59e
commit 6fcd775d48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 98 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
.gitlab-ci.yml
View File

@ -31,40 +31,75 @@ variables:
paths: paths:
- artifacts/ - artifacts/
.collect_logs: &collect_logs
artifacts:
name: "${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}"
when: always
expire_in: 7 days
paths:
- artifacts/
.docker-cache-status: &docker-cache-status .docker-cache-status: &docker-cache-status
variables: variables:
CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}" CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}"
dependencies: []
before_script: before_script:
- SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_error.log RUST_LOG=sccache::server=debug sccache --start-server - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log
RUST_LOG=sccache::server=debug
SCCACHE_CACHE_SIZE=50G
SCCACHE_DIR=/ci-cache/parity-ethereum/sccache/
sccache --start-server
- sccache -s - sccache -s
after_script: after_script:
- echo "All crate-types:" # sccache debug info
- grep 'parse_arguments.*--crate-type' sccache_error.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c - if test -e sccache_debug.log;
- echo "Non-cacheable reasons:" then
- grep CannotCache sccache_error.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c echo "All crate-types:";
grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c;
echo "Non-cacheable reasons:";
grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c;
else
echo "No logs from sccache";
exit 0;
fi
# collect log files
- mkdir -p ./artifacts
- find . -name "*.log"
- find . -name "*.log" | xargs tar --append -f ./artifacts/logs_"${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}".tar
tags: tags:
- linux-docker - linux-docker
.build-on-linux: &build-on-linux
stage: build
<<: *docker-cache-status
<<: *collect_artifacts
script:
- scripts/gitlab/build-linux.sh
- sccache -s
cargo-check 0 3: cargo-check 0 3:
stage: test stage: test
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- time cargo check --target $CARGO_TARGET --locked --no-default-features - time cargo check --target $CARGO_TARGET --locked --no-default-features --verbose --color=always
- sccache -s - sccache -s
cargo-check 1 3: cargo-check 1 3:
stage: test stage: test
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose --color=always
- sccache -s - sccache -s
cargo-check 2 3: cargo-check 2 3:
stage: test stage: test
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always
- sccache -s - sccache -s
cargo-audit: cargo-audit:
@ -72,11 +107,11 @@ cargo-audit:
<<: *docker-cache-status <<: *docker-cache-status
script: script:
- cargo audit - cargo audit
- sccache -s
validate-chainspecs: validate-chainspecs:
stage: test stage: test
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- ./scripts/gitlab/validate-chainspecs.sh - ./scripts/gitlab/validate-chainspecs.sh
- sccache -s - sccache -s
@ -84,6 +119,7 @@ validate-chainspecs:
test-cpp: test-cpp:
stage: build stage: build
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- ./scripts/gitlab/test-cpp.sh - ./scripts/gitlab/test-cpp.sh
- sccache -s - sccache -s
@ -91,56 +127,68 @@ test-cpp:
test-linux: test-linux:
stage: build stage: build
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_logs
script: script:
- ./scripts/gitlab/test-linux.sh - ./scripts/gitlab/test-linux.sh stable
- sccache -s - sccache -s
build-android: test-linux-beta:
stage: build
image: parity/rust-parity-ethereum-android-build:stretch
variables:
CARGO_TARGET: armv7-linux-androideabi
<<: *docker-cache-status
<<: *collect_artifacts
script:
- scripts/gitlab/build-linux.sh
tags:
- linux-docker
build-linux: &build-linux
stage: build stage: build
only: *releaseable_branches only: *releaseable_branches
<<: *docker-cache-status <<: *docker-cache-status
<<: *collect_artifacts <<: *collect_logs
script: script:
- scripts/gitlab/build-linux.sh - ./scripts/gitlab/test-linux.sh beta
- sccache -s - sccache -s
test-linux-nightly:
stage: build
only: *releaseable_branches
<<: *docker-cache-status
<<: *collect_logs
script:
- ./scripts/gitlab/test-linux.sh nightly
- sccache -s
allow_failure: true
build-android:
<<: *build-on-linux
image: parity/rust-parity-ethereum-android-build:stretch
variables:
CARGO_TARGET: armv7-linux-androideabi
build-linux:
<<: *build-on-linux
only: *releaseable_branches
build-linux-i386: build-linux-i386:
<<: *build-linux <<: *build-on-linux
only: *releaseable_branches
image: parity/rust-parity-ethereum-build:i386 image: parity/rust-parity-ethereum-build:i386
variables: variables:
CARGO_TARGET: i686-unknown-linux-gnu CARGO_TARGET: i686-unknown-linux-gnu
build-linux-arm64: build-linux-arm64:
<<: *build-linux <<: *build-on-linux
only: *releaseable_branches
image: parity/rust-parity-ethereum-build:arm64 image: parity/rust-parity-ethereum-build:arm64
variables: variables:
CARGO_TARGET: aarch64-unknown-linux-gnu CARGO_TARGET: aarch64-unknown-linux-gnu
build-linux-armhf: build-linux-armhf:
<<: *build-linux <<: *build-on-linux
only: *releaseable_branches
image: parity/rust-parity-ethereum-build:armhf image: parity/rust-parity-ethereum-build:armhf
variables: variables:
CARGO_TARGET: armv7-unknown-linux-gnueabihf CARGO_TARGET: armv7-unknown-linux-gnueabihf
build-darwin: build-darwin:
stage: build stage: build
only: *releaseable_branches
<<: *collect_artifacts <<: *collect_artifacts
only: *releaseable_branches
variables: variables:
CARGO_TARGET: x86_64-apple-darwin CARGO_TARGET: x86_64-apple-darwin
CARGO_HOME: "${CI_PROJECT_DIR}/.cargo" CARGO_HOME: "${CI_PROJECT_DIR}/.cargo"
CC: gcc CC: gcc
CXX: g++ CXX: g++
script: script:

4
parity-clib/examples/cpp/CMakeLists.txt
View File

@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.5) cmake_minimum_required(VERSION 3.5)
include(ExternalProject) include(ExternalProject)
include_directories("${CMAKE_SOURCE_DIR}/../..") include_directories("${CMAKE_SOURCE_DIR}/../..")
set (CMAKE_CXX_STANDARD 11) # Enfore C++11 set (CMAKE_CXX_STANDARD 11) # Enforce C++11
add_executable(parity-example main.cpp) add_executable(parity-example main.cpp)
ExternalProject_Add( ExternalProject_Add(
@ -9,7 +9,7 @@ ExternalProject_Add(
DOWNLOAD_COMMAND "" DOWNLOAD_COMMAND ""
CONFIGURE_COMMAND "" CONFIGURE_COMMAND ""
BUILD_COMMAND "" BUILD_COMMAND ""
COMMAND cargo build -p parity-clib # Note: use --release in a real project COMMAND cargo build -p parity-clib --verbose --color=always # Note: use --release in a real project
BINARY_DIR "${CMAKE_SOURCE_DIR}/../../../target" BINARY_DIR "${CMAKE_SOURCE_DIR}/../../../target"
INSTALL_COMMAND "" INSTALL_COMMAND ""
LOG_BUILD ON) LOG_BUILD ON)

12
scripts/gitlab/build-linux.sh
View File

@ -18,13 +18,13 @@ cat .cargo/config
echo "_____ Building target: "$CARGO_TARGET" _____" echo "_____ Building target: "$CARGO_TARGET" _____"
if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ] if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
then then
time cargo build --target $CARGO_TARGET --release -p parity-clib --features final time cargo build --target $CARGO_TARGET --verbose --color=always --release -p parity-clib --features final
else else
time cargo build --target $CARGO_TARGET --release --features final time cargo build --target $CARGO_TARGET --verbose --color=always --release --features final
time cargo build --target $CARGO_TARGET --release -p evmbin time cargo build --target $CARGO_TARGET --verbose --color=always --release -p evmbin
time cargo build --target $CARGO_TARGET --release -p ethstore-cli time cargo build --target $CARGO_TARGET --verbose --color=always --release -p ethstore-cli
time cargo build --target $CARGO_TARGET --release -p ethkey-cli time cargo build --target $CARGO_TARGET --verbose --color=always --release -p ethkey-cli
time cargo build --target $CARGO_TARGET --release -p whisper-cli time cargo build --target $CARGO_TARGET --verbose --color=always --release -p whisper-cli
fi fi
echo "_____ Post-processing binaries _____" echo "_____ Post-processing binaries _____"

10
scripts/gitlab/build-windows.sh
View File

@ -14,11 +14,11 @@ echo "RUSTC_WRAPPER: " $RUSTC_WRAPPER
echo "SCCACHE_DIR: " $SCCACHE_DIR echo "SCCACHE_DIR: " $SCCACHE_DIR
echo "_____ Building target: "$CARGO_TARGET" _____" echo "_____ Building target: "$CARGO_TARGET" _____"
time cargo build --target $CARGO_TARGET --release --features final time cargo build --target $CARGO_TARGET --verbose --release --features final
time cargo build --target $CARGO_TARGET --release -p evmbin time cargo build --target $CARGO_TARGET --verbose --release -p evmbin
time cargo build --target $CARGO_TARGET --release -p ethstore-cli time cargo build --target $CARGO_TARGET --verbose --release -p ethstore-cli
time cargo build --target $CARGO_TARGET --release -p ethkey-cli time cargo build --target $CARGO_TARGET --verbose --release -p ethkey-cli
time cargo build --target $CARGO_TARGET --release -p whisper-cli time cargo build --target $CARGO_TARGET --verbose --release -p whisper-cli
echo "__________Sign binaries__________" echo "__________Sign binaries__________"
scripts/gitlab/sign-win.cmd $keyfile $certpass target/$CARGO_TARGET/release/parity.exe scripts/gitlab/sign-win.cmd $keyfile $certpass target/$CARGO_TARGET/release/parity.exe

7
scripts/gitlab/test-linux.sh
View File

@ -1,4 +1,6 @@
#!/bin/bash #!/bin/bash
# ARGUMENT $1 Rust flavor to run test with (stable/beta/nightly)
echo "________Running test-linux.sh________" echo "________Running test-linux.sh________"
set -e # fail on any error set -e # fail on any error
set -u # treat unset variables as error set -u # treat unset variables as error
@ -8,5 +10,8 @@ OPTIONS="--release"
#use nproc `linux only #use nproc `linux only
THREADS=$(nproc) THREADS=$(nproc)
rustup default $1
rustup show
echo "________Running Parity Full Test Suite________" echo "________Running Parity Full Test Suite________"
time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET -- --test-threads $THREADS time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET --verbose --color=always -- --test-threads $THREADS

2
scripts/gitlab/validate-chainspecs.sh
View File

@ -6,7 +6,7 @@ echo "________Running validate_chainspecs.sh________"
ERR=0 ERR=0
echo "________Validate chainspecs________" echo "________Validate chainspecs________"
time cargo build --release -p chainspec time cargo build --release -p chainspec --verbose --color=always
for spec in ethcore/res/*.json; do for spec in ethcore/res/*.json; do
if ! ./target/release/chainspec "$spec"; then ERR=1; fi if ! ./target/release/chainspec "$spec"; then ERR=1; fi