From 729b10e1e0fa7e02ae50f531be57badb64c5a539 Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Wed, 4 Mar 2020 17:49:43 +0300 Subject: [PATCH] Github Actions (#11528) * actions * add build scripts * chmod +x * remove clang env * add ARM builds * install LLVM for Windows * check LLVM * remove fi * check platform * fix LLVM install on windows * diff cache * remove unexpected token && * remove build directory cache from windows * remove Show ENV * remove checksum * fix cross and RUSTFLAGS * final fix for ARM and Windows build * typo * fix PATH for artifacts * sudo for install set PATH . for artifacts.zip * target names for artifacts * platform names for artifacts * remove ARM builds PLATFORM for build scripts * remove PLATFORM and change PATH for artifacts.zip * change PATH for artifacts * add sccache div to test and build * env RUSTC_WRAPPER sccache * sccache tag 0.2.13 up cache to latest v1.1.2 set SCCACHE_CACHE_SIZE: "1G" * remove dependencies from build * add audit check add tags for Build sccache --show-stats * remove beta and nightly toolchains * refactoring cache && sccache * fix Windows PATH o sccache HOME * "*"->'*' * install scache from https://github.com/mozilla/sccache/releases * fix lint * url for sccache * sccache up * shell: pwsh for sccache * up install script for sccache * fix link * splitting tests into multiple tasks * Jobs renaming * typo fix * remove Continue on error * TEST: remove cargo build cache * TEST: sccache show stats before * TEST: 1 * TEST: 2 * TEST: 3 MacOS fix sccache PATH * TEST 4: enable target cache * remove build * TEST 5: check cache and up Cargo.toml * TEST 6 * TEST 7: restore cargo cache splt build-tests and run-tests * TEST 8: fix build-test * TEST 9: fix sccache * TEST 10: lint run-test * TEST 11: build and test in one job * prepare for merge * test.yml -> check.yml * add install sccache script * sh -> ps1 * add os arg to ps cript * os up * fix os arg * ls sccache tar * echo url * echo sccache url * add swith for select platform * fix args * TEST CLEAN 1: clean biuld dir for all platforms * TEST CLEAN 2: incude build step continue-on-error in test * TEST CLEAN 3: script for clean * ready for merge --- .github/workflows/audit.yml | 18 +++++ .github/workflows/build-test.yml | 98 ++++++++++++++++++++++++++ .github/workflows/build.yml | 90 +++++++++++++++++++++++ .github/workflows/check.yml | 85 ++++++++++++++++++++++ scripts/actions/build-linux.sh | 22 ++++++ scripts/actions/build-windows.sh | 22 ++++++ scripts/actions/clean-target.sh | 8 +++ scripts/actions/install-sccache.ps1 | 21 ++++++ scripts/actions/validate-chainspecs.sh | 27 +++++++ 9 files changed, 391 insertions(+) create mode 100644 .github/workflows/audit.yml create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/check.yml create mode 100755 scripts/actions/build-linux.sh create mode 100755 scripts/actions/build-windows.sh create mode 100755 scripts/actions/clean-target.sh create mode 100755 scripts/actions/install-sccache.ps1 create mode 100755 scripts/actions/validate-chainspecs.sh diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 000000000..4c6b7e224 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,18 @@ +name: Security audit +on: + pull_request: + paths: Cargo.lock + schedule: + - cron: '0 0 * * *' +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v1 + with: + fetch-depth: 50 + - name: Run cargo audit + uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000..ebd8b7e60 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,98 @@ +name: Build and Test Suite + +on: + pull_request: + push: + branches: + - master + - stable +jobs: + build-tests: + name: Test and Build + strategy: + matrix: + platform: + - ubuntu-16.04 + - macos-latest + - windows-latest + toolchain: + - stable + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout sources + uses: actions/checkout@v2 + with: + fetch-depth: 50 + - name: Checkout submodules + shell: bash + run: git submodule update --init --recursive + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + override: true + - name: Cache cargo registry + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo index + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-git-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo build + uses: actions/cache@v1.1.2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache linux + if: matrix.platform == 'ubuntu-16.04' + uses: actions/cache@v1.1.2 + with: + path: "/home/runner/.cache/sccache" + key: ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache MacOS + if: matrix.platform == 'macos-latest' + uses: actions/cache@v1.1.2 + with: + path: "/Users/runner/Library/Caches/Mozilla.sccache" + key: ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache Windows + if: matrix.platform == 'windows-latest' + uses: actions/cache@v1.1.2 + with: + path: "C:\\Users\\runneradmin\\AppData\\Local\\Mozilla\\sccache\\cache" + key: ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }} + - name: Install sccache for ${{ matrix.platform }} + shell: pwsh + run: pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}} + - name: Install LLVM for Windows + if: matrix.platform == 'windows-latest' + run: choco install llvm + - name: Sccache statistics + run: sccache --show-stats + - name: Build tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all --release --features "json-tests" --verbose --no-run + - name: Run tests for ${{ matrix.platform }} + if: matrix.platform == 'windows-latest' + continue-on-error: true #Skip step if Windows tests failure + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all --release --features "json-tests" --verbose + - name: Run tests for ${{ matrix.platform }} + if: matrix.platform != 'windows-latest' + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all --release --features "json-tests" --verbose + - name: Stop sccache + if: always() + run: sccache --stop-server + - name: Prepare build directory for cache + shell: bash + run: bash scripts/actions/clean-target.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..5f8149c04 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,90 @@ +name: Build Release Suite + +on: + push: + branches: + - stable + tags: + - v2* + +jobs: + build: + name: Build Release + strategy: + matrix: + platform: + - ubuntu-16.04 + - macos-latest + - windows-latest + toolchain: + - stable + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout sources + uses: actions/checkout@v2 + with: + fetch-depth: 50 + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + profile: minimal + override: true + - name: Cache cargo registry + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo index + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-git-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo build + uses: actions/cache@v1.1.2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache linux + if: matrix.platform == 'ubuntu-16.04' + uses: actions/cache@v1.1.2 + with: + path: "/home/runner/.cache/sccache" + key: ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache MacOS + if: matrix.platform == 'macos-latest' + uses: actions/cache@v1.1.2 + with: + path: "/Users/runner/Library/Caches/Mozilla.sccache" + key: ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache sccache Windows + if: matrix.platform == 'windows-latest' + uses: actions/cache@v1.1.2 + with: + path: "C:\\Users\\runneradmin\\AppData\\Local\\Mozilla\\sccache\\cache" + key: ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }} + - name: Install sccache for ${{ matrix.platform }} + shell: pwsh + run: pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}} + - name: Install LLVM for Windows + if: matrix.platform == 'windows-latest' + run: choco install llvm + - name: Sccache statistics + run: sccache --show-stats + - name: Build OpenEthereum for windows + if: matrix.platform == 'windows-latest' + run: sh scripts/actions/build-windows.sh ${{matrix.platform}} + - name: Build OpenEthereum for ${{matrix.platform}} + if: matrix.platform != 'windows-latest' + run: sh scripts/actions/build-linux.sh ${{matrix.platform}} + - name: Stop sccache + if: always() + run: sccache --stop-server + - name: Download artifact + uses: actions/upload-artifact@v1 + with: + name: ${{matrix.platform}}.artifacts.zip + path: artifacts/ + - name: Prepare build directory for cache + shell: bash + run: bash scripts/actions/clean-target.sh diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000..f4059a461 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,85 @@ +name: Check + +on: + pull_request: + push: + branches: + - master + - stable +jobs: + check: + name: Check + runs-on: ubuntu-16.04 + steps: + - name: Checkout sources + uses: actions/checkout@v2 + with: + fetch-depth: 50 + - name: Checkout submodules + shell: bash + run: git submodule update --init --recursive + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - name: Cache cargo registry + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo index + uses: actions/cache@v1.1.2 + with: + path: ~/.cargo/git + key: ${{ runner.os }}-cargo-git-${{ hashFiles('**/Cargo.lock') }} + - name: Cache cargo build + uses: actions/cache@v1.1.2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} + # Install sccache based on https://github.com/denoland/rusty_v8/blob/master/.github/workflows/ci.yml#L69 + - name: Cache sccache + uses: actions/cache@v1.1.2 + with: + path: "/home/runner/.cache/sccache" + key: ${{ runner.os }}-sccache-check-${{ hashFiles('**/Cargo.lock') }} + - name: Install sccache for Linux + shell: pwsh + run: pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}} + - name: Sccache statistics + run: sccache --show-stats + - name: Run cargo check 1/3 + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --no-default-features --verbose + - name: Run cargo check 2/3 + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose + - name: Run cargo check 3/3 + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose + - name: Run cargo check evmbin + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked -p evmbin --verbose + - name: Run cargo check benches + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --all --benches --verbose + - name: Run validate chainspecs + run: ./scripts/actions/validate-chainspecs.sh + - name: Stop sccache + if: always() + run: sccache --stop-server + - name: Prepare build directory for cache + shell: bash + run: bash scripts/actions/clean-target.sh diff --git a/scripts/actions/build-linux.sh b/scripts/actions/build-linux.sh new file mode 100755 index 000000000..994d2afa7 --- /dev/null +++ b/scripts/actions/build-linux.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e # fail on any error +set -u # treat unset variables as error +#strip ON +export RUSTFLAGS=" -Clink-arg=-s -Ctarget-feature=+aes,+sse2,+ssse3" + +echo "_____ Build OpenEthereum and tools _____" + +time cargo build --verbose --color=always --release --features final +time cargo build --verbose --color=always --release -p evmbin +time cargo build --verbose --color=always --release -p ethstore-cli +time cargo build --verbose --color=always --release -p ethkey-cli + +echo "_____ Post-processing binaries _____" +rm -rf artifacts/* +mkdir -p artifacts/ + +cp -v target/release/parity artifacts/parity +cp -v target/release/parity-evm artifacts/parity-evm +cp -v target/release/ethstore artifacts/ethstore +cp -v target/release/ethkey artifacts/ethkey diff --git a/scripts/actions/build-windows.sh b/scripts/actions/build-windows.sh new file mode 100755 index 000000000..df8966d27 --- /dev/null +++ b/scripts/actions/build-windows.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e # fail on any error +set -u # treat unset variables as error + + # NOTE: Enables the aes-ni instructions for RustCrypto dependency. + # If you change this please remember to also update .cargo/config +export RUSTFLAGS=" -Ctarget-feature=+aes,+sse2,+ssse3 -Ctarget-feature=+crt-static -Clink-arg=-s" + +echo "_____ Build OpenEthereum and tools _____" +time cargo build --verbose --release --features final +time cargo build --verbose --release -p evmbin +time cargo build --verbose --release -p ethstore-cli +time cargo build --verbose --release -p ethkey-cli + +echo "_____ Post-processing binaries _____" +rm -rf artifacts +mkdir -p artifacts + +cp --verbose target/release/parity.exe artifacts/parity.exe +cp --verbose target/release/parity-evm.exe artifacts/parity-evm.exe +cp --verbose target/release/ethstore.exe artifacts/ethstore.exe +cp --verbose target/release/ethkey.exe artifacts/ethkey.exe diff --git a/scripts/actions/clean-target.sh b/scripts/actions/clean-target.sh new file mode 100755 index 000000000..6d161d5c2 --- /dev/null +++ b/scripts/actions/clean-target.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e # fail on any error +set -u # treat unset variables as error + +find ./target/release -maxdepth 1 -type f -delete; +rm -fr ./target/release/{deps,.fingerprint}/*{parity,ethcore,ethkey,ethstore,parity-evm}*; +rm -f ./target/.rustc_info.json; diff --git a/scripts/actions/install-sccache.ps1 b/scripts/actions/install-sccache.ps1 new file mode 100755 index 000000000..eff586c43 --- /dev/null +++ b/scripts/actions/install-sccache.ps1 @@ -0,0 +1,21 @@ +#!/usr/bin/env pwsh +$os=$args[0] +$SCCACHE_CACHE_SIZE="1G" +$SCCACHE_IDLE_TIMEOUT=0 +$version="0.2.12" +echo "Current OS:" $os +switch ($os){ + "macOS" {$platform = "x86_64-apple-darwin"} + "Linux" {$platform = "x86_64-unknown-linux-musl"} + "Windows" {$platform ="x86_64-pc-windows-msvc"} +} +echo "Target arch: " $platform +$basename = "sccache-$version-$platform" +$url = "https://github.com/mozilla/sccache/releases/download/"+"$version/$basename.tar.gz" +echo "Download sccache from "+$url +curl -LO $url +tar -xzvf "$basename.tar.gz" +ls $basename/ +. $basename/sccache --start-server +echo "::add-path::$(pwd)/$basename" +echo "::set-env name=RUSTC_WRAPPER::sccache" diff --git a/scripts/actions/validate-chainspecs.sh b/scripts/actions/validate-chainspecs.sh new file mode 100755 index 000000000..bf88b74c0 --- /dev/null +++ b/scripts/actions/validate-chainspecs.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e # fail on any error +set -u # treat unset variables as error +echo "________Running validate_chainspecs.sh________" + +ERR=0 + +echo "________Validate chainspecs________" +time cargo build --release -p chainspec --verbose --color=always + +for spec in ethcore/res/*.json; do + if ! ./target/release/chainspec "$spec"; then ERR=1; fi +done + +for spec in ethcore/res/ethereum/*.json; do + if ! ./target/release/chainspec "$spec"; then ERR=1; fi +done + +echo "________Mainnet contains Istanbul EIPs________" +for eip in $(grep --only-matching "eip.*Transition" ethcore/res/ethereum/istanbul_test.json); do + if ! grep -q $eip ethcore/res/ethereum/foundation.json; then + echo "ERROR: $eip is missing in the foundation json spec" + ERR=1 + fi +done + +exit $ERR