2.3.6 stable backports (#10470)

* version: bump stable

* CI publish to aws (#10446)

* move publish aws from gitlab.yml to gitlab scripts

* gitlab.yml cleaning
move publish AWS to gitlab scripts
remove dependencies from android build

* Ensure static validator set changes are recognized (#10467)

* CI aws git checkout (#10451)

* Updating the CI system with the publication of releases and binary files on github

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* move publish aws from gitlab.yml to gitlab scripts

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* gitlab.yml cleaning
move publish AWS to gitlab scripts
remove dependencies from android build

Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com>

* Revert "Updating the CI system with the publication of releases and binary files on github"

This reverts commit da87e06f2e4751dbca08a898b52926aef5ad0aba.

* remove no-git for aws

* microfix

* no need in no_git then

* Revert "CI aws git checkout (#10451)" (#10456)

* Revert "CI aws git checkout (#10451)"

This reverts commit 3e1d73126c.

* Update .gitlab-ci.yml

revert aws script with small fixes

* Delete publish-aws.sh

* Tests parallelized (#10452)

* tests splitted, phase 1

* typo

* fix wrong launch commands

* typos

* rearrangements

* use `nproc` function for threads

* use nproc for threads

* let theads be auto, build-andriod no more in regular run

* split val chain and cargo check

* renamed some files

* wrong phase

* check rust files before test jobs

* lint error

* rust files modivied var

* test except changes

* add rust_changes except

* lint error

* fixes

* .gitlab-ci.yml can't be excluded

* pipeline shouldn't start

* pipeline must go

* pipeline must go 2

* pipeline must go 3

* pipeline must go 4

* pipeline must go 5

* pipeline must go 6

* pipeline must go 7

* pipeline must not go 1

* pipeline must go 8

* avoid skippng tests yet, reintroducing them after the caching

* test theory

* parallelized cargo check with combusting helicopters

* less uploads

* alias for cargo checks

* nice template

* Fix `ci-skip-issue` name

* Fix build flag

* Remove ci-skip-tests flag for stable

.gitlab-ci.yml

@@ -12,12 +12,11 @@ variables:
   CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
 
-.no_git:                           &no_git
+.no_git:                           &no_git #disable git strategy
   variables:
     GIT_STRATEGY:                  none
     GIT_SUBMODULE_STRATEGY:        none
 
-
 .releaseable_branches:             # list of git refs for building GitLab artifacts (think "pre-release binaries")
   only:                            &releaseable_branches
     - stable
@@ -25,7 +24,6 @@ variables:
     - tags
     - schedules
 
-
 .collect_artifacts:                &collect_artifacts
   artifacts:
     name:                          "${CI_JOB_NAME}_${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}"
@@ -34,34 +32,66 @@ variables:
     paths:
       - artifacts/
 
-test-linux:
+.docker-cache-status:                      &docker-cache-status
-  stage:                           test
+  dependencies:                    []
-  variables:
+  before_script:
-    RUN_TESTS:                     all
+    - sccache -s
-  script:
+  after_script:
-    - scripts/gitlab/test-all.sh
     - sccache -s
   tags:
     - linux-docker
 
-test-audit:
+
+cargo-check 0 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --no-default-features
+
+cargo-check 1 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --no-default-features
+
+cargo-check 2 3:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio"
+
+cargo-audit:
   stage:                           test
   script:
-    - set -e
-    - set -u
     - cargo audit
   tags:
     - linux-docker
 
+validate-chainspecs:
+  stage:                           test
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/validate-chainspecs.sh
+
+test-cpp:
+  stage:                           build
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/test-cpp.sh
+
+test-linux:
+  stage:                           build
+  <<:                              *docker-cache-status
+  script:
+    - ./scripts/gitlab/test-linux.sh
+
 build-linux:                       &build-linux
   stage:                           build
   only:                            *releaseable_branches
+  <<:                              *docker-cache-status
   script:
-    - scripts/gitlab/build-unix.sh
+    - scripts/gitlab/build-linux.sh
-    - sccache -s
   <<:                              *collect_artifacts
-  tags:
-    - linux-docker
 
 build-linux-i386:
   <<:                              *build-linux
@@ -89,7 +119,7 @@ build-darwin:
     CC:                            gcc
     CXX:                           g++
   script:
-    - scripts/gitlab/build-unix.sh
+    - scripts/gitlab/build-linux.sh
   tags:
     - rust-osx
   <<:                              *collect_artifacts
@@ -102,7 +132,7 @@ build-windows:
   script:
     - sh scripts/gitlab/build-windows.sh
   tags:
-   - rust-windows
+    - rust-windows
   <<:                              *collect_artifacts
 
 publish-docker:
@@ -178,17 +208,16 @@ publish-awss3-release:
   script:
     - echo "__________Push binaries to AWS S3____________"
     - case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
-        (beta|stable|nightly)
+       (beta|stable|nightly)
-          export BUCKET=releases.parity.io/ethereum;
+         export BUCKET=releases.parity.io/ethereum;
-          ;;
+         ;;
-        (*)
+       (*)
-          export BUCKET=builds-parity;
+         export BUCKET=builds-parity;
-          ;;
+         ;;
-      esac
+     esac
     - aws s3 sync ./artifacts s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
-  after_script:
+    - echo "__________Read from S3____________"
-    - aws s3 ls s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
+    - aws s3 ls s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}} --recursive --human-readable --summarize
-        --recursive --human-readable --summarize
   tags:
     - linux-docker
 
@@ -205,13 +234,12 @@ publish-docs:
     - linux-docker
 
 build-android:
-  stage:                           optional
+  stage:                           build
   image:                           parity/rust-android:gitlab-ci
   variables:
     CARGO_TARGET:                  armv7-linux-androideabi
   script:
-    - scripts/gitlab/build-unix.sh
+    - scripts/gitlab/build-linux.sh
   tags:
     - linux-docker
-  allow_failure:                   true
   <<:                              *collect_artifacts

Cargo.lock

@@ -2393,7 +2393,7 @@ dependencies = [
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "panic_hook 0.1.0",
- "parity-ethereum 2.3.5",
+ "parity-ethereum 2.3.6",
  "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-current-thread 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2411,7 +2411,7 @@ dependencies = [
 
 [[package]]
 name = "parity-ethereum"
-version = "2.3.5"
+version = "2.3.6"
 dependencies = [
  "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2462,7 +2462,7 @@ dependencies = [
  "parity-rpc 1.12.0",
  "parity-runtime 0.1.0",
  "parity-updater 1.12.0",
- "parity-version 2.3.5",
+ "parity-version 2.3.6",
  "parity-whisper 0.1.0",
  "parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2614,7 +2614,7 @@ dependencies = [
  "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-runtime 0.1.0",
  "parity-updater 1.12.0",
- "parity-version 2.3.5",
+ "parity-version 2.3.6",
  "parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2712,7 +2712,7 @@ dependencies = [
  "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-hash-fetch 1.12.0",
  "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "parity-version 2.3.5",
+ "parity-version 2.3.6",
  "parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2722,7 +2722,7 @@ dependencies = [
 
 [[package]]
 name = "parity-version"
-version = "2.3.5"
+version = "2.3.6"
 dependencies = [
  "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -2,7 +2,7 @@
 description = "Parity Ethereum client"
 name = "parity-ethereum"
 # NOTE Make sure to update util/version/Cargo.toml as well
-version = "2.3.5"
+version = "2.3.6"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 

ethcore/src/engines/authority_round/mod.rs

@@ -1406,8 +1406,10 @@ impl Engine<EthereumMachine> for AuthorityRound {
 
 		let first = chain_head.number() == 0;
 
-		// apply immediate transitions.
+		// Apply transitions that don't require finality and should be enacted immediately (e.g from chain spec)
 		if let Some(change) = self.validators.is_epoch_end(first, chain_head) {
+			info!(target: "engine", "Immediately applying validator set change signalled at block {}", chain_head.number());
+			self.epoch_manager.lock().note_new_epoch();
 			let change = combine_proofs(chain_head.number(), &change, &[]);
 			return Some(change)
 		}

ethcore/src/json_tests/skip.rs

@@ -18,7 +18,7 @@
 
 use ethjson;
 
-#[cfg(all(not(test), feature = "ci-skip-tests"))]
+#[cfg(all(not(test), feature = "ci-skip-issue"))]
 compile_error!("ci-skip-tests can only be enabled for testing builds.");
 
 #[cfg(feature="ci-skip-issue")]

scripts/gitlab/rust-changes.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
+echo "________Running rust_changes.sh________"
 set -e # fail on any error
 set -u # treat unset variables as error
 
+echo "__________Checking if Rust files were changed__________"
 git log --graph --oneline --decorate=short -n 10
 
 case ${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}} in
@@ -26,5 +28,3 @@ then
 fi
 
 rustup show
-
-exec ./test.sh

scripts/gitlab/test-cpp.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+echo "________Running test-cpp.sh________"
+set -e # fail on any error
+set -u # treat unset variables as error
+#use nproc `linux only
+THREADS=$(nproc)
+
+echo "________Running the C++ example________"
+DIR=parity-clib/examples/cpp/build
+mkdir -p $DIR
+cd $DIR
+cmake ..
+make -j $THREADS
+# Note: we don't try to run the example because it tries to sync Kovan, and we don't want
+#       that to happen on CI
+cd -
+rm -rf $DIR

scripts/gitlab/test-linux.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+echo "________Running test-linux.sh________"
+set -e # fail on any error
+set -u # treat unset variables as error
+
+FEATURES="json-tests"
+OPTIONS="--release"
+#use nproc `linux only
+THREADS=$(nproc)
+
+echo "________Running Parity Full Test Suite________"
+time cargo test $OPTIONS --features "$FEATURES" --locked --all --target $CARGO_TARGET -- --test-threads $THREADS

scripts/gitlab/validate-chainspecs.sh

@@ -1,7 +1,12 @@
-#!/usr/bin/env sh
+#!/bin/bash
+set -e # fail on any error
+set -u # treat unset variables as error
+echo "________Running validate_chainspecs.sh________"
 
 ERR=0
-cargo build --release -p chainspec
+
+echo "________Validate chainspecs________"
+time cargo build --release -p chainspec
 
 for spec in ethcore/res/*.json; do
     if ! ./target/release/chainspec "$spec"; then ERR=1; fi

test.sh

@@ -1,79 +0,0 @@
-#!/bin/sh
-# Running Parity Full Test Suite
-echo "________Running test.sh________"
-
-FEATURES="json-tests,ci-skip-issue"
-OPTIONS="--release"
-VALIDATE=1
-THREADS=8
-
-set -e
-
-
-validate () {
-  if [ "$VALIDATE" -eq "1" ]
-  then
-    echo "________Validate build________"
-    time cargo check $@ --locked --no-default-features
-    time cargo check $@ --locked --manifest-path util/io/Cargo.toml --no-default-features
-    time cargo check $@ --locked --manifest-path util/io/Cargo.toml --features "mio"
-
-    # Validate chainspecs
-    echo "________Validate chainspecs________"
-    time ./scripts/validate_chainspecs.sh
-  else
-    echo "# not validating due to \$VALIDATE!=1"
-  fi
-}
-
-cpp_test () {
-  case $CARGO_TARGET in
-    (x86_64-unknown-linux-gnu)
-      # Running the C++ example
-      echo "________Running the C++ example________"
-      DIR=parity-clib/examples/cpp/build
-      mkdir -p $DIR
-      cd $DIR
-      cmake ..
-      make -j $THREADS
-      # Note: we don't try to run the example because it tries to sync Kovan, and we don't want
-      #       that to happen on CI
-      cd -
-      rm -rf $DIR
-      ;;
-    (*)
-      echo "________Skipping the C++ example________"
-      ;;
-  esac
-}
-
-cargo_test () {
-  echo "________Running Parity Full Test Suite________"
-  git submodule update --init --recursive
-  time cargo test $OPTIONS --features "$FEATURES" --locked --all $@ -- --test-threads $THREADS
-}
-
-
-if [ "$CARGO_TARGET" ]
-then
-  validate --target $CARGO_TARGET
-else
-  validate
-fi
-
-test "${RUN_TESTS}" = "all" && cpp_test
-
-if [ "$CARGO_TARGET" ]
-then
-
-  case "${RUN_TESTS}" in
-    (cargo|all)
-      cargo_test --target $CARGO_TARGET $@
-      ;;
-    ('')
-      cargo_test --no-run --target $CARGO_TARGET $@
-      ;;
-  esac
-else
-  cargo_test $@
-fi

util/version/Cargo.toml

@@ -3,7 +3,7 @@
 [package]
 name = "parity-version"
 # NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION)
-version = "2.3.5"
+version = "2.3.6"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"