CI optimizations (#10297)

* CI optimizations

* fix stripping

* new dockerfile

* no need n submodule upd

* review

* moved dockerfile

* it becomes large

* onchain update depends on s3

* fix dependency

* fix cache status

* fix cache status

* new cache status

.gitlab-ci.yml

@@ -2,11 +2,14 @@ stages:
   - test
   - build
   - publish
+  - publish-onchain
   - optional
 
-image:                             parity/rust:gitlab-ci
+image:                             parity/rust-parity-ethereum-build:stretch
 
 variables:
+  GIT_STRATEGY:                    fetch
+  GIT_SUBMODULE_STRATEGY:          recursive
   CI_SERVER_NAME:                  "GitLab CI"
   CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
@@ -40,25 +43,28 @@ test-linux:
   variables:
     RUN_TESTS:                     all
   script:
-    - scripts/gitlab/test-all.sh stable
+    - scripts/gitlab/test-all.sh
   tags:
-    - rust-stable
+    - linux-docker
 
 test-audit:
   stage:                           test
   script:
-    - scripts/gitlab/cargo-audit.sh
+    - set -e
+    - set -u
+    - cargo audit
   tags:
-    - rust-stable
+    - linux-docker
 
 build-linux:
   stage:                           build
   only:                            *releaseable_branches
   script:
     - scripts/gitlab/build-unix.sh
+    - sccache -s
   <<:                              *collect_artifacts
   tags:
-    - rust-stable
+    - linux-docker
 
 build-darwin:
   stage:                           build
@@ -112,19 +118,66 @@ publish-snap:
   allow_failure:                   true
   <<:                              *collect_artifacts
 
-publish-awss3:
+publish:onnet:update:
-  stage:                           publish
+  stage:                           publish-onchain
   only:                            *releaseable_branches
-  cache: {}
+  cache:                           {}
   dependencies:
     - build-linux
     - build-darwin
     - build-windows
+    - publish:awss3:release
   before_script:                   *determine_version
   script:
-    - scripts/gitlab/publish-awss3.sh
+    - scripts/gitlab/publish-onnet-update.sh
   tags:
-    - shell
+    - linux-docker
+
+# configures aws for fast uploads/syncs
+.s3_before_script:                &s3_before_script
+  before_script:
+    - mkdir -p ${HOME}/.aws
+    - |
+      cat > ${HOME}/.aws/config <<EOC
+      [default]
+      s3 =
+        max_concurrent_requests = 20
+        max_queue_size = 10000
+        multipart_threshold = 64MB
+        multipart_chunksize = 16MB
+        max_bandwidth = 50MB/s
+        use_accelerate_endpoint = false
+        addressing_style = path
+      EOC
+
+publish:awss3:release:
+  image:                           parity/awscli:latest
+  stage:                           publish
+  only:                            *releaseable_branches
+  cache:                           {}
+  dependencies:
+    - build-linux
+    - build-darwin
+    - build-windows
+  variables:
+    GIT_STRATEGY:                  none
+  <<:                              *s3_before_script
+  script:
+    - echo "__________Push binaries to AWS S3____________"
+    - case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
+        (beta|stable|nightly)
+          export BUCKET=releases.parity.io/ethereum;
+          ;;
+        (*)
+          export BUCKET=builds-parity;
+          ;;
+      esac
+    - aws s3 sync ./artifacts s3://${BUCKET}/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
+  after_script:
+    - aws s3 ls s3://${BUCKET}/latest/
+        --recursive --human-readable --summarize
+  tags:
+  - linux-docker
 
 publish-docs:
   stage:                            publish
@@ -150,22 +203,3 @@ build-android:
   allow_failure:                   true
   <<:                              *collect_artifacts
 
-test-beta:
-  stage:                           optional
-  variables:
-    RUN_TESTS:                     cargo
-  script:
-    - scripts/gitlab/test-all.sh beta
-  tags:
-    - rust-beta
-  allow_failure:                   true
-
-test-nightly:
-  stage:                           optional
-  variables:
-    RUN_TESTS:                     all
-  script:
-    - scripts/gitlab/test-all.sh nightly
-  tags:
-    - rust-nightly
-  allow_failure:                   true

scripts/gitlab/build-unix.sh

@@ -15,7 +15,6 @@ export RUSTFLAGS=" -C link-arg=-s"
 echo "_____ Building target: "$CARGO_TARGET" _____"
 if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
 then
-# only thing we need for android
   time cargo build --target $CARGO_TARGET --release -p parity-clib --features final
 else
   time cargo build --target $CARGO_TARGET --release --features final
@@ -26,14 +25,11 @@ else
 fi
 
 echo "_____ Post-processing binaries _____"
-rm -rf artifacts
+mkdir -p artifacts/$CARGO_TARGET
-mkdir -p artifacts
+cd artifacts/$CARGO_TARGET
-cd artifacts
+
-mkdir -p $CARGO_TARGET
-cd $CARGO_TARGET
 if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
 then
-# only thing we need for android
  cp -v ../../target/$CARGO_TARGET/release/libparity.so ./libparity.so
 else
  cp -v ../../target/$CARGO_TARGET/release/parity ./parity
@@ -54,4 +50,3 @@ do
     ./parity tools hash $binary > $binary.sha3
   fi
 done
-

scripts/gitlab/cargo-audit.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -e # fail on any error
-set -u # treat unset variables as error
-
-cargo install cargo-audit
-cargo audit

scripts/gitlab/publish-onnet-update.sh

@@ -36,19 +36,3 @@ do
   esac
   cd ..
 done
-
-echo "__________Push binaries to AWS S3____________"
-aws configure set aws_access_key_id $s3_key
-aws configure set aws_secret_access_key $s3_secret
-
-case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
-  (beta|stable|nightly)
-    export S3_BUCKET=releases.parity.io/ethereum;
-    ;;
-  (*)
-    export S3_BUCKET=builds-parity;
-    ;;
-esac
-
-aws s3 sync ./ s3://$S3_BUCKET/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/
-

scripts/gitlab/test-all.sh

@@ -1,6 +1,4 @@
 #!/bin/bash
-# ARGUMENT $1 Rust flavor to test with (stable/beta/nightly)
-
 set -e # fail on any error
 set -u # treat unset variables as error
 
@@ -27,9 +25,6 @@ then
   exit 0
 fi
 
-rustup default $1
-
-git submodule update --init --recursive
 rustup show
 
 exec ./test.sh

test.sh

@@ -1,33 +1,12 @@
 #!/bin/sh
 # Running Parity Full Test Suite
+echo "________Running test.sh________"
 
 FEATURES="json-tests,ci-skip-issue"
 OPTIONS="--release"
 VALIDATE=1
 THREADS=8
 
-case $1 in
-  --no-json)
-    FEATURES="ipc"
-    shift # past argument=value
-    ;;
-  --no-release)
-    OPTIONS=""
-    shift
-    ;;
-  --no-validate)
-    VALIDATE=0
-    shift
-    ;;
-  --no-run)
-    OPTIONS="--no-run"
-    shift
-    ;;
-  *)
-    # unknown option
-    ;;
-esac
-
 set -e