Backport Master CI PRs to Beta (#7890)

* Resolve conflicts * Resolve conflicts * Update gitlab-build.sh (#7855) fix build ```version``` after https://github.com/paritytech/parity/pull/7723 * Resolve conflicts * Update gitlab-test.sh (#7883) * Update gitlab-test.sh https://github.com/paritytech/parity/issues/7871 * Update aura-test.sh * Backport master ci prs * Bump beta to 1.9.3 * Make track beta * Downgrade rustc_version
2018-02-14 16:59:16 +01:00
parent b60511e3d2
commit 1de095f3bd
13 changed files with 190 additions and 122 deletions
--- a/scripts/aura-test.sh
+++ b/scripts/aura-test.sh
@@ -1,9 +1,12 @@
 #!/bin/bash
+set -e # fail on any error
+set -u # treat unset variables as error
+
 cargo build -j $(nproc) --release --features final $CARGOFLAGS
 git clone https://github.com/paritytech/parity-import-tests
 cp target/release/parity parity-import-tests/aura/parity
 cd parity-import-tests/aura
 echo "Start Aura test"
-parity import blocks.rlp --chain chain.json
-parity restore snap --chain chain.json
+./parity import blocks.rlp --chain chain.json
+./parity restore snap --chain chain.json
 echo "Aura test complete"
--- a/scripts/gitlab-build.sh
+++ b/scripts/gitlab-build.sh
@@ -2,16 +2,18 @@

 set -e # fail on any error
 set -u # treat unset variables as error
-#ARGUMENTS: 1. BUILD_PLATFORM (target for binaries) 2. PLATFORM (target for cargo) 3. ARC (architecture) 4. & 5. CC & CXX flags
+#ARGUMENTS: 1. BUILD_PLATFORM (target for binaries) 2. PLATFORM (target for cargo) 3. ARC (architecture) 4. & 5. CC & CXX flags 6. binary identifier
 BUILD_PLATFORM=$1
 PLATFORM=$2
 ARC=$3
 CC=$4
 CXX=$5
-VER="$(grep -m 1 version Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n")"
+IDENT=$6
+VER="$(grep -m 1 "version =" Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n")"
 S3WIN=""
 echo "--------------------"
 echo "Build for platform: " $BUILD_PLATFORM
+echo "Build identifier:   " $IDENT
 echo "Cargo target:       " $PLATFORM
 echo "CC&CXX flags:       " $CC ", " $CXX
 echo "Architecture:       " $ARC
@@ -20,6 +22,14 @@ echo "Parity version:     " $VER
 echo "Branch:             " $CI_BUILD_REF_NAME
 echo "--------------------"

+echo "Rhash version:"
+# NOTE for md5 and sha256 we want to display filename as well
+# hence we use --* instead of -p *
+MD5_BIN="rhash --md5"
+SHA256_BIN="rhash --sha256"
+# NOTE For SHA3 we need only hash (hence -p)
+SHA3_BIN="rhash -p %{sha3-256}"
+
 set_env () {
  echo "Set ENVIROMENT"
  export HOST_CC=gcc
@@ -50,21 +60,34 @@ build () {
  echo "Build ethkey-cli:"
  cargo build --target $PLATFORM --release -p ethkey-cli
 }
-strip_md5 () {
+strip_binaries () {
  echo "Strip binaries:"
  $STRIP_BIN -v target/$PLATFORM/release/parity
  $STRIP_BIN -v target/$PLATFORM/release/parity-evm
  $STRIP_BIN -v target/$PLATFORM/release/ethstore
  $STRIP_BIN -v target/$PLATFORM/release/ethkey;
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
+}
+calculate_checksums () {
  echo "Checksum calculation:"
+  rhash --version
  rm -rf *.md5
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
-  echo "Parity file SHA3:" $SHA3
-  md5sum target/$PLATFORM/release/parity > parity.md5
-  md5sum target/$PLATFORM/release/parity-evm > parity-evm.md5
-  md5sum target/$PLATFORM/release/ethstore > ethstore.md5
-  md5sum target/$PLATFORM/release/ethkey > ethkey.md5
+  rm -rf *.sha256
+
+  export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
+  # NOTE rhash 1.3.1 doesnt support keccak, workaround
+  if [ "$SHA3" == "%{sha3-256}" ]; then
+    export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
+  fi
+
+  echo "Parity file SHA3: $SHA3"
+  $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.sha256
 }
 make_deb () {
  rm -rf deb
@@ -98,8 +121,9 @@ make_deb () {
  cp target/$PLATFORM/release/parity-evm deb/usr/bin/parity-evm
  cp target/$PLATFORM/release/ethstore deb/usr/bin/ethstore
  cp target/$PLATFORM/release/ethkey deb/usr/bin/ethkey
-  dpkg-deb -b deb "parity_"$VER"_"$ARC".deb"
-  md5sum "parity_"$VER"_"$ARC".deb" > "parity_"$VER"_"$ARC".deb.md5"
+  dpkg-deb -b deb "parity_"$VER"_"$IDENT"_"$ARC".deb"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.sha256"
 }
 make_rpm () {
  rm -rf /install
@@ -108,9 +132,12 @@ make_rpm () {
  cp target/$PLATFORM/release/parity-evm /install/usr/bin/parity-evm
  cp target/$PLATFORM/release/ethstore /install/usr/bin/ethstore
  cp target/$PLATFORM/release/ethkey /install/usr/bin/ethkey
+
+  rm -rf "parity-"$VER"-1."$ARC".rpm" || true
  fpm -s dir -t rpm -n parity -v $VER --epoch 1 --license GPLv3 -d openssl --provides parity --url https://parity.io --vendor "Parity Technologies" -a x86_64 -m "<devops@parity.io>" --description "Ethereum network client by Parity Technologies" -C /install/
-  cp "parity-"$VER"-1."$ARC".rpm" "parity_"$VER"_"$ARC".rpm"
-  md5sum "parity_"$VER"_"$ARC".rpm" > "parity_"$VER"_"$ARC".rpm.md5"
+  cp "parity-"$VER"-1."$ARC".rpm" "parity_"$VER"_"$IDENT"_"$ARC".rpm"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.sha256"
 }
 make_pkg () {
  echo "make PKG"
@@ -123,19 +150,14 @@ make_pkg () {
  cd ..
  packagesbuild -v mac/Parity.pkgproj
  productsign --sign 'Developer ID Installer: PARITY TECHNOLOGIES LIMITED (P2PX3JU8FT)' target/release/Parity\ Ethereum.pkg target/release/Parity\ Ethereum-signed.pkg
-  mv target/release/Parity\ Ethereum-signed.pkg "parity_"$VER"_"$ARC".pkg"
-  md5sum "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.md5"
+  mv target/release/Parity\ Ethereum-signed.pkg "parity_"$VER"_"$IDENT"_"$ARC".pkg"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT >> "parity_"$VER"_"$IDENT"_"$ARC".pkg.md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT >> "parity_"$VER"_"$IDENT"_"$ARC".pkg.sha256"
+}
+sign_exe () {
+  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
 }
 make_exe () {
-  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
-  SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity.exe -p %h)
-  echo "Checksum calculation:"
-  rm -rf *.md5
-  echo "Parity file SHA3:" $SHA3
-  rhash --md5 target/$PLATFORM/release/parity.exe -p %h > parity.exe.md5
-  rhash --md5 target/$PLATFORM/release/parity-evm.exe -p %h > parity-evm.exe.md5
-  rhash --md5 target/$PLATFORM/release/ethstore.exe -p %h > ethstore.exe.md5
-  rhash --md5 target/$PLATFORM/release/ethkey.exe -p %h > ethkey.exe.md5
  ./msbuild.cmd
  ./sign.cmd $keyfile $certpass windows/ptray/x64/release/ptray.exe
  cd nsis
@@ -143,9 +165,10 @@ make_exe () {
  echo "makensis.exe installer.nsi" > nsis.cmd
  ./nsis.cmd
  cd ..
-  cp nsis/installer.exe "parity_"$VER"_"$ARC"."$EXT
-  ./sign.cmd $keyfile $certpass "parity_"$VER"_"$ARC"."$EXT
-  rhash --md5 "parity_"$VER"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$ARC"."$EXT".md5"
+  cp nsis/installer.exe "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
+  ./sign.cmd $keyfile $certpass "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
 }
 push_binaries () {
  echo "Push binaries to AWS S3"
@@ -160,25 +183,36 @@ push_binaries () {
  aws s3 rm --recursive s3://$S3_BUCKET/$CI_BUILD_REF_NAME/$BUILD_PLATFORM
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity$S3WIN --body target/$PLATFORM/release/parity$S3WIN
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity$S3WIN.md5 --body parity$S3WIN.md5
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity$S3WIN.sha256 --body parity$S3WIN.sha256
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity-evm$S3WIN --body target/$PLATFORM/release/parity-evm$S3WIN
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity-evm$S3WIN.md5 --body parity-evm$S3WIN.md5
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/parity-evm$S3WIN.sha256 --body parity-evm$S3WIN.sha256
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethstore$S3WIN --body target/$PLATFORM/release/ethstore$S3WIN
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethstore$S3WIN.md5 --body ethstore$S3WIN.md5
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethstore$S3WIN.sha256 --body ethstore$S3WIN.sha256
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethkey$S3WIN --body target/$PLATFORM/release/ethkey$S3WIN
  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethkey$S3WIN.md5 --body ethkey$S3WIN.md5
-  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/"parity_"$VER"_"$ARC"."$EXT --body "parity_"$VER"_"$ARC"."$EXT
-  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/"parity_"$VER"_"$ARC"."$EXT".md5" --body "parity_"$VER"_"$ARC"."$EXT".md5"
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/ethkey$S3WIN.sha256 --body ethkey$S3WIN.sha256
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/"parity_"$VER"_"$IDENT"_"$ARC"."$EXT --body "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/"parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5" --body "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
+  aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/$BUILD_PLATFORM/"parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256" --body "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
 }
 make_archive () {
  echo "add artifacts to archive"
  rm -rf parity.zip
-  zip -r parity.zip target/$PLATFORM/release/parity$S3WIN target/$PLATFORM/release/parity-evm$S3WIN target/$PLATFORM/release/ethstore$S3WIN target/$PLATFORM/release/ethkey$S3WIN parity$S3WIN.md5 parity-evm$S3WIN.md5 ethstore$S3WIN.md5 ethkey$S3WIN.md5
+  zip -r parity.zip target/$PLATFORM/release/parity$S3WIN target/$PLATFORM/release/parity-evm$S3WIN target/$PLATFORM/release/ethstore$S3WIN target/$PLATFORM/release/ethkey$S3WIN parity$S3WIN.md5 parity-evm$S3WIN.md5 ethstore$S3WIN.md5 ethkey$S3WIN.md5 parity$S3WIN.sha256 parity-evm$S3WIN.sha256 ethstore$S3WIN.sha256 ethkey$S3WIN.sha256
 }
-push_release () {
+
+updater_push_release () {
  echo "push release"
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$PLATFORM
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$PLATFORM
+
+  DATA="commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity$S3WIN&secret=$RELEASES_SECRET"
+  # Mainnet
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
+  # Kovan
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
 }
+
 case $BUILD_PLATFORM in
  x86_64-unknown-linux-gnu)
    #set strip bin
@@ -186,11 +220,12 @@ case $BUILD_PLATFORM in
    #package extention
    EXT="deb"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
+    updater_push_release
    ;;
  x86_64-unknown-debian-gnu)
    STRIP_BIN="strip"
@@ -198,86 +233,88 @@ case $BUILD_PLATFORM in
    LIBSSL="libssl1.1 (>=1.1.0)"
    echo "Use libssl1.1 (>=1.1.0) for Debian builds"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  x86_64-unknown-centos-gnu)
    STRIP_BIN="strip"
    EXT="rpm"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_rpm
    make_archive
    push_binaries
-    push_release
    ;;
  i686-unknown-linux-gnu)
    STRIP_BIN="strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  armv7-unknown-linux-gnueabihf)
    STRIP_BIN="arm-linux-gnueabihf-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  arm-unknown-linux-gnueabihf)
    STRIP_BIN="arm-linux-gnueabihf-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  aarch64-unknown-linux-gnu)
    STRIP_BIN="aarch64-linux-gnu-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  x86_64-apple-darwin)
    STRIP_BIN="strip"
    PLATFORM="x86_64-apple-darwin"
    EXT="pkg"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_pkg
    make_archive
    push_binaries
-    push_release
+    updater_push_release
    ;;
  x86_64-unknown-snap-gnu)
    ARC="amd64"
    EXT="snap"
+    apt install -y expect zip
    snapcraft clean
    echo "Prepare snapcraft.yaml for build on Gitlab CI in Docker image"
    sed -i 's/git/'"$VER"'/g' snap/snapcraft.yaml
-    if [[ "$CI_BUILD_REF_NAME" = "stable" ]];
+    if [[ "$CI_BUILD_REF_NAME" = "beta" || "$VER" == *1.9* ]];
      then
-        sed -i -e 's/grade: devel/grade: stable/' snap/snapcraft.yaml;
+        sed -i -e 's/grade: devel/grade: beta/' snap/snapcraft.yaml;
    fi
    mv -f snap/snapcraft.yaml snapcraft.yaml
    snapcraft -d
@@ -293,18 +330,21 @@ case $BUILD_PLATFORM in
    snapcraft push "parity_"$VER"_amd64.snap"
    snapcraft status parity
    snapcraft logout
-    md5sum "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
+    $MD5_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
+    $SHA256_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.sha256"
    echo "add artifacts to archive"
    rm -rf parity.zip
-    zip -r parity.zip "parity_"$VER"_amd64.snap" "parity_"$VER"_amd64.snap.md5"
+    zip -r parity.zip "parity_"$VER"_amd64.snap" "parity_"$VER"_amd64.snap.md5" "parity_"$VER"_amd64.snap.sha256"
    ;;
  x86_64-pc-windows-msvc)
    set_env_win
    EXT="exe"
    S3WIN=".exe"
    build
+    sign_exe
+    calculate_checksums
    make_exe
    make_archive
    push_binaries
-    push_release
+    updater_push_release
 esac
--- a/scripts/gitlab-push-release.sh
+++ b/scripts/gitlab-push-release.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+DATA="secret=$RELEASES_SECRET"
+
+echo "Pushing release to Mainnet"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"
+
+echo "Pushing release to Kovan"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"
--- a/scripts/gitlab-test.sh
+++ b/scripts/gitlab-test.sh
@@ -21,11 +21,12 @@ rust_test () {
  rustup show
  if [[ "${RUST_FILES_MODIFIED}" == "0" ]];
    then echo "Skipping Rust tests since no Rust files modified.";
-    else ./test.sh;
-  fi
-  if [[ "$CI_COMMIT_REF_NAME" == "nightly" ]];
-    then sh scripts/aura-test.sh;
+    else ./test.sh || exit $?;
  fi
+  # if [[ "$CI_COMMIT_REF_NAME" == "nightly" ]];
+  #   ### @TODO re-enable fail after https://github.com/paritytech/parity-import-tests/issues/3
+  #   then sh scripts/aura-test.sh; # || exit $?;
+  # fi
 }
 js_test () {
  git submodule update --init --recursive
--- a/scripts/safe_curl.sh
+++ b/scripts/safe_curl.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+
+DATA=$1
+ADDRESS=$2
+
+CODE=$(curl -o out.txt -w '%{http_code}' --data $DATA $ADDRESS)
+cat out.txt && rm out.txt
+echo "\n"
+
+if [[ $CODE -eq 200 ]]; then
+	echo 'Pushed to updater service.';
+elif [[ $CODE -eq 202 ]]; then
+	echo 'Updater service ignored request.';
+else
+	echo 'Unable to push info to updater service.';
+	exit 2
+fi
+
--- a/scripts/snapcraft.yaml
+++ b/scripts/snapcraft.yaml
@@ -1,22 +0,0 @@
-name: parity
-version: master
-summary: Fast, light, robust Ethereum implementation
-description: |
-  Parity's goal is to be the fastest, lightest, and most secure Ethereum
-  client. We are developing Parity using the sophisticated and cutting-edge
-  Rust programming language. Parity is licensed under the GPLv3, and can be
-  used for all your Ethereum needs.
-
-grade: devel
-confinement: strict
-
-apps:
-  parity:
-    command: parity
-    plugs: [network, network-bind]
-
-parts:
-  parity:
-    source: ..
-    plugin: rust
-    build-packages: [g++, libudev-dev, libssl-dev, make, pkg-config]