Fix checksums and auto-update push (#7846)

* Re-enable auto-update for windows. Fail build on non 2xx response from updater.

* Fix format of md5 and sha256 files.

* Fix identation.

* Add critical to metadata.

* Add some logs.

* Fix pushing build.

* Attempt to fix release pushing.

* Fix missing rhash?

* Workaround broken rhash.

.editorconfig

@@ -9,7 +9,7 @@ trim_trailing_whitespace=true
 max_line_length=120
 insert_final_newline=true
 
-[.travis.yml]
+[*.{yml,sh}]
 indent_style=space
 indent_size=2
 tab_width=8

.gitlab-ci.yml

@@ -239,8 +239,6 @@ push-release:
     - triggers
   image: parity/rust:gitlab-ci
   script:
-    - rustup default stable
+    - scripts/gitlab-push-release.sh
-    - curl --data "secret=$RELEASES_SECRET" http://update.parity.io:1337/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF
-    - curl --data "secret=$RELEASES_SECRET" http://update.parity.io:1338/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF
   tags:
     - curl

scripts/gitlab-build.sh

@@ -22,6 +22,14 @@ echo "Parity version:     " $VER
 echo "Branch:             " $CI_BUILD_REF_NAME
 echo "--------------------"
 
+echo "Rhash version:"
+# NOTE for md5 and sha256 we want to display filename as well
+# hence we use --* instead of -p *
+MD5_BIN="rhash --md5"
+SHA256_BIN="rhash --sha256"
+# NOTE For SHA3 we need only hash (hence -p)
+SHA3_BIN="rhash -p %{sha3-256}"
+
 set_env () {
   echo "Set ENVIROMENT"
   export HOST_CC=gcc
@@ -52,26 +60,34 @@ build () {
   echo "Build ethkey-cli:"
   cargo build --target $PLATFORM --release -p ethkey-cli
 }
-strip_md5 () {
+strip_binaries () {
   echo "Strip binaries:"
   $STRIP_BIN -v target/$PLATFORM/release/parity
   $STRIP_BIN -v target/$PLATFORM/release/parity-evm
   $STRIP_BIN -v target/$PLATFORM/release/ethstore
   $STRIP_BIN -v target/$PLATFORM/release/ethkey;
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
+}
+calculate_checksums () {
   echo "Checksum calculation:"
+  rhash --version
   rm -rf *.md5
   rm -rf *.sha256
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
+
-  echo "Parity file SHA3:" $SHA3
+  export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
-  md5sum target/$PLATFORM/release/parity > parity.md5
+  # NOTE rhash 1.3.1 doesnt support keccak, workaround
-  sha256sum target/$PLATFORM/release/parity > parity.sha256
+  if [ "$SHA3" == "%{sha3-256}" ]; then
-  md5sum target/$PLATFORM/release/parity-evm > parity-evm.md5
+    export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
-  sha256sum target/$PLATFORM/release/parity-evm > parity-evm.sha256
+  fi
-  md5sum target/$PLATFORM/release/ethstore > ethstore.md5
+
-  sha256sum target/$PLATFORM/release/ethstore > ethstore.sha256
+  echo "Parity file SHA3: $SHA3"
-  md5sum target/$PLATFORM/release/ethkey > ethkey.md5
+  $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5
-  sha256sum target/$PLATFORM/release/ethkey > ethkey.sha256
+  $SHA256_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.sha256
 }
 make_deb () {
   rm -rf deb
@@ -106,8 +122,8 @@ make_deb () {
   cp target/$PLATFORM/release/ethstore deb/usr/bin/ethstore
   cp target/$PLATFORM/release/ethkey deb/usr/bin/ethkey
   dpkg-deb -b deb "parity_"$VER"_"$IDENT"_"$ARC".deb"
-  md5sum "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.md5"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.md5"
-  sha256sum "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.sha256"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.sha256"
 }
 make_rpm () {
   rm -rf /install
@@ -116,10 +132,12 @@ make_rpm () {
   cp target/$PLATFORM/release/parity-evm /install/usr/bin/parity-evm
   cp target/$PLATFORM/release/ethstore /install/usr/bin/ethstore
   cp target/$PLATFORM/release/ethkey /install/usr/bin/ethkey
+
+  rm -rf "parity-"$VER"-1."$ARC".rpm" || true
   fpm -s dir -t rpm -n parity -v $VER --epoch 1 --license GPLv3 -d openssl --provides parity --url https://parity.io --vendor "Parity Technologies" -a x86_64 -m "<devops@parity.io>" --description "Ethereum network client by Parity Technologies" -C /install/
   cp "parity-"$VER"-1."$ARC".rpm" "parity_"$VER"_"$IDENT"_"$ARC".rpm"
-  md5sum "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.md5"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.md5"
-  sha256sum "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.sha256"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.sha256"
 }
 make_pkg () {
   echo "make PKG"
@@ -133,24 +151,13 @@ make_pkg () {
   packagesbuild -v mac/Parity.pkgproj
   productsign --sign 'Developer ID Installer: PARITY TECHNOLOGIES LIMITED (P2PX3JU8FT)' target/release/Parity\ Ethereum.pkg target/release/Parity\ Ethereum-signed.pkg
   mv target/release/Parity\ Ethereum-signed.pkg "parity_"$VER"_"$ARC".pkg"
-  md5sum "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.md5"
+  $MD5_BIN "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.md5"
-  sha256sum "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.sha256"
+  $SHA256_BIN "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.sha256"
+}
+sign_exe () {
+  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
 }
 make_exe () {
-  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
-  SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity.exe -p %h)
-  echo "Checksum calculation:"
-  rm -rf *.md5
-  rm -rf *.sha256
-  echo "Parity file SHA3:" $SHA3
-  rhash --md5 target/$PLATFORM/release/parity.exe -p %h > parity.exe.md5
-  rhash --sha256 target/$PLATFORM/release/parity.exe -p %h > parity.exe.sha256
-  rhash --md5 target/$PLATFORM/release/parity-evm.exe -p %h > parity-evm.exe.md5
-  rhash --sha256 target/$PLATFORM/release/parity-evm.exe -p %h > parity-evm.exe.sha256
-  rhash --md5 target/$PLATFORM/release/ethstore.exe -p %h > ethstore.exe.md5
-  rhash --sha256 target/$PLATFORM/release/ethstore.exe -p %h > ethstore.exe.sha256
-  rhash --md5 target/$PLATFORM/release/ethkey.exe -p %h > ethkey.exe.md5
-  rhash --sha256 target/$PLATFORM/release/ethkey.exe -p %h > ethkey.exe.sha256
   ./msbuild.cmd
   ./sign.cmd $keyfile $certpass windows/ptray/x64/release/ptray.exe
   cd nsis
@@ -160,8 +167,8 @@ make_exe () {
   cd ..
   cp nsis/installer.exe "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
   ./sign.cmd $keyfile $certpass "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
-  rhash --md5 "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
-  rhash --sha256 "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
 }
 push_binaries () {
   echo "Push binaries to AWS S3"
@@ -195,11 +202,17 @@ make_archive () {
   rm -rf parity.zip
   zip -r parity.zip target/$PLATFORM/release/parity$S3WIN target/$PLATFORM/release/parity-evm$S3WIN target/$PLATFORM/release/ethstore$S3WIN target/$PLATFORM/release/ethkey$S3WIN parity$S3WIN.md5 parity-evm$S3WIN.md5 ethstore$S3WIN.md5 ethkey$S3WIN.md5 parity$S3WIN.sha256 parity-evm$S3WIN.sha256 ethstore$S3WIN.sha256 ethkey$S3WIN.sha256
 }
-push_release () {
+
+updater_push_release () {
   echo "push release"
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$PLATFORM
+
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$PLATFORM
+  DATA="commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity$S3WIN&secret=$RELEASES_SECRET"
+  # Mainnet
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
+  # Kovan
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
 }
+
 case $BUILD_PLATFORM in
   x86_64-unknown-linux-gnu)
     #set strip bin
@@ -207,11 +220,12 @@ case $BUILD_PLATFORM in
     #package extention
     EXT="deb"
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
+    updater_push_release
     ;;
   x86_64-unknown-debian-gnu)
     STRIP_BIN="strip"
@@ -219,76 +233,77 @@ case $BUILD_PLATFORM in
     LIBSSL="libssl1.1 (>=1.1.0)"
     echo "Use libssl1.1 (>=1.1.0) for Debian builds"
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
     ;;
   x86_64-unknown-centos-gnu)
     STRIP_BIN="strip"
     EXT="rpm"
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_rpm
     make_archive
     push_binaries
-    push_release
     ;;
   i686-unknown-linux-gnu)
     STRIP_BIN="strip"
     EXT="deb"
     set_env
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
     ;;
   armv7-unknown-linux-gnueabihf)
     STRIP_BIN="arm-linux-gnueabihf-strip"
     EXT="deb"
     set_env
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
     ;;
   arm-unknown-linux-gnueabihf)
     STRIP_BIN="arm-linux-gnueabihf-strip"
     EXT="deb"
     set_env
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
     ;;
   aarch64-unknown-linux-gnu)
     STRIP_BIN="aarch64-linux-gnu-strip"
     EXT="deb"
     set_env
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_deb
     make_archive
     push_binaries
-    push_release
     ;;
   x86_64-apple-darwin)
     STRIP_BIN="strip"
     PLATFORM="x86_64-apple-darwin"
     EXT="pkg"
     build
-    strip_md5
+    strip_binaries
+    calculate_checksums
     make_pkg
     make_archive
     push_binaries
-    push_release
+    updater_push_release
     ;;
   x86_64-unknown-snap-gnu)
     cd snap
@@ -298,8 +313,8 @@ case $BUILD_PLATFORM in
     sed -i 's/master/'"$VER"'/g' snapcraft.yaml
     snapcraft
     cp "parity_"$CI_BUILD_REF_NAME"_amd64.snap" "parity_"$VER"_amd64.snap"
-    md5sum "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
+    $MD5_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
-    sha256sum "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.sha256"
+    $SHA256_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.sha256"
     push_binaries
     ;;
   x86_64-pc-windows-msvc)
@@ -307,8 +322,10 @@ case $BUILD_PLATFORM in
     EXT="exe"
     S3WIN=".exe"
     build
+    sign_exe
+    calculate_checksums
     make_exe
     make_archive
     push_binaries
-    push_release
+    updater_push_release
 esac

scripts/gitlab-push-release.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+DATA="secret=$RELEASES_SECRET"
+
+echo "Pushing release to Mainnet"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"
+
+echo "Pushing release to Kovan"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"

scripts/safe_curl.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+
+DATA=$1
+ADDRESS=$2
+
+CODE=$(curl -o out.txt -w '%{http_code}' --data $DATA $ADDRESS)
+cat out.txt && rm out.txt
+echo "\n"
+
+if [[ $CODE -eq 200 ]]; then
+	echo 'Pushed to updater service.';
+elif [[ $CODE -eq 202 ]]; then
+	echo 'Updater service ignored request.';
+else
+	echo 'Unable to push info to updater service.';
+	exit 2
+fi
+

util/version/Cargo.toml

@@ -12,6 +12,9 @@ build = "build.rs"
 # Used by auto-updater and for Parity version string.
 track = "nightly"
 
+# Indicates a critical release in this track (i.e. consensus issue)
+critical = false
+
 # Latest supported fork blocks for various networks. Used ONLY by auto-updater.
 [package.metadata.forks]
 foundation = 4370000