Fix checksums and auto-update push (#7846)

* Re-enable auto-update for windows. Fail build on non 2xx response from updater. * Fix format of md5 and sha256 files. * Fix identation. * Add critical to metadata. * Add some logs. * Fix pushing build. * Attempt to fix release pushing. * Fix missing rhash? * Workaround broken rhash.
2018-02-13 19:31:06 +01:00 · 2018-02-13 19:31:06 +01:00 · 5584739b83
parent 8b0004b4cf
commit 5584739b83
6 changed files with 110 additions and 60 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -9,7 +9,7 @@ trim_trailing_whitespace=true
 max_line_length=120
 insert_final_newline=true

-[.travis.yml]
+[*.{yml,sh}]
 indent_style=space
 indent_size=2
 tab_width=8
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -239,8 +239,6 @@ push-release:
    - triggers
  image: parity/rust:gitlab-ci
  script:
-    - rustup default stable
-    - curl --data "secret=$RELEASES_SECRET" http://update.parity.io:1337/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF
-    - curl --data "secret=$RELEASES_SECRET" http://update.parity.io:1338/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF
+    - scripts/gitlab-push-release.sh
  tags:
    - curl
--- a/scripts/gitlab-build.sh
+++ b/scripts/gitlab-build.sh
@ -22,6 +22,14 @@ echo "Parity version:     " $VER
 echo "Branch:             " $CI_BUILD_REF_NAME
 echo "--------------------"

+echo "Rhash version:"
+# NOTE for md5 and sha256 we want to display filename as well
+# hence we use --* instead of -p *
+MD5_BIN="rhash --md5"
+SHA256_BIN="rhash --sha256"
+# NOTE For SHA3 we need only hash (hence -p)
+SHA3_BIN="rhash -p %{sha3-256}"
+
 set_env () {
  echo "Set ENVIROMENT"
  export HOST_CC=gcc
@ -52,26 +60,34 @@ build () {
  echo "Build ethkey-cli:"
  cargo build --target $PLATFORM --release -p ethkey-cli
 }
-strip_md5 () {
+strip_binaries () {
  echo "Strip binaries:"
  $STRIP_BIN -v target/$PLATFORM/release/parity
  $STRIP_BIN -v target/$PLATFORM/release/parity-evm
  $STRIP_BIN -v target/$PLATFORM/release/ethstore
  $STRIP_BIN -v target/$PLATFORM/release/ethkey;
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
+}
+calculate_checksums () {
  echo "Checksum calculation:"
+  rhash --version
  rm -rf *.md5
  rm -rf *.sha256
-  export SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity -p %h)
-  echo "Parity file SHA3:" $SHA3
-  md5sum target/$PLATFORM/release/parity > parity.md5
-  sha256sum target/$PLATFORM/release/parity > parity.sha256
-  md5sum target/$PLATFORM/release/parity-evm > parity-evm.md5
-  sha256sum target/$PLATFORM/release/parity-evm > parity-evm.sha256
-  md5sum target/$PLATFORM/release/ethstore > ethstore.md5
-  sha256sum target/$PLATFORM/release/ethstore > ethstore.sha256
-  md5sum target/$PLATFORM/release/ethkey > ethkey.md5
-  sha256sum target/$PLATFORM/release/ethkey > ethkey.sha256
+
+  export SHA3="$($SHA3_BIN target/$PLATFORM/release/parity$S3WIN)"
+  # NOTE rhash 1.3.1 doesnt support keccak, workaround
+  if [ "$SHA3" == "%{sha3-256}" ]; then
+    export SHA3="$(target/$PLATFORM/release/parity$S3WIN tools hash target/$PLATFORM/release/parity$S3WIN)"
+  fi
+
+  echo "Parity file SHA3: $SHA3"
+  $MD5_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/parity$S3WIN > parity$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/parity-evm$S3WIN > parity-evm$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethstore$S3WIN > ethstore$S3WIN.sha256
+  $MD5_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.md5
+  $SHA256_BIN target/$PLATFORM/release/ethkey$S3WIN > ethkey$S3WIN.sha256
 }
 make_deb () {
  rm -rf deb
@ -106,8 +122,8 @@ make_deb () {
  cp target/$PLATFORM/release/ethstore deb/usr/bin/ethstore
  cp target/$PLATFORM/release/ethkey deb/usr/bin/ethkey
  dpkg-deb -b deb "parity_"$VER"_"$IDENT"_"$ARC".deb"
-  md5sum "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.md5"
-  sha256sum "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.sha256"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".deb" > "parity_"$VER"_"$IDENT"_"$ARC".deb.sha256"
 }
 make_rpm () {
  rm -rf /install
@ -116,10 +132,12 @@ make_rpm () {
  cp target/$PLATFORM/release/parity-evm /install/usr/bin/parity-evm
  cp target/$PLATFORM/release/ethstore /install/usr/bin/ethstore
  cp target/$PLATFORM/release/ethkey /install/usr/bin/ethkey
+
+  rm -rf "parity-"$VER"-1."$ARC".rpm" || true
  fpm -s dir -t rpm -n parity -v $VER --epoch 1 --license GPLv3 -d openssl --provides parity --url https://parity.io --vendor "Parity Technologies" -a x86_64 -m "<devops@parity.io>" --description "Ethereum network client by Parity Technologies" -C /install/
  cp "parity-"$VER"-1."$ARC".rpm" "parity_"$VER"_"$IDENT"_"$ARC".rpm"
-  md5sum "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.md5"
-  sha256sum "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.sha256"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC".rpm" > "parity_"$VER"_"$IDENT"_"$ARC".rpm.sha256"
 }
 make_pkg () {
  echo "make PKG"
@ -133,24 +151,13 @@ make_pkg () {
  packagesbuild -v mac/Parity.pkgproj
  productsign --sign 'Developer ID Installer: PARITY TECHNOLOGIES LIMITED (P2PX3JU8FT)' target/release/Parity\ Ethereum.pkg target/release/Parity\ Ethereum-signed.pkg
  mv target/release/Parity\ Ethereum-signed.pkg "parity_"$VER"_"$ARC".pkg"
-  md5sum "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.md5"
-  sha256sum "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.sha256"
+  $MD5_BIN "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.md5"
+  $SHA256_BIN "parity_"$VER"_"$ARC"."$EXT >> "parity_"$VER"_"$ARC".pkg.sha256"
+}
+sign_exe () {
+  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
 }
 make_exe () {
-  ./sign.cmd $keyfile $certpass "target/$PLATFORM/release/parity.exe"
-  SHA3=$(rhash --sha3-256 target/$PLATFORM/release/parity.exe -p %h)
-  echo "Checksum calculation:"
-  rm -rf *.md5
-  rm -rf *.sha256
-  echo "Parity file SHA3:" $SHA3
-  rhash --md5 target/$PLATFORM/release/parity.exe -p %h > parity.exe.md5
-  rhash --sha256 target/$PLATFORM/release/parity.exe -p %h > parity.exe.sha256
-  rhash --md5 target/$PLATFORM/release/parity-evm.exe -p %h > parity-evm.exe.md5
-  rhash --sha256 target/$PLATFORM/release/parity-evm.exe -p %h > parity-evm.exe.sha256
-  rhash --md5 target/$PLATFORM/release/ethstore.exe -p %h > ethstore.exe.md5
-  rhash --sha256 target/$PLATFORM/release/ethstore.exe -p %h > ethstore.exe.sha256
-  rhash --md5 target/$PLATFORM/release/ethkey.exe -p %h > ethkey.exe.md5
-  rhash --sha256 target/$PLATFORM/release/ethkey.exe -p %h > ethkey.exe.sha256
  ./msbuild.cmd
  ./sign.cmd $keyfile $certpass windows/ptray/x64/release/ptray.exe
  cd nsis
@ -160,8 +167,8 @@ make_exe () {
  cd ..
  cp nsis/installer.exe "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
  ./sign.cmd $keyfile $certpass "parity_"$VER"_"$IDENT"_"$ARC"."$EXT
-  rhash --md5 "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
-  rhash --sha256 "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
+  $MD5_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".md5"
+  $SHA256_BIN "parity_"$VER"_"$IDENT"_"$ARC"."$EXT -p %h > "parity_"$VER"_"$IDENT"_"$ARC"."$EXT".sha256"
 }
 push_binaries () {
  echo "Push binaries to AWS S3"
@ -195,11 +202,17 @@ make_archive () {
  rm -rf parity.zip
  zip -r parity.zip target/$PLATFORM/release/parity$S3WIN target/$PLATFORM/release/parity-evm$S3WIN target/$PLATFORM/release/ethstore$S3WIN target/$PLATFORM/release/ethkey$S3WIN parity$S3WIN.md5 parity-evm$S3WIN.md5 ethstore$S3WIN.md5 ethkey$S3WIN.md5 parity$S3WIN.sha256 parity-evm$S3WIN.sha256 ethstore$S3WIN.sha256 ethkey$S3WIN.sha256
 }
-push_release () {
+
+updater_push_release () {
  echo "push release"
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$PLATFORM
-  curl --data "commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity&secret=$RELEASES_SECRET" http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$PLATFORM
+
+  DATA="commit=$CI_BUILD_REF&sha3=$SHA3&filename=parity$S3WIN&secret=$RELEASES_SECRET"
+  # Mainnet
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
+  # Kovan
+  source scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$BUILD_PLATFORM"
 }
+
 case $BUILD_PLATFORM in
  x86_64-unknown-linux-gnu)
    #set strip bin
@ -207,11 +220,12 @@ case $BUILD_PLATFORM in
    #package extention
    EXT="deb"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
+    updater_push_release
    ;;
  x86_64-unknown-debian-gnu)
    STRIP_BIN="strip"
@ -219,76 +233,77 @@ case $BUILD_PLATFORM in
    LIBSSL="libssl1.1 (>=1.1.0)"
    echo "Use libssl1.1 (>=1.1.0) for Debian builds"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  x86_64-unknown-centos-gnu)
    STRIP_BIN="strip"
    EXT="rpm"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_rpm
    make_archive
    push_binaries
-    push_release
    ;;
  i686-unknown-linux-gnu)
    STRIP_BIN="strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  armv7-unknown-linux-gnueabihf)
    STRIP_BIN="arm-linux-gnueabihf-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  arm-unknown-linux-gnueabihf)
    STRIP_BIN="arm-linux-gnueabihf-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  aarch64-unknown-linux-gnu)
    STRIP_BIN="aarch64-linux-gnu-strip"
    EXT="deb"
    set_env
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_deb
    make_archive
    push_binaries
-    push_release
    ;;
  x86_64-apple-darwin)
    STRIP_BIN="strip"
    PLATFORM="x86_64-apple-darwin"
    EXT="pkg"
    build
-    strip_md5
+    strip_binaries
+    calculate_checksums
    make_pkg
    make_archive
    push_binaries
-    push_release
+    updater_push_release
    ;;
  x86_64-unknown-snap-gnu)
    cd snap
@ -298,8 +313,8 @@ case $BUILD_PLATFORM in
    sed -i 's/master/'"$VER"'/g' snapcraft.yaml
    snapcraft
    cp "parity_"$CI_BUILD_REF_NAME"_amd64.snap" "parity_"$VER"_amd64.snap"
-    md5sum "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
-    sha256sum "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.sha256"
+    $MD5_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.md5"
+    $SHA256_BIN "parity_"$VER"_amd64.snap" > "parity_"$VER"_amd64.snap.sha256"
    push_binaries
    ;;
  x86_64-pc-windows-msvc)
@ -307,8 +322,10 @@ case $BUILD_PLATFORM in
    EXT="exe"
    S3WIN=".exe"
    build
+    sign_exe
+    calculate_checksums
    make_exe
    make_archive
    push_binaries
-    push_release
+    updater_push_release
 esac
--- a/scripts/gitlab-push-release.sh
+++ b/scripts/gitlab-push-release.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+DATA="secret=$RELEASES_SECRET"
+
+echo "Pushing release to Mainnet"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1337/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"
+
+echo "Pushing release to Kovan"
+./scripts/safe_curl.sh $DATA "http://update.parity.io:1338/push-release/$CI_BUILD_REF_NAME/$CI_BUILD_REF"
--- a/scripts/safe_curl.sh
+++ b/scripts/safe_curl.sh
@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+
+DATA=$1
+ADDRESS=$2
+
+CODE=$(curl -o out.txt -w '%{http_code}' --data $DATA $ADDRESS)
+cat out.txt && rm out.txt
+echo "\n"
+
+if [[ $CODE -eq 200 ]]; then
+	echo 'Pushed to updater service.';
+elif [[ $CODE -eq 202 ]]; then
+	echo 'Updater service ignored request.';
+else
+	echo 'Unable to push info to updater service.';
+	exit 2
+fi
+
--- a/util/version/Cargo.toml
+++ b/util/version/Cargo.toml
@ -12,6 +12,9 @@ build = "build.rs"
 # Used by auto-updater and for Parity version string.
 track = "nightly"

+# Indicates a critical release in this track (i.e. consensus issue)
+critical = false
+
 # Latest supported fork blocks for various networks. Used ONLY by auto-updater.
 [package.metadata.forks]
 foundation = 4370000