Updating the CI system (#8765)

* Updating the CI system with the publication of releases and binary files on github Signed-off-by: Denis S. Soldatov aka General-Beck <general.beck@gmail.com> * add missed scripts * chmod +x scripts * fix download link for github * rebuilding CI scripts * small fixes * update submodule wasm tests * ci: fix merge leftovers * ci: remove gitlab-next from ci triggers * ci: fix git add in docs script * ci: use nightly instead of master for publish triggers * ci: remove sleep from gitlab config * ci: replace ':' with '-' in gitlab targets * ci: fix recursive copy in docs script
2018-08-26 02:44:08 +04:00
parent 9ed43230ca
commit bd3bc5c168
25 changed files with 686 additions and 644 deletions
--- a/scripts/gitlab/build-unix.sh
+++ b/scripts/gitlab/build-unix.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+echo "__________Show ENVIROMENT__________"
+echo "CC:       " $CC
+echo "CXX:      " $CXX
+
+echo "__________CARGO CONFIG__________"
+rm -rf .cargo
+mkdir -p .cargo
+echo "[target.$CARGO_TARGET]" >> .cargo/config
+echo "linker= \"$CC\"" >> .cargo/config
+cat .cargo/config
+
+echo "_____ Building target: "$CARGO_TARGET" _____"
+time cargo build --target $CARGO_TARGET --release --features final
+time cargo build --target $CARGO_TARGET --release -p evmbin
+time cargo build --target $CARGO_TARGET --release -p ethstore-cli
+time cargo build --target $CARGO_TARGET --release -p ethkey-cli
+time cargo build --target $CARGO_TARGET --release -p whisper-cli
+
+echo "_____ Post-processing binaries _____"
+rm -rf artifacts
+mkdir -p artifacts
+cd artifacts
+mkdir -p $CARGO_TARGET
+cd $CARGO_TARGET
+cp ../../target/$CARGO_TARGET/release/{parity,parity-evm,ethstore,ethkey,whisper} .
+strip -v ./*
+echo "_____ Calculating checksums _____"
+for binary in $(ls)
+do
+  rhash --sha256 $binary -o $binary.sha256
+done
--- a/scripts/gitlab/build-windows.sh
+++ b/scripts/gitlab/build-windows.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e # fail on any error
+set -u # treat unset variables as error
+
+set INCLUDE="C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Include;C:\vs2015\VC\include;C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt"
+set LIB="C:\vs2015\VC\lib;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64"
+
+rustup default stable-x86_64-pc-windows-msvc
+echo "_____ Building _____"
+time cargo build --target $CARGO_TARGET --release --features final
+time cargo build --target $CARGO_TARGET --release -p evmbin
+time cargo build --target $CARGO_TARGET --release -p ethstore-cli
+time cargo build --target $CARGO_TARGET --release -p ethkey-cli
+time cargo build --target $CARGO_TARGET --release -p whisper-cli
+echo "__________Sign binaries__________"
+scripts/gitlab/sign.cmd $keyfile $certpass target/$CARGO_TARGET/release/parity.exe
+scripts/gitlab/sign.cmd $keyfile $certpass target/$CARGO_TARGET/release/parity-evm.exe
+scripts/gitlab/sign.cmd $keyfile $certpass target/$CARGO_TARGET/release/ethstore.exe
+scripts/gitlab/sign.cmd $keyfile $certpass target/$CARGO_TARGET/release/ethkey.exe
+scripts/gitlab/sign.cmd $keyfile $certpass target/$CARGO_TARGET/release/whisper.exe
+
+echo "_____ Post-processing binaries _____"
+rm -rf artifacts
+mkdir -p artifacts
+cd artifacts
+mkdir -p $CARGO_TARGET
+cd $CARGO_TARGET
+cp --verbose ../../target/$CARGO_TARGET/release/{parity.exe,parity-evm.exe,ethstore.exe,ethkey.exe,whisper.exe} .
+
+echo "_____ Calculating checksums _____"
+for binary in $(ls)
+do
+  rhash --sha256 $binary -o $binary.sha256
+done
+cp parity.exe.sha256 parity.sha256
--- a/scripts/gitlab/clippy.sh
+++ b/scripts/gitlab/clippy.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+cargo install clippy
+cargo clippy -- -D warnings
--- a/scripts/gitlab/coverage.sh
+++ b/scripts/gitlab/coverage.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -x
+git submodule update --init --recursive
+rm -rf target/*
+cargo test --all --exclude evmjit --no-run -- --test-threads 8|| exit $?
+KCOV_TARGET="target/cov"
+KCOV_FLAGS="--verify"
+EXCLUDE="/usr/lib,/usr/include,$HOME/.cargo,$HOME/.multirust,rocksdb,secp256k1"
+mkdir -p $KCOV_TARGET
+echo "__________Cover RUST___________"
+for FILE in `find target/debug/deps ! -name "*.*"`
+  do
+   timeout --signal=SIGKILL 5m kcov --exclude-pattern $EXCLUDE $KCOV_FLAGS $KCOV_TARGET $FILE
+  done
+timeout --signal=SIGKILL 5m kcov --exclude-pattern $EXCLUDE $KCOV_FLAGS $KCOV_TARGET target/debug/parity-*
+echo "Cover JS"
+bash <(curl -s https://codecov.io/bash)&&
+echo "Uploaded code coverage"
+exit 0
--- a/scripts/gitlab/install-readme.sh
+++ b/scripts/gitlab/install-readme.sh
@@ -0,0 +1,8 @@
+echo "Parity Wallet
+=============
+
+Welcome to Parity Wallet, your all-in-one Ethereum node and wallet.
+
+If you continue, Parity will be installed as a user service. You will be able to use the Parity Wallet through your browser by using the menu bar icon, following the shortcut in the Launchpad or navigating to http://localhost:8180/ in your browser.
+
+Parity is distributed under the terms of the GPL."
--- a/scripts/gitlab/package-snap.sh
+++ b/scripts/gitlab/package-snap.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+case ${CI_COMMIT_REF_NAME} in
+  nightly|*v2.1*) export GRADE="devel";;
+  beta|*v2.0*) export GRADE="stable";;
+  stable|*v1.11*) export GRADE="stable";;
+  *) echo "No release" exit 0;;
+esac
+SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
+echo "__________Create snap package__________"
+echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
+snapcraft clean
+echo $VERSION:$GRADE:$BUILD_ARCH
+cat scripts/gitlab/templates/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml
+cat snapcraft.yaml
+snapcraft --target-arch=$BUILD_ARCH
+ls *.snap
+echo "__________Post-processing snap package__________"
+mkdir -p artifacts
+mv -v $SNAP_PACKAGE "artifacts/"$SNAP_PACKAGE
+echo "_____ Calculating checksums _____"
+cd artifacts
+rhash --sha256 $SNAP_PACKAGE -o $SNAP_PACKAGE".sha256"
--- a/scripts/gitlab/publish-docker.sh
+++ b/scripts/gitlab/publish-docker.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+##ARGUMENTS: 1. Docker target
+set -e # fail on any error
+set -u # treat unset variables as error
+
+if [ "$CI_COMMIT_REF_NAME" == "beta" ];
+then export DOCKER_BUILD_TAG="latest";
+else export DOCKER_BUILD_TAG=$CI_COMMIT_REF_NAME;
+fi
+docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity
+
+echo "__________Docker TAG__________"
+echo $DOCKER_BUILD_TAG
+
+echo "__________Docker target__________"
+export DOCKER_TARGET=$1
+echo $DOCKER_TARGET
+
+echo "__________Docker build and push__________"
+docker build --build-arg TARGET=$DOCKER_TARGET --no-cache=true --tag parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG -f docker/hub/Dockerfile .
+docker push parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG
+docker logout
--- a/scripts/gitlab/publish-snap.sh
+++ b/scripts/gitlab/publish-snap.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+case ${CI_COMMIT_REF_NAME} in
+  nightly|*v2.1*) export CHANNEL="edge";;
+  beta|*v2.0*) export CHANNEL="beta";;
+  stable|*v1.11*) export CHANNEL="stable";;
+  *) echo "No release" exit 0;;
+esac
+echo "Release channel :" $CHANNEL " Branch/tag: " $CI_COMMIT_REF_NAME
+
+echo $SNAPCRAFT_LOGIN_PARITY_BASE64 | base64 --decode > snapcraft.login
+snapcraft login --with snapcraft.login
+snapcraft push --release $CHANNEL "artifacts/parity_"$VERSION"_"$BUILD_ARCH".snap"
+snapcraft status parity
+snapcraft logout
--- a/scripts/gitlab/push.sh
+++ b/scripts/gitlab/push.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+updater_push_release () {
+  echo "push release"
+  # Mainnet
+
+}
+echo "__________Set ENVIROMENT__________"
+DESCRIPTION="$(cat CHANGELOG.md)"
+RELEASE_TABLE="$(cat scripts/gitlab/templates/release-table.md)"
+RELEASE_TABLE="$(echo "${RELEASE_TABLE//\$VERSION/${VERSION}}")"
+#The text in the file CANGELOG.md before which the table with links is inserted. Must be present in this file necessarily
+REPLACE_TEXT="The full list of included changes:"
+case ${CI_COMMIT_REF_NAME} in
+  nightly|*v2.1*) NAME="Parity "$VERSION" nightly";;
+  beta|*v2.0*) NAME="Parity "$VERSION" beta";;
+  stable|*v1.11*) NAME="Parity "$VERSION" stable";;
+  *) echo "No release" exit 0;;
+esac
+cd artifacts
+ls -l | sort -k9
+filetest=( * )
+echo ${filetest[*]}
+for DIR in "${filetest[@]}";
+do
+  cd $DIR
+  if [[ $DIR == "*windows*" ]];
+    then
+      WIN=".exe";
+    else
+      WIN="";
+  fi
+  for binary in $(ls parity.sha256)
+  do
+    sha256=$(cat $binary | awk '{ print $1}' )
+    case $DIR in
+      x86_64* )
+        DATA="commit=$CI_BUILD_REF&sha3=$sha256&filename=parity$WIN&secret=$RELEASES_SECRET"
+        ../../scripts/gitlab/safe_curl.sh $DATA "http://update.parity.io:1337/push-build/$CI_BUILD_REF_NAME/$DIR"
+        # Kovan
+        ../../scripts/gitlab/safe_curl.sh $DATA "http://update.parity.io:1338/push-build/$CI_BUILD_REF_NAME/$DIR"
+        ;;
+    esac
+    RELEASE_TABLE="$(echo "${RELEASE_TABLE/sha$DIR/${sha256}}")"
+  done
+  cd ..
+done
+#do not touch the following 3 lines. Features of output in Markdown
+DESCRIPTION="$(echo "${DESCRIPTION/${REPLACE_TEXT}/${RELEASE_TABLE}
+
+${REPLACE_TEXT}}")"
+echo "$DESCRIPTION"
+if [["$CI_COMMIT_REF_NAME" == "nightly" ]]; then DESCRIPTION=""; fi #TODO in the future, we need to prepare a script that will do changelog
+echo "__________Create release to Github____________"
+github-release release --user devops-parity --repo parity-ethereum --tag "$CI_COMMIT_REF_NAME" --draft --name "$NAME" --description "$DESCRIPTION"
+echo "__________Push binaries to AWS S3____________"
+aws configure set aws_access_key_id $s3_key
+aws configure set aws_secret_access_key $s3_secret
+if [[ "$CI_BUILD_REF_NAME" = "beta" || "$CI_BUILD_REF_NAME" = "stable" || "$CI_BUILD_REF_NAME" = "nightly" ]];
+  then
+    export S3_BUCKET=builds-parity-published;
+  else
+    export S3_BUCKET=builds-parity;
+fi
+aws s3 sync ./ s3://$S3_BUCKET/$CI_BUILD_REF_NAME/
--- a/scripts/gitlab/rpc-docs.sh
+++ b/scripts/gitlab/rpc-docs.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+clone_repos() {
+    echo "__________Clone repos__________"
+    git clone https://github.com/parity-js/jsonrpc.git jsonrpc
+    git clone https://github.com/paritytech/wiki.git wiki
+}
+
+build_docs() {
+    echo "__________Build docs__________"
+    npm install
+    npm run build:markdown
+}
+
+update_wiki_docs() {
+    echo "__________Update WIKI docs__________"
+    for file in $(ls jsonrpc/docs); do
+        module_name=${file:0:-3}
+        mv jsonrpc/docs/$file wiki/JSONRPC-$module_name-module.md
+    done
+}
+
+setup_git() {
+    echo "__________Set github__________"
+    git config user.email "devops@parity.com"
+    git config user.name "Devops Parity"
+}
+
+commit_files() {
+    echo "__________Commit files__________"
+    git checkout -b rpcdoc-update-${CI_COMMIT_REF_NAME}
+    git add .
+    git commit -m "Update docs to ${CI_COMMIT_REF_NAME}"
+    git tag -a "${CI_COMMIT_REF_NAME}"
+}
+
+upload_files() {
+    echo "__________Upload files__________"
+    git push --tags
+}
+
+setup_git
+clone_repos
+cp -r parity/ jsonrpc/.parity/
+cd jsonrpc
+build_docs
+cd ..
+update_wiki_docs
+cd wiki
+commit_files
+upload_files
--- a/scripts/gitlab/rustfmt.sh
+++ b/scripts/gitlab/rustfmt.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+cargo install rustfmt-nightly
+cargo fmt -- --write-mode=diff
--- a/scripts/gitlab/safe_curl.sh
+++ b/scripts/gitlab/safe_curl.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -eu
+
+DATA=$1
+ADDRESS=$2
+
+CODE=$(curl -o out.txt -w '%{http_code}' --data $DATA $ADDRESS)
+cat out.txt && rm out.txt
+echo "\n"
+
+if [[ $CODE -eq 200 ]]; then
+	echo 'Pushed to updater service.';
+elif [[ $CODE -eq 202 ]]; then
+	echo 'Updater service ignored request.';
+else
+	echo 'Unable to push info to updater service.';
+	exit 2
+fi
--- a/scripts/gitlab/sign.cmd
+++ b/scripts/gitlab/sign.cmd
@@ -0,0 +1 @@
+@signtool sign /f %1 /p %2 /tr http://timestamp.comodoca.com /du https://parity.io %3
--- a/scripts/gitlab/templates/release-table.md
+++ b/scripts/gitlab/templates/release-table.md
@@ -0,0 +1,16 @@
+| OS | Arch | Download | SHA256 Checksum |
+|:---:|:---:|:---|:---|
+| linux | arm64 | [parity](https://releases.parity.io/$VERSION/aarch64-unknown-linux-gnu/parity) | <sup>`shaaarch64-unknown-linux-gnu`</sup> |
+| android | armv7 | [parity](https://releases.parity.io/$VERSION/armv7-linux-androideabi/parity) | <sup>`shaarmv7-linux-androideabi`</sup> |
+| linux | armv7 | [parity](https://releases.parity.io/$VERSION/armv7-unknown-linux-gnueabihf/parity) | <sup>`shaarmv7-unknown-linux-gnueabihf`</sup> |
+| linux | i686 | [parity](https://releases.parity.io/$VERSION/i686-unknown-linux-gnu/parity) | <sup>`shai686-unknown-linux-gnu`</sup> |
+| osx | x64 | [parity](https://releases.parity.io/$VERSION/x86_64-apple-darwin/parity) | <sup>`shax86_64-apple-darwin`</sup> |
+| windows | x64 | [parity.exe](https://releases.parity.io/$VERSION/x86_64-pc-windows-msvc/parity.exe) | <sup>`shax86_64-pc-windows-msvc`</sup> |
+| linux | x64 | [parity](https://releases.parity.io/$VERSION/x86_64-unknown-linux-gnu/parity) | <sup>`shax86_64-unknown-linux-gnu`</sup> |
+
+| OS | Alternative | Link |
+|:---:|:---:|:---|
+| <img src="https://gist.github.com/5chdn/1fce888fde1d773761f809b607757f76/raw/44c4f0fc63f1ea8e61a9513af5131ef65eaa6c75/apple.png" alt="Apple Icon by Pixel Perfect from https://www.flaticon.com/authors/pixel-perfect" style="width: 32px;"/> | Homebrew |[github.com/paritytech/homebrew-paritytech/blob/master/README.md](https://github.com/paritytech/homebrew-paritytech/blob/master/README.md) |
+| <img src="https://gist.github.com/5chdn/1fce888fde1d773761f809b607757f76/raw/44c4f0fc63f1ea8e61a9513af5131ef65eaa6c75/linux.png" alt="Linux Icon by Pixel Perfect from https://www.flaticon.com/authors/pixel-perfect" style="width: 32px;"/> | Snapcraft | [snapcraft.io/parity](https://snapcraft.io/parity/) |
+| <img src="https://gist.github.com/5chdn/1fce888fde1d773761f809b607757f76/raw/44c4f0fc63f1ea8e61a9513af5131ef65eaa6c75/settings.png" alt="Settings Icon by Pixel Perfect from https://www.flaticon.com/authors/pixel-perfect" style="width: 32px;"/> | Docker | [hub.docker.com/r/parity/parity](https://hub.docker.com/r/parity/parity) |
+| <img src="https://gist.github.com/5chdn/1fce888fde1d773761f809b607757f76/raw/44c4f0fc63f1ea8e61a9513af5131ef65eaa6c75/settings.png" alt="Settings Icon by Pixel Perfect from https://www.flaticon.com/authors/pixel-perfect" style="width: 32px;"/> | Other binaries | [vanity-service.parity.io/parity-binaries?format=markdown&version=$VERSION](https://vanity-service.parity.io/parity-binaries?format=markdown&version=$VERSION) |
--- a/scripts/gitlab/templates/snapcraft.template.yaml
+++ b/scripts/gitlab/templates/snapcraft.template.yaml
@@ -0,0 +1,58 @@
+name:               parity
+version:            $VERSION
+architectures:      [$BUILD_ARCH]
+grade:              $GRADE
+confinement:        strict
+
+summary:            Fast, light, robust Ethereum implementation
+description:        |
+  Parity's goal is to be the fastest, lightest, and most secure Ethereum
+  client. We are developing Parity using the sophisticated and cutting-edge
+  Rust programming language. Parity is licensed under the GPLv3, and can be
+  used for all your Ethereum needs.
+
+
+apps:
+  parity:
+    command:        parity
+    plugs:          [home, network, network-bind, mount-observe, x11, unity7, desktop, desktop-legacy, wayland]
+    desktop:        usr/share/applications/parity.desktop
+  parity-evm:
+    command:        parity-evm
+    plugs:          [home, network, network-bind]
+  ethkey:
+    command:        ethkey
+    plugs:          [home]
+  ethstore:
+    command:        ethstore
+    plugs:          [home]
+  whisper:
+    command:        whisper
+    plugs:          [home]
+
+icon:               snap/gui/icon.png
+
+parts:
+  desktop-icon:
+    source:         ./snap
+    plugin:         nil
+    override-build:        |
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/share/applications
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/share/pixmaps
+      cp -v gui/parity.desktop $SNAPCRAFT_PART_INSTALL/usr/share/applications/
+      cp -v gui/icon.png $SNAPCRAFT_PART_INSTALL/usr/share/pixmaps/
+  parity:
+    source:         ./artifacts/$CARGO_TARGET
+    plugin:         nil
+    override-build:        |
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/bin
+      cp -v parity $SNAPCRAFT_PART_INSTALL/usr/bin/parity
+      cp -v parity-evm $SNAPCRAFT_PART_INSTALL/usr/bin/parity-evm
+      cp -v ethkey $SNAPCRAFT_PART_INSTALL/usr/bin/ethkey
+      cp -v ethstore $SNAPCRAFT_PART_INSTALL/usr/bin/ethstore
+      cp -v whisper $SNAPCRAFT_PART_INSTALL/usr/bin/whisper
+    stage-packages: [libc6, libssl1.0.0, libudev1, libstdc++6, cmake]
+  df:
+    plugin:         nil
+    stage-packages: [coreutils]
+    stage:          [bin/df]
--- a/scripts/gitlab/test.sh
+++ b/scripts/gitlab/test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# ARGUMENT $1 Rust flavor to test with (stable/beta/nightly)
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+rustup default $1
+
+if [[ "$CI_COMMIT_REF_NAME" = "beta" || "$CI_COMMIT_REF_NAME" = "stable" ]]; then
+  export GIT_COMPARE=$CI_COMMIT_REF_NAME~;
+else
+  export GIT_COMPARE=master;
+fi
+
+export RUST_FILES_MODIFIED="$(git --no-pager diff --name-only $GIT_COMPARE...$CI_COMMIT_SHA | grep -v -e ^\\. -e ^LICENSE -e ^README.md -e ^test.sh -e ^windows/ -e ^scripts/ -e ^mac/ -e ^nsis/ | wc -l)"
+echo "RUST_FILES_MODIFIED: $RUST_FILES_MODIFIED"
+
+git submodule update --init --recursive
+rustup show
+if [[ "${RUST_FILES_MODIFIED}" == "0" ]];
+then echo "__________Skipping Rust tests since no Rust files modified__________";
+else ./test.sh || exit $?;
+fi
+
+# if [[ "$CI_COMMIT_REF_NAME" == "nightly" ]];
+#   ### @TODO re-enable fail after https://github.com/paritytech/parity-import-tests/issues/3
+#   then sh scripts/aura-test.sh; # || exit $?;
+# fi
				`@@ -0,0 +1 @@`
				`@signtool sign /f %1 /p %2 /tr http://timestamp.comodoca.com /du https://parity.io %3`