Update publishing (#10644)

* docker images are now built on k8s: test run

* copy check_sync.sh in build-linux job

* copy scripts/docker/hub/* in build-linux job

* removed cache var

* cleanup, no more nightly dockers

* cleanup in dockerfile

* some new tags

* removed sccsche debug log, cleanup

* no_gits, new artifacts dir, changed scripts. Test run.

* define version once

* one source for TRACK

* stop kovan onchain updates

* moved changes for two images to a new branch

* rename Dockerfile

* no need in libudev-dev

.gitlab-ci.yml

@@ -32,12 +32,15 @@ variables:
     expire_in:                     1 mos
     paths:
       - artifacts/
+      - tools/
 
 .docker-cache-status:              &docker-cache-status
   variables:
     CARGO_HOME:                    "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}"
   dependencies:                    []
   before_script:
+    - rustup show
+    - cargo --version
     - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log
         RUST_LOG=sccache=debug
         sccache --start-server
@@ -46,10 +49,7 @@ variables:
     # sccache debug info
     - if test -e sccache_debug.log;
       then
-        echo "_____sccache_debug.log listing start:_____";
+        echo "_____All crate-types:_____";
-        cat sccache_debug.log;
-        echo "_____sccache_debug.log listing end_____";
-        echo "All crate-types:";
         grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c;
         echo "_____Non-cacheable reasons:_____";
         grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c;
@@ -67,6 +67,16 @@ variables:
   script:
     - scripts/gitlab/build-linux.sh
     - sccache -s
+  after_script:
+    - mkdir -p tools
+    - cp -r scripts/docker/hub/* ./tools
+    - cp scripts/gitlab/publish-snap.sh ./tools
+    - cp scripts/gitlab/publish-onchain.sh ./tools
+    - cp scripts/gitlab/safe-curl.sh ./tools
+    - echo v"$(sed -r -n '1,/^version/s/^version\s*=\s*"([^"]+)".*$/\1/p' Cargo.toml)" |
+      tee ./tools/VERSION
+    - echo "$(sed -r -n '1,/^track/s/^track\s*=\s*"([^"]+)".*$/\1/p' ./util/version/Cargo.toml)" |
+      tee ./tools/TRACK
 
 
 cargo-check 0 3:
@@ -195,19 +205,35 @@ build-windows:
 
 publish-docker:
   stage:                           publish
+  <<:                              *no_git
   only:                            *releaseable_branches
-  cache:                           {}
+  except:
+    variables:
+      - $SCHEDULE_TAG == "nightly"
   dependencies:
     - build-linux
-  tags:
+  environment:
-    - shell
+    name: parity-build
+  cache:                           {}
+  image:                           docker:stable
+  services:
+    - docker:dind
+  variables:
+    DOCKER_HOST:                   tcp://localhost:2375
+    DOCKER_DRIVER:                 overlay2
+    GIT_STRATEGY:                  none
+    # DOCKERFILE:                  tools/Dockerfile
+    # CONTAINER_IMAGE:             parity/parity 
   script:
-    - scripts/gitlab/publish-docker.sh parity
+    # we stopped pushing nightlies to dockerhub, will push to own registry prb.
+    - ./tools/publish-docker.sh
+  tags:
+    - kubernetes-parity-build
 
 publish-snap:                      &publish-snap
   stage:                           publish
+  <<:                              *no_git
   only:                            *releaseable_branches
-  <<:                              *collect_artifacts
   image:                           snapcore/snapcraft
   variables:
     BUILD_ARCH:                    amd64
@@ -217,12 +243,13 @@ publish-snap:                      &publish-snap
   tags:
     - linux-docker
   script:
-    - scripts/gitlab/publish-snap.sh
+    - ./tools/publish-snap.sh
 
 publish-snap-i386:
   <<:                              *publish-snap
   variables:
     BUILD_ARCH:                    i386
+    CARGO_TARGET:                  i686-unknown-linux-gnu
   dependencies:
     - build-linux-i386
 
@@ -230,6 +257,7 @@ publish-snap-arm64:
   <<:                              *publish-snap
   variables:
     BUILD_ARCH:                    arm64
+    CARGO_TARGET:                  aarch64-unknown-linux-gnu
   dependencies:
     - build-linux-arm64
 
@@ -237,11 +265,13 @@ publish-snap-armhf:
   <<:                              *publish-snap
   variables:
     BUILD_ARCH:                    armhf
+    CARGO_TARGET:                  armv7-unknown-linux-gnueabihf
   dependencies:
     - build-linux-armhf
 
 publish-onchain:
   stage:                           publish
+  <<:                              *no_git
   only:                            *releaseable_branches
   cache:                           {}
   dependencies:
@@ -249,7 +279,7 @@ publish-onchain:
     - build-darwin
     - build-windows
   script:
-    - scripts/gitlab/publish-onchain.sh
+    - ./tools/publish-onchain.sh
   tags:
     - linux-docker
 

scripts/docker/hub/Dockerfile

@@ -1,30 +1,47 @@
 FROM ubuntu:xenial
-LABEL MAINTAINER="Parity Technologies <devops-team@parity.io>"
 
-# install tools and dependencies
+# metadata
-RUN apt update && apt install -y --no-install-recommends openssl file curl jq
+ARG VCS_REF
+ARG BUILD_DATE
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="parity/parity" \
+	io.parity.image.description="Parity Ethereum. The Fastest and most Advanced Ethereum Client." \
+	io.parity.image.source="https://github.com/paritytech/parity-ethereum/blob/${VCS_REF}/\
+scripts/docker/hub/Dockerfile" \
+	io.parity.image.documentation="https://wiki.parity.io/Parity-Ethereum" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}"
 
 # show backtraces
 ENV RUST_BACKTRACE 1
 
-# cleanup Docker image
+# install tools and dependencies
-RUN apt autoremove -y \
+RUN set -eux; \
-  && apt clean -y \
+	apt-get update; \
-  && rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*
+	apt-get install -y --no-install-recommends \
-
+		file curl jq; \
-RUN groupadd -g 1000 parity \
+# apt cleanup
-  && useradd -m -u 1000 -g parity -s /bin/sh parity
+	apt-get autoremove -y; \
+	apt-get clean; \
+	rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*; \
+# add user
+	groupadd -g 1000 parity; \
+	useradd -m -u 1000 -g parity -s /bin/sh parity
 
 WORKDIR /home/parity
 
-# add parity-ethereum to docker image
+# add parity-ethereum binary to docker image
 COPY artifacts/x86_64-unknown-linux-gnu/parity /bin/parity
-
+COPY tools/check_sync.sh /check_sync.sh
-COPY scripts/docker/hub/check_sync.sh /check_sync.sh
 
 # switch to user parity here
 USER parity
 
+# check if executable works in this container
+RUN parity --version
+
 EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp
 
 ENTRYPOINT ["/bin/parity"]

scripts/docker/hub/publish-docker.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -e # fail on any error
+
+VERSION=$(cat ./tools/VERSION)
+echo "Parity Ethereum version = ${VERSION}"
+
+test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" \
+    || ( echo "no docker credentials provided"; exit 1 )
+docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity"
+echo "__________Docker info__________"
+docker info
+
+# we stopped pushing nightlies to dockerhub, will push to own registry prb.
+case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
+    "$SCHEDULE_TAG")
+        echo "Docker TAG - 'parity/parity:${SCHEDULE_TAG}'";
+        docker build --no-cache \
+            --build-arg VCS_REF="${CI_COMMIT_SHA}" \
+            --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+            --tag "parity/parity:${SCHEDULE_TAG}" \
+            --file tools/Dockerfile .;
+        docker push "parity/parity:${SCHEDULE_TAG}";;
+    "beta")
+        echo "Docker TAGs - 'parity/parity:beta', 'parity/parity:latest', \
+            'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'";
+        docker build --no-cache \
+            --build-arg VCS_REF="${CI_COMMIT_SHA}" \
+            --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+            --tag "parity/parity:beta" \
+            --tag "parity/parity:latest" \
+            --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
+            --file tools/Dockerfile .;
+        docker push "parity/parity:beta";
+        docker push "parity/parity:latest";
+        docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";;
+    "stable")
+        echo "Docker TAGs - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}', 'parity/parity:stable'";
+        docker build --no-cache \
+            --build-arg VCS_REF="${CI_COMMIT_SHA}" \
+            --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+            --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
+            --tag "parity/parity:stable" \
+            --file tools/Dockerfile .;
+        docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";
+        docker push "parity/parity:stable";;
+    *)
+        echo "Docker TAG - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'"
+        docker build --no-cache \
+            --build-arg VCS_REF="${CI_COMMIT_SHA}" \
+            --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+            --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
+            --file tools/Dockerfile .;
+        docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";;
+esac
+
+docker logout

scripts/gitlab/publish-docker.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-##ARGUMENTS: 1. Docker target
-set -e # fail on any error
-set -u # treat unset variables as error
-
-if [ "$CI_COMMIT_REF_NAME" == "master" ];
-	then export DOCKER_BUILD_TAG="${SCHEDULE_TAG:-latest}";
-	else export DOCKER_BUILD_TAG=$CI_COMMIT_REF_NAME;
-fi
-docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity
-
-echo "__________Docker TAG__________"
-echo $DOCKER_BUILD_TAG
-
-echo "__________Docker target__________"
-export DOCKER_TARGET=$1
-echo $DOCKER_TARGET
-
-echo "__________Docker build and push__________"
-docker build --build-arg TARGET=$DOCKER_TARGET --no-cache=true --tag parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG -f scripts/docker/hub/Dockerfile .
-docker push parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG
-docker logout

scripts/gitlab/publish-docs.sh

@@ -33,8 +33,8 @@ update_wiki_docs() {
 
 setup_git() {
     echo "__________Set github__________"
-    git config --global user.email "devops@parity.com"
+    git config --global user.email "devops-team@parity.io"
-    git config --global user.name "Devops Parity"
+    git config --global user.name "Devops Team Parity"
 }
 
 set_remote_wiki() {

scripts/gitlab/publish-onchain.sh

@@ -7,10 +7,7 @@ echo "__________Register Release__________"
 DATA="secret=$RELEASES_SECRET"
 
 echo "Pushing release to Mainnet"
-./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
+./tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
-
-echo "Pushing release to Kovan"
-./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
 
 cd artifacts
 ls -l | sort -k9
@@ -29,9 +26,7 @@ do
   case $DIR in
     x86_64* )
       DATA="commit=$CI_COMMIT_SHA&sha3=$sha3&filename=parity$WIN&secret=$RELEASES_SECRET"
-      ../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
+      ../../tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
-      # Kovan
-      ../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
       ;;
   esac
   cd ..

scripts/gitlab/publish-snap.sh

@@ -3,15 +3,11 @@
 set -e # fail on any error
 set -u # treat unset variables as error
 
-# some necromancy:
-# gsub(/"/, "", $2) deletes "qoutes"
-# gsub(/ /, "", $2) deletes whitespaces
-TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml`
-echo Track is: $TRACK
 # prepare variables
-VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
+TRACK=$(cat ./tools/TRACK)
+echo "Track is: ${TRACK}"
+VERSION=$(cat ./tools/VERSION)
 SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
-CARGO_TARGET="$(ls artifacts)"
 # Choose snap release channel based on parity ethereum version track
 case ${TRACK} in
   nightly) export GRADE="devel" CHANNEL="edge";;
@@ -20,12 +16,8 @@ case ${TRACK} in
   *) echo "No release" && exit 0;;
 esac
 
-# Release untagged versions from branches to the candidate snap channel
-case ${CI_COMMIT_REF_NAME} in
-  beta|stable) export GRADE="stable" CHANNEL="candidate";;
-esac
 echo "__________Create snap package__________"
-echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
+echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME "Track: " ${TRACK}
 echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET
 
 sed -e 's/$VERSION/'"$VERSION"'/g' \