diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 660563600..33cfbd9d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,12 +32,15 @@ variables: expire_in: 1 mos paths: - artifacts/ + - tools/ .docker-cache-status: &docker-cache-status variables: CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}" dependencies: [] before_script: + - rustup show + - cargo --version - SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log RUST_LOG=sccache=debug sccache --start-server @@ -46,10 +49,7 @@ variables: # sccache debug info - if test -e sccache_debug.log; then - echo "_____sccache_debug.log listing start:_____"; - cat sccache_debug.log; - echo "_____sccache_debug.log listing end_____"; - echo "All crate-types:"; + echo "_____All crate-types:_____"; grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c; echo "_____Non-cacheable reasons:_____"; grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c; @@ -67,6 +67,16 @@ variables: script: - scripts/gitlab/build-linux.sh - sccache -s + after_script: + - mkdir -p tools + - cp -r scripts/docker/hub/* ./tools + - cp scripts/gitlab/publish-snap.sh ./tools + - cp scripts/gitlab/publish-onchain.sh ./tools + - cp scripts/gitlab/safe-curl.sh ./tools + - echo v"$(sed -r -n '1,/^version/s/^version\s*=\s*"([^"]+)".*$/\1/p' Cargo.toml)" | + tee ./tools/VERSION + - echo "$(sed -r -n '1,/^track/s/^track\s*=\s*"([^"]+)".*$/\1/p' ./util/version/Cargo.toml)" | + tee ./tools/TRACK cargo-check 0 3: @@ -195,19 +205,35 @@ build-windows: publish-docker: stage: publish + <<: *no_git only: *releaseable_branches - cache: {} + except: + variables: + - $SCHEDULE_TAG == "nightly" dependencies: - build-linux - tags: - - shell + environment: + name: parity-build + cache: {} + image: docker:stable + services: + - docker:dind + variables: + DOCKER_HOST: tcp://localhost:2375 + DOCKER_DRIVER: overlay2 + GIT_STRATEGY: none + # DOCKERFILE: tools/Dockerfile + # CONTAINER_IMAGE: parity/parity script: - - scripts/gitlab/publish-docker.sh parity + # we stopped pushing nightlies to dockerhub, will push to own registry prb. + - ./tools/publish-docker.sh + tags: + - kubernetes-parity-build publish-snap: &publish-snap stage: publish + <<: *no_git only: *releaseable_branches - <<: *collect_artifacts image: snapcore/snapcraft variables: BUILD_ARCH: amd64 @@ -217,12 +243,13 @@ publish-snap: &publish-snap tags: - linux-docker script: - - scripts/gitlab/publish-snap.sh + - ./tools/publish-snap.sh publish-snap-i386: <<: *publish-snap variables: BUILD_ARCH: i386 + CARGO_TARGET: i686-unknown-linux-gnu dependencies: - build-linux-i386 @@ -230,6 +257,7 @@ publish-snap-arm64: <<: *publish-snap variables: BUILD_ARCH: arm64 + CARGO_TARGET: aarch64-unknown-linux-gnu dependencies: - build-linux-arm64 @@ -237,11 +265,13 @@ publish-snap-armhf: <<: *publish-snap variables: BUILD_ARCH: armhf + CARGO_TARGET: armv7-unknown-linux-gnueabihf dependencies: - build-linux-armhf publish-onchain: stage: publish + <<: *no_git only: *releaseable_branches cache: {} dependencies: @@ -249,7 +279,7 @@ publish-onchain: - build-darwin - build-windows script: - - scripts/gitlab/publish-onchain.sh + - ./tools/publish-onchain.sh tags: - linux-docker diff --git a/scripts/docker/hub/Dockerfile b/scripts/docker/hub/Dockerfile index c4f2ff429..675364946 100644 --- a/scripts/docker/hub/Dockerfile +++ b/scripts/docker/hub/Dockerfile @@ -1,30 +1,47 @@ FROM ubuntu:xenial -LABEL MAINTAINER="Parity Technologies " -# install tools and dependencies -RUN apt update && apt install -y --no-install-recommends openssl file curl jq +# metadata +ARG VCS_REF +ARG BUILD_DATE + +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="parity/parity" \ + io.parity.image.description="Parity Ethereum. The Fastest and most Advanced Ethereum Client." \ + io.parity.image.source="https://github.com/paritytech/parity-ethereum/blob/${VCS_REF}/\ +scripts/docker/hub/Dockerfile" \ + io.parity.image.documentation="https://wiki.parity.io/Parity-Ethereum" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" # show backtraces ENV RUST_BACKTRACE 1 -# cleanup Docker image -RUN apt autoremove -y \ - && apt clean -y \ - && rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* - -RUN groupadd -g 1000 parity \ - && useradd -m -u 1000 -g parity -s /bin/sh parity +# install tools and dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + file curl jq; \ +# apt cleanup + apt-get autoremove -y; \ + apt-get clean; \ + rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*; \ +# add user + groupadd -g 1000 parity; \ + useradd -m -u 1000 -g parity -s /bin/sh parity WORKDIR /home/parity -# add parity-ethereum to docker image +# add parity-ethereum binary to docker image COPY artifacts/x86_64-unknown-linux-gnu/parity /bin/parity - -COPY scripts/docker/hub/check_sync.sh /check_sync.sh +COPY tools/check_sync.sh /check_sync.sh # switch to user parity here USER parity +# check if executable works in this container +RUN parity --version + EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp ENTRYPOINT ["/bin/parity"] diff --git a/scripts/docker/hub/publish-docker.sh b/scripts/docker/hub/publish-docker.sh new file mode 100755 index 000000000..6602d55c2 --- /dev/null +++ b/scripts/docker/hub/publish-docker.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -e # fail on any error + +VERSION=$(cat ./tools/VERSION) +echo "Parity Ethereum version = ${VERSION}" + +test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" \ + || ( echo "no docker credentials provided"; exit 1 ) +docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity" +echo "__________Docker info__________" +docker info + +# we stopped pushing nightlies to dockerhub, will push to own registry prb. +case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in + "$SCHEDULE_TAG") + echo "Docker TAG - 'parity/parity:${SCHEDULE_TAG}'"; + docker build --no-cache \ + --build-arg VCS_REF="${CI_COMMIT_SHA}" \ + --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + --tag "parity/parity:${SCHEDULE_TAG}" \ + --file tools/Dockerfile .; + docker push "parity/parity:${SCHEDULE_TAG}";; + "beta") + echo "Docker TAGs - 'parity/parity:beta', 'parity/parity:latest', \ + 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'"; + docker build --no-cache \ + --build-arg VCS_REF="${CI_COMMIT_SHA}" \ + --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + --tag "parity/parity:beta" \ + --tag "parity/parity:latest" \ + --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \ + --file tools/Dockerfile .; + docker push "parity/parity:beta"; + docker push "parity/parity:latest"; + docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";; + "stable") + echo "Docker TAGs - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}', 'parity/parity:stable'"; + docker build --no-cache \ + --build-arg VCS_REF="${CI_COMMIT_SHA}" \ + --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \ + --tag "parity/parity:stable" \ + --file tools/Dockerfile .; + docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}"; + docker push "parity/parity:stable";; + *) + echo "Docker TAG - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'" + docker build --no-cache \ + --build-arg VCS_REF="${CI_COMMIT_SHA}" \ + --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + --tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \ + --file tools/Dockerfile .; + docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";; +esac + +docker logout diff --git a/scripts/gitlab/publish-docker.sh b/scripts/gitlab/publish-docker.sh deleted file mode 100755 index e8697fac2..000000000 --- a/scripts/gitlab/publish-docker.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -##ARGUMENTS: 1. Docker target -set -e # fail on any error -set -u # treat unset variables as error - -if [ "$CI_COMMIT_REF_NAME" == "master" ]; - then export DOCKER_BUILD_TAG="${SCHEDULE_TAG:-latest}"; - else export DOCKER_BUILD_TAG=$CI_COMMIT_REF_NAME; -fi -docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity - -echo "__________Docker TAG__________" -echo $DOCKER_BUILD_TAG - -echo "__________Docker target__________" -export DOCKER_TARGET=$1 -echo $DOCKER_TARGET - -echo "__________Docker build and push__________" -docker build --build-arg TARGET=$DOCKER_TARGET --no-cache=true --tag parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG -f scripts/docker/hub/Dockerfile . -docker push parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG -docker logout diff --git a/scripts/gitlab/publish-docs.sh b/scripts/gitlab/publish-docs.sh index 2508c8255..16a781eeb 100755 --- a/scripts/gitlab/publish-docs.sh +++ b/scripts/gitlab/publish-docs.sh @@ -33,8 +33,8 @@ update_wiki_docs() { setup_git() { echo "__________Set github__________" - git config --global user.email "devops@parity.com" - git config --global user.name "Devops Parity" + git config --global user.email "devops-team@parity.io" + git config --global user.name "Devops Team Parity" } set_remote_wiki() { diff --git a/scripts/gitlab/publish-onchain.sh b/scripts/gitlab/publish-onchain.sh index 588cbdfb5..9c7b866fb 100755 --- a/scripts/gitlab/publish-onchain.sh +++ b/scripts/gitlab/publish-onchain.sh @@ -7,10 +7,7 @@ echo "__________Register Release__________" DATA="secret=$RELEASES_SECRET" echo "Pushing release to Mainnet" -./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA" - -echo "Pushing release to Kovan" -./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA" +./tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA" cd artifacts ls -l | sort -k9 @@ -29,9 +26,7 @@ do case $DIR in x86_64* ) DATA="commit=$CI_COMMIT_SHA&sha3=$sha3&filename=parity$WIN&secret=$RELEASES_SECRET" - ../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR" - # Kovan - ../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR" + ../../tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR" ;; esac cd .. diff --git a/scripts/gitlab/publish-snap.sh b/scripts/gitlab/publish-snap.sh index 5e0231af0..d9f10ce3e 100755 --- a/scripts/gitlab/publish-snap.sh +++ b/scripts/gitlab/publish-snap.sh @@ -3,15 +3,11 @@ set -e # fail on any error set -u # treat unset variables as error -# some necromancy: -# gsub(/"/, "", $2) deletes "qoutes" -# gsub(/ /, "", $2) deletes whitespaces -TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml` -echo Track is: $TRACK # prepare variables -VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)" +TRACK=$(cat ./tools/TRACK) +echo "Track is: ${TRACK}" +VERSION=$(cat ./tools/VERSION) SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap" -CARGO_TARGET="$(ls artifacts)" # Choose snap release channel based on parity ethereum version track case ${TRACK} in nightly) export GRADE="devel" CHANNEL="edge";; @@ -20,12 +16,8 @@ case ${TRACK} in *) echo "No release" && exit 0;; esac -# Release untagged versions from branches to the candidate snap channel -case ${CI_COMMIT_REF_NAME} in - beta|stable) export GRADE="stable" CHANNEL="candidate";; -esac echo "__________Create snap package__________" -echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME +echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME "Track: " ${TRACK} echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET sed -e 's/$VERSION/'"$VERSION"'/g' \