grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update publishing (#10644)

Browse Source 
* docker images are now built on k8s: test run

* copy check_sync.sh in build-linux job

* copy scripts/docker/hub/* in build-linux job

* removed cache var

* cleanup, no more nightly dockers

* cleanup in dockerfile

* some new tags

* removed sccsche debug log, cleanup

* no_gits, new artifacts dir, changed scripts. Test run.

* define version once

* one source for TRACK

* stop kovan onchain updates

* moved changes for two images to a new branch

* rename Dockerfile

* no need in libudev-dev
This commit is contained in:
TriplEight 2019-05-31 13:27:00 +02:00 committed by GitHub
parent 9584faee55
commit 7e89bab4aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 136 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
.gitlab-ci.yml
View File

@ -32,12 +32,15 @@ variables:
expire_in: 1 mos
paths:
- artifacts/
- tools/
.docker-cache-status: &docker-cache-status
variables:
CARGO_HOME: "/ci-cache/parity-ethereum/cargo/${CI_JOB_NAME}"
dependencies: []
before_script:
- rustup show
- cargo --version
- SCCACHE_ERROR_LOG=/builds/parity/parity-ethereum/sccache_debug.log
RUST_LOG=sccache=debug
sccache --start-server
@ -46,10 +49,7 @@ variables:
# sccache debug info
- if test -e sccache_debug.log;
then
echo "_____sccache_debug.log listing start:_____";
cat sccache_debug.log;
echo "_____sccache_debug.log listing end_____";
echo "All crate-types:";
echo "_____All crate-types:_____";
grep 'parse_arguments.*--crate-type' sccache_debug.log | sed -re 's/.*"--crate-type", "([^"]+)".*/\1/' | sort | uniq -c;
echo "_____Non-cacheable reasons:_____";
grep CannotCache sccache_debug.log | sed -re 's/.*CannotCache\((.+)\).*/\1/' | sort | uniq -c;
@ -67,6 +67,16 @@ variables:
script:
- scripts/gitlab/build-linux.sh
- sccache -s
after_script:
- mkdir -p tools
- cp -r scripts/docker/hub/* ./tools
- cp scripts/gitlab/publish-snap.sh ./tools
- cp scripts/gitlab/publish-onchain.sh ./tools
- cp scripts/gitlab/safe-curl.sh ./tools
- echo v"$(sed -r -n '1,/^version/s/^version\s*=\s*"([^"]+)".*$/\1/p' Cargo.toml)" |
tee ./tools/VERSION
- echo "$(sed -r -n '1,/^track/s/^track\s*=\s*"([^"]+)".*$/\1/p' ./util/version/Cargo.toml)" |
tee ./tools/TRACK
cargo-check 0 3:
@ -195,19 +205,35 @@ build-windows:
publish-docker:
stage: publish
<<: *no_git
only: *releaseable_branches
cache: {}
except:
variables:
- $SCHEDULE_TAG == "nightly"
dependencies:
- build-linux
tags:
- shell
environment:
name: parity-build
cache: {}
image: docker:stable
services:
- docker:dind
variables:
DOCKER_HOST: tcp://localhost:2375
DOCKER_DRIVER: overlay2
GIT_STRATEGY: none
# DOCKERFILE: tools/Dockerfile
# CONTAINER_IMAGE: parity/parity
script:
- scripts/gitlab/publish-docker.sh parity
# we stopped pushing nightlies to dockerhub, will push to own registry prb.
- ./tools/publish-docker.sh
tags:
- kubernetes-parity-build
publish-snap: &publish-snap
stage: publish
<<: *no_git
only: *releaseable_branches
<<: *collect_artifacts
image: snapcore/snapcraft
variables:
BUILD_ARCH: amd64
@ -217,12 +243,13 @@ publish-snap: &publish-snap
tags:
- linux-docker
script:
- scripts/gitlab/publish-snap.sh
- ./tools/publish-snap.sh
publish-snap-i386:
<<: *publish-snap
variables:
BUILD_ARCH: i386
CARGO_TARGET: i686-unknown-linux-gnu
dependencies:
- build-linux-i386
@ -230,6 +257,7 @@ publish-snap-arm64:
<<: *publish-snap
variables:
BUILD_ARCH: arm64
CARGO_TARGET: aarch64-unknown-linux-gnu
dependencies:
- build-linux-arm64
@ -237,11 +265,13 @@ publish-snap-armhf:
<<: *publish-snap
variables:
BUILD_ARCH: armhf
CARGO_TARGET: armv7-unknown-linux-gnueabihf
dependencies:
- build-linux-armhf
publish-onchain:
stage: publish
<<: *no_git
only: *releaseable_branches
cache: {}
dependencies:
@ -249,7 +279,7 @@ publish-onchain:
- build-darwin
- build-windows
script:
- scripts/gitlab/publish-onchain.sh
- ./tools/publish-onchain.sh
tags:
- linux-docker

43
scripts/docker/hub/Dockerfile
View File

@ -1,30 +1,47 @@
FROM ubuntu:xenial
LABEL MAINTAINER="Parity Technologies <devops-team@parity.io>"
# install tools and dependencies
RUN apt update && apt install -y --no-install-recommends openssl file curl jq
# metadata
ARG VCS_REF
ARG BUILD_DATE
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="parity/parity" \
io.parity.image.description="Parity Ethereum. The Fastest and most Advanced Ethereum Client." \
io.parity.image.source="https://github.com/paritytech/parity-ethereum/blob/${VCS_REF}/\
scripts/docker/hub/Dockerfile" \
io.parity.image.documentation="https://wiki.parity.io/Parity-Ethereum" \
io.parity.image.revision="${VCS_REF}" \
io.parity.image.created="${BUILD_DATE}"
# show backtraces
ENV RUST_BACKTRACE 1
# cleanup Docker image
RUN apt autoremove -y \
&& apt clean -y \
&& rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*
RUN groupadd -g 1000 parity \
&& useradd -m -u 1000 -g parity -s /bin/sh parity
# install tools and dependencies
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
file curl jq; \
# apt cleanup
apt-get autoremove -y; \
apt-get clean; \
rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*; \
# add user
groupadd -g 1000 parity; \
useradd -m -u 1000 -g parity -s /bin/sh parity
WORKDIR /home/parity
# add parity-ethereum to docker image
# add parity-ethereum binary to docker image
COPY artifacts/x86_64-unknown-linux-gnu/parity /bin/parity
COPY scripts/docker/hub/check_sync.sh /check_sync.sh
COPY tools/check_sync.sh /check_sync.sh
# switch to user parity here
USER parity
# check if executable works in this container
RUN parity --version
EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp
ENTRYPOINT ["/bin/parity"]

57
scripts/docker/hub/publish-docker.sh Executable file
View File

@ -0,0 +1,57 @@
#!/bin/sh
set -e # fail on any error
VERSION=$(cat ./tools/VERSION)
echo "Parity Ethereum version = ${VERSION}"
test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" \
|| ( echo "no docker credentials provided"; exit 1 )
docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity"
echo "__________Docker info__________"
docker info
# we stopped pushing nightlies to dockerhub, will push to own registry prb.
case "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" in
"$SCHEDULE_TAG")
echo "Docker TAG - 'parity/parity:${SCHEDULE_TAG}'";
docker build --no-cache \
--build-arg VCS_REF="${CI_COMMIT_SHA}" \
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
--tag "parity/parity:${SCHEDULE_TAG}" \
--file tools/Dockerfile .;
docker push "parity/parity:${SCHEDULE_TAG}";;
"beta")
echo "Docker TAGs - 'parity/parity:beta', 'parity/parity:latest', \
'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'";
docker build --no-cache \
--build-arg VCS_REF="${CI_COMMIT_SHA}" \
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
--tag "parity/parity:beta" \
--tag "parity/parity:latest" \
--tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
--file tools/Dockerfile .;
docker push "parity/parity:beta";
docker push "parity/parity:latest";
docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";;
"stable")
echo "Docker TAGs - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}', 'parity/parity:stable'";
docker build --no-cache \
--build-arg VCS_REF="${CI_COMMIT_SHA}" \
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
--tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
--tag "parity/parity:stable" \
--file tools/Dockerfile .;
docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";
docker push "parity/parity:stable";;
*)
echo "Docker TAG - 'parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}'"
docker build --no-cache \
--build-arg VCS_REF="${CI_COMMIT_SHA}" \
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
--tag "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}" \
--file tools/Dockerfile .;
docker push "parity/parity:${VERSION}-${CI_COMMIT_REF_NAME}";;
esac
docker logout

22
scripts/gitlab/publish-docker.sh
View File

@ -1,22 +0,0 @@
#!/bin/bash
##ARGUMENTS: 1. Docker target
set -e # fail on any error
set -u # treat unset variables as error
if [ "$CI_COMMIT_REF_NAME" == "master" ];
then export DOCKER_BUILD_TAG="${SCHEDULE_TAG:-latest}";
else export DOCKER_BUILD_TAG=$CI_COMMIT_REF_NAME;
fi
docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity
echo "__________Docker TAG__________"
echo $DOCKER_BUILD_TAG
echo "__________Docker target__________"
export DOCKER_TARGET=$1
echo $DOCKER_TARGET
echo "__________Docker build and push__________"
docker build --build-arg TARGET=$DOCKER_TARGET --no-cache=true --tag parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG -f scripts/docker/hub/Dockerfile .
docker push parity/$DOCKER_TARGET:$DOCKER_BUILD_TAG
docker logout

4
scripts/gitlab/publish-docs.sh
View File

@ -33,8 +33,8 @@ update_wiki_docs() {
setup_git() {
echo "__________Set github__________"
git config --global user.email "devops@parity.com"
git config --global user.name "Devops Parity"
git config --global user.email "devops-team@parity.io"
git config --global user.name "Devops Team Parity"
}
set_remote_wiki() {

9
scripts/gitlab/publish-onchain.sh
View File

@ -7,10 +7,7 @@ echo "__________Register Release__________"
DATA="secret=$RELEASES_SECRET"
echo "Pushing release to Mainnet"
./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
echo "Pushing release to Kovan"
./scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
./tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-release/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$CI_COMMIT_SHA"
cd artifacts
ls -l | sort -k9
@ -29,9 +26,7 @@ do
case $DIR in
x86_64* )
DATA="commit=$CI_COMMIT_SHA&sha3=$sha3&filename=parity$WIN&secret=$RELEASES_SECRET"
../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
# Kovan
../../scripts/gitlab/safe-curl.sh $DATA "http://update.parity.io:1338/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
../../tools/safe-curl.sh $DATA "http://update.parity.io:1337/push-build/${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}/$DIR"
;;
esac
cd ..

16
scripts/gitlab/publish-snap.sh
View File

@ -3,15 +3,11 @@
set -e # fail on any error
set -u # treat unset variables as error
# some necromancy:
# gsub(/"/, "", $2) deletes "qoutes"
# gsub(/ /, "", $2) deletes whitespaces
TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml`
echo Track is: $TRACK
# prepare variables
VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
TRACK=$(cat ./tools/TRACK)
echo "Track is: ${TRACK}"
VERSION=$(cat ./tools/VERSION)
SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
CARGO_TARGET="$(ls artifacts)"
# Choose snap release channel based on parity ethereum version track
case ${TRACK} in
nightly) export GRADE="devel" CHANNEL="edge";;
@ -20,12 +16,8 @@ case ${TRACK} in
*) echo "No release" && exit 0;;
esac
# Release untagged versions from branches to the candidate snap channel
case ${CI_COMMIT_REF_NAME} in
beta|stable) export GRADE="stable" CHANNEL="candidate";;
esac
echo "__________Create snap package__________"
echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME "Track: " ${TRACK}
echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET
sed -e 's/$VERSION/'"$VERSION"'/g' \