Backports for Beta 2.4.0 (#10416)

* no-git for publish jobs, empty artifacts dir (#10393)

* no-git for publish jobs, empty artifacts dir

* fix syntax

* prettiness

* fix prettiness

* should get rid of git in publishing

* Fix to_pod storage trie value decoding (#10368)

* snap: reenable i386, arm64, armhf architecture publishing (#10386)

* snap: reenable i386, arm64, armhf architecture publishing

* gitlab: fix indent

* gitlab: fix yml syntax

* Linker for crosscomile

* fix target to linker

* new docker image

* fix lint, add build to this PR

* calc SHA3 using rhash

* add new images for i386, armhf

* show snap target & artifacts

* set CARGO_TARGET for publish snap

* move detect Version to publish snap

* rm libc6 dep from snap-template up pub-snap script

* clean up cargo config before add linker

* move linker config to docker images

.gitlab-ci.yml

@@ -2,11 +2,9 @@ stages:
   - test
   - build
   - publish
-  - publish-onchain
   - optional
 
-image:                             parity/rust:gitlab-ci
+image:                             parity/rust-parity-ethereum-build:xenial
-
 variables:
   GIT_STRATEGY:                    fetch
   GIT_SUBMODULE_STRATEGY:          recursive
@@ -14,6 +12,12 @@ variables:
   CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
 
+.no_git:                           &no_git
+  variables:
+    GIT_STRATEGY:                  none
+    GIT_SUBMODULE_STRATEGY:        none
+
+
 .releaseable_branches:             # list of git refs for building GitLab artifacts (think "pre-release binaries")
   only:                            &releaseable_branches
     - stable
@@ -30,14 +34,6 @@ variables:
     paths:
     - artifacts/
 
-.determine_version:                &determine_version
-  - VERSION="$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
-  - DATE_STR="$(date +%Y%m%d)"
-  - ID_SHORT="$(echo ${CI_COMMIT_SHA} | cut -c 1-7)"
-  - test "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" = "nightly" && VERSION="${VERSION}-${ID_SHORT}-${DATE_STR}"
-  - export VERSION
-  - echo "Version = ${VERSION}"
-
 test-linux:
   stage:                           test
   variables:
@@ -57,7 +53,7 @@ test-audit:
   tags:
     - linux-docker
 
-build-linux:
+build-linux:                       &build-linux
   stage:                           build
   only:                            *releaseable_branches
   script:
@@ -67,6 +63,24 @@ build-linux:
   tags:
     - linux-docker
 
+build-linux-i386:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:i386
+  variables:
+    CARGO_TARGET:                  i686-unknown-linux-gnu
+
+build-linux-arm64:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:arm64
+  variables:
+    CARGO_TARGET:                  aarch64-unknown-linux-gnu
+
+build-linux-armhf:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:armhf
+  variables:
+    CARGO_TARGET:                  armv7-unknown-linux-gnueabihf
+
 build-darwin:
   stage:                           build
   only:                            *releaseable_branches
@@ -94,6 +108,7 @@ build-windows:
 publish-docker:
   stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache: {}
   dependencies:
     - build-linux
@@ -102,35 +117,54 @@ publish-docker:
   script:
     - scripts/gitlab/publish-docker.sh parity
 
-publish-snap:
+publish-snap:                      &publish-snap
-  stage:                           optional #publish
+  stage:                           publish
   only:                            *releaseable_branches
+  # <<:                              *no_git
   image:                           snapcore/snapcraft
   variables:
     BUILD_ARCH:                    amd64
   cache: {}
-  before_script:                   *determine_version
   dependencies:
     - build-linux
   tags:
-    - rust-stable
+    - linux-docker
   script:
     - scripts/gitlab/publish-snap.sh
-  allow_failure:                   true
   <<:                              *collect_artifacts
 
-publish-onnet-update:
+publish-snap-i386:
-  stage:                           publish-onchain
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    i386
+  dependencies:
+    - build-linux-i386
+
+publish-snap-arm64:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    arm64
+  dependencies:
+    - build-linux-arm64
+
+publish-snap-armhf:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    armhf
+  dependencies:
+    - build-linux-armhf
+
+publish-onchain:
+  stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache:                           {}
   dependencies:
     - build-linux
     - build-darwin
     - build-windows
-    - publish-awss3-release
-  before_script:                   *determine_version
   script:
-    - scripts/gitlab/publish-onnet-update.sh
+    - scripts/gitlab/publish-onchain.sh
   tags:
     - linux-docker
 
@@ -155,6 +189,7 @@ publish-awss3-release:
   image:                           parity/awscli:latest
   stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache:                           {}
   dependencies:
     - build-linux
@@ -203,4 +238,3 @@ build-android:
     - linux-docker
   allow_failure:                   true
   <<:                              *collect_artifacts
-

ethcore/src/state/mod.rs

@@ -1020,7 +1020,7 @@ impl<B: Backend> State<B> {
 		let trie = self.factories.trie.readonly(accountdb, &root)?;
 		for o_kv in trie.iter()? {
 			if let Ok((key, val)) = o_kv {
-				pod_storage.insert(key[..].into(), U256::from(&val[..]).into());
+				pod_storage.insert(key[..].into(), rlp::decode::<U256>(&val[..]).expect("Decoded from trie which was encoded from the same type; qed").into());
 			}
 		}
 

scripts/gitlab/build-unix.sh

@@ -11,6 +11,9 @@ echo "CC:               " $CC
 echo "CXX:              " $CXX
 #strip ON
 export RUSTFLAGS=" -C link-arg=-s"
+# Linker for crosscomile
+echo "_____ Linker _____"
+cat .cargo/config
 
 echo "_____ Building target: "$CARGO_TARGET" _____"
 if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
@@ -25,6 +28,7 @@ else
 fi
 
 echo "_____ Post-processing binaries _____"
+rm -rf artifacts/*
 mkdir -p artifacts/$CARGO_TARGET
 cd artifacts/$CARGO_TARGET
 
@@ -42,11 +46,6 @@ fi
 echo "_____ Calculating checksums _____"
 for binary in $(ls)
 do
-  rhash --sha256 $binary -o $binary.sha256
+  rhash --sha256 $binary -o $binary.sha256 #do we still need this hash (SHA2)?
-  if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
+  rhash --sha3-256 $binary -o $binary.sha3
-  then
-    echo "> ${binary} cannot be hashed with cross-compiled binary"
-  else
-    ./parity tools hash $binary > $binary.sha3
-  fi
 done

scripts/gitlab/publish-snap.sh

@@ -8,7 +8,10 @@ set -u # treat unset variables as error
 # gsub(/ /, "", $2) deletes whitespaces
 TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml`
 echo Track is: $TRACK
-
+# prepare variables
+VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
+SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
+CARGO_TARGET="$(ls artifacts)"
 # Choose snap release channel based on parity ethereum version track
 case ${TRACK} in
   nightly) export GRADE="devel" CHANNEL="edge";;
@@ -21,24 +24,16 @@ esac
 case ${CI_COMMIT_REF_NAME} in
   beta|stable) export GRADE="stable" CHANNEL="candidate";;
 esac
-
-VERSION="v"$VERSION
-SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
-
 echo "__________Create snap package__________"
 echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
-echo $VERSION:$GRADE:$BUILD_ARCH
+echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET
-# cat scripts/snap/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml
+
-# a bit more necromancy (substitutions):
-pwd
-cd /builds/$CI_PROJECT_PATH/scripts/snap/
 sed -e 's/$VERSION/'"$VERSION"'/g' \
     -e 's/$GRADE/'"$GRADE"'/g' \
     -e 's/$BUILD_ARCH/'"$BUILD_ARCH"'/g' \
     -e 's/$CARGO_TARGET/'"$CARGO_TARGET"'/g' \
-    snapcraft.template.yaml > /builds/$CI_PROJECT_PATH/snapcraft.yaml
+    scripts/snap/snapcraft.template.yaml > snapcraft.yaml
-cd /builds/$CI_PROJECT_PATH
+
-pwd
 apt update
 apt install -y --no-install-recommends rhash
 cat snapcraft.yaml

scripts/snap/snapcraft.template.yaml

@@ -50,4 +50,4 @@ parts:
       cp -v ethkey $SNAPCRAFT_PART_INSTALL/usr/bin/ethkey
       cp -v ethstore $SNAPCRAFT_PART_INSTALL/usr/bin/ethstore
       cp -v whisper $SNAPCRAFT_PART_INSTALL/usr/bin/whisper
-    stage-packages: [libc6, libudev1, libstdc++6, cmake, libdb5.3]
+    stage-packages: [libudev1, libstdc++6, cmake, libdb5.3]