Backports for Stable 2.3.5 (#10414)

* no-git for publish jobs, empty artifacts dir (#10393)

* no-git for publish jobs, empty artifacts dir

* fix syntax

* prettiness

* fix prettiness

* should get rid of git in publishing

* snap: reenable i386, arm64, armhf architecture publishing (#10386)

* snap: reenable i386, arm64, armhf architecture publishing

* gitlab: fix indent

* gitlab: fix yml syntax

* Linker for crosscomile

* fix target to linker

* new docker image

* fix lint, add build to this PR

* calc SHA3 using rhash

* add new images for i386, armhf

* show snap target & artifacts

* set CARGO_TARGET for publish snap

* move detect Version to publish snap

* rm libc6 dep from snap-template up pub-snap script

* clean up cargo config before add linker

* move linker config to docker images

* tx pool: always accept local transactions (#10375)

* tx pool: always accept local transactions

* tx pool: `choose` local txs with same sender and nonce

* Fix to_pod storage trie value decoding (#10368)

.gitlab-ci.yml

@@ -2,11 +2,9 @@ stages:
   - test
   - build
   - publish
-  - publish-onchain
   - optional
 
-image:                             parity/rust:gitlab-ci
+image:                             parity/rust-parity-ethereum-build:xenial
-
 variables:
   GIT_STRATEGY:                    fetch
   GIT_SUBMODULE_STRATEGY:          recursive
@@ -14,6 +12,12 @@ variables:
   CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
   CARGO_TARGET:                    x86_64-unknown-linux-gnu
 
+.no_git:                           &no_git
+  variables:
+    GIT_STRATEGY:                  none
+    GIT_SUBMODULE_STRATEGY:        none
+
+
 .releaseable_branches:             # list of git refs for building GitLab artifacts (think "pre-release binaries")
   only:                            &releaseable_branches
     - stable
@@ -30,14 +34,6 @@ variables:
     paths:
     - artifacts/
 
-.determine_version:                &determine_version
-  - VERSION="$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
-  - DATE_STR="$(date +%Y%m%d)"
-  - ID_SHORT="$(echo ${CI_COMMIT_SHA} | cut -c 1-7)"
-  - test "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" = "nightly" && VERSION="${VERSION}-${ID_SHORT}-${DATE_STR}"
-  - export VERSION
-  - echo "Version = ${VERSION}"
-
 test-linux:
   stage:                           test
   variables:
@@ -57,7 +53,7 @@ test-audit:
   tags:
     - linux-docker
 
-build-linux:
+build-linux:                       &build-linux
   stage:                           build
   only:                            *releaseable_branches
   script:
@@ -67,6 +63,24 @@ build-linux:
   tags:
     - linux-docker
 
+build-linux-i386:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:i386
+  variables:
+    CARGO_TARGET:                  i686-unknown-linux-gnu
+
+build-linux-arm64:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:arm64
+  variables:
+    CARGO_TARGET:                  aarch64-unknown-linux-gnu
+
+build-linux-armhf:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:armhf
+  variables:
+    CARGO_TARGET:                  armv7-unknown-linux-gnueabihf
+
 build-darwin:
   stage:                           build
   only:                            *releaseable_branches
@@ -94,6 +108,7 @@ build-windows:
 publish-docker:
   stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache: {}
   dependencies:
     - build-linux
@@ -102,35 +117,54 @@ publish-docker:
   script:
     - scripts/gitlab/publish-docker.sh parity
 
-publish-snap:
+publish-snap:                      &publish-snap
-  stage:                           optional #publish
+  stage:                           publish
   only:                            *releaseable_branches
+  # <<:                              *no_git
   image:                           snapcore/snapcraft
   variables:
-    BUILD_ARCH:                  amd64
+    BUILD_ARCH:                    amd64
   cache: {}
-  before_script:                   *determine_version
   dependencies:
     - build-linux
   tags:
-    - rust-stable
+    - linux-docker
   script:
     - scripts/gitlab/publish-snap.sh
-  allow_failure:                   true
   <<:                              *collect_artifacts
 
-publish-onnet-update:
+publish-snap-i386:
-  stage:                           publish-onchain
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    i386
+  dependencies:
+    - build-linux-i386
+
+publish-snap-arm64:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    arm64
+  dependencies:
+    - build-linux-arm64
+
+publish-snap-armhf:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    armhf
+  dependencies:
+    - build-linux-armhf
+
+publish-onchain:
+  stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache:                           {}
   dependencies:
     - build-linux
     - build-darwin
     - build-windows
-    - publish-awss3-release
-  before_script:                   *determine_version
   script:
-    - scripts/gitlab/publish-onnet-update.sh
+    - scripts/gitlab/publish-onchain.sh
   tags:
     - linux-docker
 
@@ -155,6 +189,7 @@ publish-awss3-release:
   image:                           parity/awscli:latest
   stage:                           publish
   only:                            *releaseable_branches
+  <<:                              *no_git
   cache:                           {}
   dependencies:
     - build-linux
@@ -203,4 +238,3 @@ build-android:
     - linux-docker
   allow_failure:                   true
   <<:                              *collect_artifacts
-

ethcore/src/state/mod.rs

@@ -1017,7 +1017,7 @@ impl<B: Backend> State<B> {
 		let trie = self.factories.trie.readonly(accountdb.as_hashdb(), &root)?;
 		for o_kv in trie.iter()? {
 			if let Ok((key, val)) = o_kv {
-				pod_storage.insert(key[..].into(), U256::from(&val[..]).into());
+				pod_storage.insert(key[..].into(), rlp::decode::<U256>(&val[..]).expect("Decoded from trie which was encoded from the same type; qed").into());
 			}
 		}
 

miner/src/pool/scoring.rs

@@ -123,15 +123,16 @@ impl<P> txpool::Scoring<P> for NonceAndGasPrice where P: ScoredTransaction + txp
 	}
 
 	fn should_replace(&self, old: &P, new: &P) -> scoring::Choice {
+		let both_local = old.priority().is_local() && new.priority().is_local();
 		if old.sender() == new.sender() {
 			// prefer earliest transaction
 			match new.nonce().cmp(&old.nonce()) {
+				cmp::Ordering::Equal => self.choose(old, new),
+				_ if both_local => scoring::Choice::InsertNew,
 				cmp::Ordering::Less => scoring::Choice::ReplaceOld,
 				cmp::Ordering::Greater => scoring::Choice::RejectNew,
-				cmp::Ordering::Equal => self.choose(old, new),
 			}
-		} else if old.priority().is_local() && new.priority().is_local() {
+		} else if both_local {
-			// accept local transactions over the limit
 			scoring::Choice::InsertNew
 		} else {
 			let old_score = (old.priority(), old.gas_price());
@@ -141,7 +142,7 @@ impl<P> txpool::Scoring<P> for NonceAndGasPrice where P: ScoredTransaction + txp
 			} else {
 				scoring::Choice::RejectNew
 			}
-	 	}
+		}
 	}
 
 	fn should_ignore_sender_limit(&self, new: &P) -> bool {
@@ -154,11 +155,66 @@ mod tests {
 	use super::*;
 
 	use std::sync::Arc;
-	use ethkey::{Random, Generator};
+	use ethkey::{Random, Generator, KeyPair};
 	use pool::tests::tx::{Tx, TxExt};
 	use txpool::Scoring;
 	use txpool::scoring::Choice::*;
 
+	fn local_tx_verified(tx: Tx, keypair: &KeyPair) -> VerifiedTransaction {
+		let mut verified_tx = tx.unsigned().sign(keypair.secret(), None).verified();
+		verified_tx.priority = ::pool::Priority::Local;
+		verified_tx
+	}
+
+	#[test]
+	fn should_always_accept_local_transactions_unless_same_sender_and_nonce() {
+		let scoring = NonceAndGasPrice(PrioritizationStrategy::GasPriceOnly);
+
+		// same sender txs
+		let keypair = Random.generate().unwrap();
+
+		let same_sender_tx1 = local_tx_verified(Tx {
+			nonce: 1,
+			gas_price: 1,
+			..Default::default()
+		}, &keypair);
+
+		let same_sender_tx2 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 100,
+			..Default::default()
+		}, &keypair);
+
+		let same_sender_tx3 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 200,
+			..Default::default()
+		}, &keypair);
+
+		// different sender txs
+		let different_sender_tx1 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 1,
+			..Default::default()
+		}, &Random.generate().unwrap());
+
+		let different_sender_tx2 = local_tx_verified(Tx {
+			nonce: 1,
+			gas_price: 10,
+			..Default::default()
+		}, &Random.generate().unwrap());
+
+		assert_eq!(scoring.should_replace(&same_sender_tx1, &same_sender_tx2), InsertNew);
+		assert_eq!(scoring.should_replace(&same_sender_tx2, &same_sender_tx1), InsertNew);
+
+		assert_eq!(scoring.should_replace(&different_sender_tx1, &different_sender_tx2), InsertNew);
+		assert_eq!(scoring.should_replace(&different_sender_tx2, &different_sender_tx1), InsertNew);
+
+		// txs with same sender and nonce
+		assert_eq!(scoring.should_replace(&same_sender_tx2, &same_sender_tx3), ReplaceOld);
+		assert_eq!(scoring.should_replace(&same_sender_tx3, &same_sender_tx2), RejectNew);
+	}
+
 	#[test]
 	fn should_replace_same_sender_by_nonce() {
 		let scoring = NonceAndGasPrice(PrioritizationStrategy::GasPriceOnly);

scripts/gitlab/build-unix.sh

@@ -11,6 +11,9 @@ echo "CC:               " $CC
 echo "CXX:              " $CXX
 #strip ON
 export RUSTFLAGS=" -C link-arg=-s"
+# Linker for crosscomile
+echo "_____ Linker _____"
+cat .cargo/config
 
 echo "_____ Building target: "$CARGO_TARGET" _____"
 if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
@@ -25,6 +28,7 @@ else
 fi
 
 echo "_____ Post-processing binaries _____"
+rm -rf artifacts/*
 mkdir -p artifacts/$CARGO_TARGET
 cd artifacts/$CARGO_TARGET
 
@@ -42,11 +46,6 @@ fi
 echo "_____ Calculating checksums _____"
 for binary in $(ls)
 do
-  rhash --sha256 $binary -o $binary.sha256
+  rhash --sha256 $binary -o $binary.sha256 #do we still need this hash (SHA2)?
-  if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
+  rhash --sha3-256 $binary -o $binary.sha3
-  then
-    echo "> ${binary} cannot be hashed with cross-compiled binary"
-  else
-    ./parity tools hash $binary > $binary.sha3
-  fi
 done

scripts/gitlab/publish-snap.sh

@@ -8,7 +8,10 @@ set -u # treat unset variables as error
 # gsub(/ /, "", $2) deletes whitespaces
 TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml`
 echo Track is: $TRACK
-
+# prepare variables
+VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
+SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
+CARGO_TARGET="$(ls artifacts)"
 # Choose snap release channel based on parity ethereum version track
 case ${TRACK} in
   nightly) export GRADE="devel" CHANNEL="edge";;
@@ -21,24 +24,16 @@ esac
 case ${CI_COMMIT_REF_NAME} in
   beta|stable) export GRADE="stable" CHANNEL="candidate";;
 esac
-
-VERSION="v"$VERSION
-SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
-
 echo "__________Create snap package__________"
 echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
-echo $VERSION:$GRADE:$BUILD_ARCH
+echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET
-# cat scripts/snap/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml
+
-# a bit more necromancy (substitutions):
-pwd
-cd /builds/$CI_PROJECT_PATH/scripts/snap/
 sed -e 's/$VERSION/'"$VERSION"'/g' \
     -e 's/$GRADE/'"$GRADE"'/g' \
     -e 's/$BUILD_ARCH/'"$BUILD_ARCH"'/g' \
     -e 's/$CARGO_TARGET/'"$CARGO_TARGET"'/g' \
-    snapcraft.template.yaml > /builds/$CI_PROJECT_PATH/snapcraft.yaml
+    scripts/snap/snapcraft.template.yaml > snapcraft.yaml
-cd /builds/$CI_PROJECT_PATH
+
-pwd
 apt update
 apt install -y --no-install-recommends rhash
 cat snapcraft.yaml

scripts/snap/snapcraft.template.yaml

@@ -50,4 +50,4 @@ parts:
       cp -v ethkey $SNAPCRAFT_PART_INSTALL/usr/bin/ethkey
       cp -v ethstore $SNAPCRAFT_PART_INSTALL/usr/bin/ethstore
       cp -v whisper $SNAPCRAFT_PART_INSTALL/usr/bin/whisper
-    stage-packages: [libc6, libudev1, libstdc++6, cmake, libdb5.3]
+    stage-packages: [libudev1, libstdc++6, cmake, libdb5.3]