Backports for Stable 2.3.5 (#10414)

* no-git for publish jobs, empty artifacts dir (#10393) * no-git for publish jobs, empty artifacts dir * fix syntax * prettiness * fix prettiness * should get rid of git in publishing * snap: reenable i386, arm64, armhf architecture publishing (#10386) * snap: reenable i386, arm64, armhf architecture publishing * gitlab: fix indent * gitlab: fix yml syntax * Linker for crosscomile * fix target to linker * new docker image * fix lint, add build to this PR * calc SHA3 using rhash * add new images for i386, armhf * show snap target & artifacts * set CARGO_TARGET for publish snap * move detect Version to publish snap * rm libc6 dep from snap-template up pub-snap script * clean up cargo config before add linker * move linker config to docker images * tx pool: always accept local transactions (#10375) * tx pool: always accept local transactions * tx pool: `choose` local txs with same sender and nonce * Fix to_pod storage trie value decoding (#10368)
2019-02-25 15:24:44 +01:00 · 2019-02-25 15:24:44 +01:00 · 0d63c932af
commit 0d63c932af
parent 0b78a1b5a0
7 changed files with 135 additions and 51 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -2,11 +2,9 @@ stages:
  - test
  - build
  - publish
-  - publish-onchain
  - optional

-image:                             parity/rust:gitlab-ci
-
+image:                             parity/rust-parity-ethereum-build:xenial
 variables:
  GIT_STRATEGY:                    fetch
  GIT_SUBMODULE_STRATEGY:          recursive
@ -14,6 +12,12 @@ variables:
  CARGO_HOME:                      "${CI_PROJECT_DIR}/.cargo"
  CARGO_TARGET:                    x86_64-unknown-linux-gnu

+.no_git:                           &no_git
+  variables:
+    GIT_STRATEGY:                  none
+    GIT_SUBMODULE_STRATEGY:        none
+
+
 .releaseable_branches:             # list of git refs for building GitLab artifacts (think "pre-release binaries")
  only:                            &releaseable_branches
    - stable
@ -30,14 +34,6 @@ variables:
    paths:
    - artifacts/

-.determine_version:                &determine_version
-  - VERSION="$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
-  - DATE_STR="$(date +%Y%m%d)"
-  - ID_SHORT="$(echo ${CI_COMMIT_SHA} | cut -c 1-7)"
-  - test "${SCHEDULE_TAG:-${CI_COMMIT_REF_NAME}}" = "nightly" && VERSION="${VERSION}-${ID_SHORT}-${DATE_STR}"
-  - export VERSION
-  - echo "Version = ${VERSION}"
-
 test-linux:
  stage:                           test
  variables:
@ -57,7 +53,7 @@ test-audit:
  tags:
    - linux-docker

-build-linux:
+build-linux:                       &build-linux
  stage:                           build
  only:                            *releaseable_branches
  script:
@ -67,6 +63,24 @@ build-linux:
  tags:
    - linux-docker

+build-linux-i386:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:i386
+  variables:
+    CARGO_TARGET:                  i686-unknown-linux-gnu
+
+build-linux-arm64:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:arm64
+  variables:
+    CARGO_TARGET:                  aarch64-unknown-linux-gnu
+
+build-linux-armhf:
+  <<:                              *build-linux
+  image:                           parity/rust-parity-ethereum-build:armhf
+  variables:
+    CARGO_TARGET:                  armv7-unknown-linux-gnueabihf
+
 build-darwin:
  stage:                           build
  only:                            *releaseable_branches
@ -94,6 +108,7 @@ build-windows:
 publish-docker:
  stage:                           publish
  only:                            *releaseable_branches
+  <<:                              *no_git
  cache: {}
  dependencies:
    - build-linux
@ -102,35 +117,54 @@ publish-docker:
  script:
    - scripts/gitlab/publish-docker.sh parity

-publish-snap:
-  stage:                           optional #publish
+publish-snap:                      &publish-snap
+  stage:                           publish
  only:                            *releaseable_branches
+  # <<:                              *no_git
  image:                           snapcore/snapcraft
  variables:
-    BUILD_ARCH:                  amd64
+    BUILD_ARCH:                    amd64
  cache: {}
-  before_script:                   *determine_version
  dependencies:
    - build-linux
  tags:
-    - rust-stable
+    - linux-docker
  script:
    - scripts/gitlab/publish-snap.sh
-  allow_failure:                   true
  <<:                              *collect_artifacts

-publish-onnet-update:
-  stage:                           publish-onchain
+publish-snap-i386:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    i386
+  dependencies:
+    - build-linux-i386
+
+publish-snap-arm64:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    arm64
+  dependencies:
+    - build-linux-arm64
+
+publish-snap-armhf:
+  <<:                              *publish-snap
+  variables:
+    BUILD_ARCH:                    armhf
+  dependencies:
+    - build-linux-armhf
+
+publish-onchain:
+  stage:                           publish
  only:                            *releaseable_branches
+  <<:                              *no_git
  cache:                           {}
  dependencies:
    - build-linux
    - build-darwin
    - build-windows
-    - publish-awss3-release
-  before_script:                   *determine_version
  script:
-    - scripts/gitlab/publish-onnet-update.sh
+    - scripts/gitlab/publish-onchain.sh
  tags:
    - linux-docker

@ -155,6 +189,7 @@ publish-awss3-release:
  image:                           parity/awscli:latest
  stage:                           publish
  only:                            *releaseable_branches
+  <<:                              *no_git
  cache:                           {}
  dependencies:
    - build-linux
@ -203,4 +238,3 @@ build-android:
    - linux-docker
  allow_failure:                   true
  <<:                              *collect_artifacts
-
--- a/ethcore/src/state/mod.rs
+++ b/ethcore/src/state/mod.rs
@ -1017,7 +1017,7 @@ impl<B: Backend> State<B> {
 		let trie = self.factories.trie.readonly(accountdb.as_hashdb(), &root)?;
 		for o_kv in trie.iter()? {
 			if let Ok((key, val)) = o_kv {
-				pod_storage.insert(key[..].into(), U256::from(&val[..]).into());
+				pod_storage.insert(key[..].into(), rlp::decode::<U256>(&val[..]).expect("Decoded from trie which was encoded from the same type; qed").into());
 			}
 		}

--- a/miner/src/pool/scoring.rs
+++ b/miner/src/pool/scoring.rs
@ -123,15 +123,16 @@ impl<P> txpool::Scoring<P> for NonceAndGasPrice where P: ScoredTransaction + txp
 	}

 	fn should_replace(&self, old: &P, new: &P) -> scoring::Choice {
+		let both_local = old.priority().is_local() && new.priority().is_local();
 		if old.sender() == new.sender() {
 			// prefer earliest transaction
 			match new.nonce().cmp(&old.nonce()) {
+				cmp::Ordering::Equal => self.choose(old, new),
+				_ if both_local => scoring::Choice::InsertNew,
 				cmp::Ordering::Less => scoring::Choice::ReplaceOld,
 				cmp::Ordering::Greater => scoring::Choice::RejectNew,
-				cmp::Ordering::Equal => self.choose(old, new),
 			}
-		} else if old.priority().is_local() && new.priority().is_local() {
-			// accept local transactions over the limit
+		} else if both_local {
 			scoring::Choice::InsertNew
 		} else {
 			let old_score = (old.priority(), old.gas_price());
@ -141,7 +142,7 @@ impl<P> txpool::Scoring<P> for NonceAndGasPrice where P: ScoredTransaction + txp
 			} else {
 				scoring::Choice::RejectNew
 			}
-	 	}
+		}
 	}

 	fn should_ignore_sender_limit(&self, new: &P) -> bool {
@ -154,11 +155,66 @@ mod tests {
 	use super::*;

 	use std::sync::Arc;
-	use ethkey::{Random, Generator};
+	use ethkey::{Random, Generator, KeyPair};
 	use pool::tests::tx::{Tx, TxExt};
 	use txpool::Scoring;
 	use txpool::scoring::Choice::*;

+	fn local_tx_verified(tx: Tx, keypair: &KeyPair) -> VerifiedTransaction {
+		let mut verified_tx = tx.unsigned().sign(keypair.secret(), None).verified();
+		verified_tx.priority = ::pool::Priority::Local;
+		verified_tx
+	}
+
+	#[test]
+	fn should_always_accept_local_transactions_unless_same_sender_and_nonce() {
+		let scoring = NonceAndGasPrice(PrioritizationStrategy::GasPriceOnly);
+
+		// same sender txs
+		let keypair = Random.generate().unwrap();
+
+		let same_sender_tx1 = local_tx_verified(Tx {
+			nonce: 1,
+			gas_price: 1,
+			..Default::default()
+		}, &keypair);
+
+		let same_sender_tx2 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 100,
+			..Default::default()
+		}, &keypair);
+
+		let same_sender_tx3 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 200,
+			..Default::default()
+		}, &keypair);
+
+		// different sender txs
+		let different_sender_tx1 = local_tx_verified(Tx {
+			nonce: 2,
+			gas_price: 1,
+			..Default::default()
+		}, &Random.generate().unwrap());
+
+		let different_sender_tx2 = local_tx_verified(Tx {
+			nonce: 1,
+			gas_price: 10,
+			..Default::default()
+		}, &Random.generate().unwrap());
+
+		assert_eq!(scoring.should_replace(&same_sender_tx1, &same_sender_tx2), InsertNew);
+		assert_eq!(scoring.should_replace(&same_sender_tx2, &same_sender_tx1), InsertNew);
+
+		assert_eq!(scoring.should_replace(&different_sender_tx1, &different_sender_tx2), InsertNew);
+		assert_eq!(scoring.should_replace(&different_sender_tx2, &different_sender_tx1), InsertNew);
+
+		// txs with same sender and nonce
+		assert_eq!(scoring.should_replace(&same_sender_tx2, &same_sender_tx3), ReplaceOld);
+		assert_eq!(scoring.should_replace(&same_sender_tx3, &same_sender_tx2), RejectNew);
+	}
+
 	#[test]
 	fn should_replace_same_sender_by_nonce() {
 		let scoring = NonceAndGasPrice(PrioritizationStrategy::GasPriceOnly);
--- a/scripts/gitlab/build-unix.sh
+++ b/scripts/gitlab/build-unix.sh
@ -11,6 +11,9 @@ echo "CC:               " $CC
 echo "CXX:              " $CXX
 #strip ON
 export RUSTFLAGS=" -C link-arg=-s"
+# Linker for crosscomile
+echo "_____ Linker _____"
+cat .cargo/config

 echo "_____ Building target: "$CARGO_TARGET" _____"
 if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
@ -25,6 +28,7 @@ else
 fi

 echo "_____ Post-processing binaries _____"
+rm -rf artifacts/*
 mkdir -p artifacts/$CARGO_TARGET
 cd artifacts/$CARGO_TARGET

@ -42,11 +46,6 @@ fi
 echo "_____ Calculating checksums _____"
 for binary in $(ls)
 do
-  rhash --sha256 $binary -o $binary.sha256
-  if [ "${CARGO_TARGET}" = "armv7-linux-androideabi" ]
-  then
-    echo "> ${binary} cannot be hashed with cross-compiled binary"
-  else
-    ./parity tools hash $binary > $binary.sha3
-  fi
+  rhash --sha256 $binary -o $binary.sha256 #do we still need this hash (SHA2)?
+  rhash --sha3-256 $binary -o $binary.sha3
 done
--- a/scripts/gitlab/publish-onnet-update.sh
+++ b/scripts/gitlab/publish-onnet-update.sh
--- a/scripts/gitlab/publish-snap.sh
+++ b/scripts/gitlab/publish-snap.sh
@ -8,7 +8,10 @@ set -u # treat unset variables as error
 # gsub(/ /, "", $2) deletes whitespaces
 TRACK=`awk -F '=' '/^track/ {gsub(/"/, "", $2); gsub(/ /, "", $2); print $2}' ./util/version/Cargo.toml`
 echo Track is: $TRACK
-
+# prepare variables
+VERSION=v"$(sed -r -n '1,/^version/s/^version = "([^"]+)".*$/\1/p' Cargo.toml)"
+SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
+CARGO_TARGET="$(ls artifacts)"
 # Choose snap release channel based on parity ethereum version track
 case ${TRACK} in
  nightly) export GRADE="devel" CHANNEL="edge";;
@ -21,24 +24,16 @@ esac
 case ${CI_COMMIT_REF_NAME} in
  beta|stable) export GRADE="stable" CHANNEL="candidate";;
 esac
-
-VERSION="v"$VERSION
-SNAP_PACKAGE="parity_"$VERSION"_"$BUILD_ARCH".snap"
-
 echo "__________Create snap package__________"
 echo "Release channel :" $GRADE " Branch/tag: " $CI_COMMIT_REF_NAME
-echo $VERSION:$GRADE:$BUILD_ARCH
-# cat scripts/snap/snapcraft.template.yaml | envsubst '$VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET' > snapcraft.yaml
-# a bit more necromancy (substitutions):
-pwd
-cd /builds/$CI_PROJECT_PATH/scripts/snap/
+echo $VERSION:$GRADE:$BUILD_ARCH:$CARGO_TARGET
+
 sed -e 's/$VERSION/'"$VERSION"'/g' \
    -e 's/$GRADE/'"$GRADE"'/g' \
    -e 's/$BUILD_ARCH/'"$BUILD_ARCH"'/g' \
    -e 's/$CARGO_TARGET/'"$CARGO_TARGET"'/g' \
-    snapcraft.template.yaml > /builds/$CI_PROJECT_PATH/snapcraft.yaml
-cd /builds/$CI_PROJECT_PATH
-pwd
+    scripts/snap/snapcraft.template.yaml > snapcraft.yaml
+
 apt update
 apt install -y --no-install-recommends rhash
 cat snapcraft.yaml
--- a/scripts/snap/snapcraft.template.yaml
+++ b/scripts/snap/snapcraft.template.yaml
@ -50,4 +50,4 @@ parts:
      cp -v ethkey $SNAPCRAFT_PART_INSTALL/usr/bin/ethkey
      cp -v ethstore $SNAPCRAFT_PART_INSTALL/usr/bin/ethstore
      cp -v whisper $SNAPCRAFT_PART_INSTALL/usr/bin/whisper
-    stage-packages: [libc6, libudev1, libstdc++6, cmake, libdb5.3]
+    stage-packages: [libudev1, libstdc++6, cmake, libdb5.3]