Backports for stable 2.1.7 (#9975)

* version: bump stable to 2.1.7

* Adjust requests costs for light client (#9925)

* PIP Table Cost relative to average peers instead of max peers

* Add tracing in PIP new_cost_table

* Update stat peer_count

* Use number of leeching peers for Light serve costs

* Fix test::light_params_load_share_depends_on_max_peers (wrong type)

* Remove (now) useless test

* Remove `load_share` from LightParams.Config
Prevent div. by 0

* Add LEECHER_COUNT_FACTOR

* PR Grumble: u64 to u32 for f64 casting

* Prevent u32 overflow for avg_peer_count

* Add tests for LightSync::Statistics

* Fix empty steps (#9939)

* Don't send empty step twice or empty step then block.

* Perform basic validation of locally sealed blocks.

* Don't include empty step twice.

* prevent silent errors in daemon mode, closes #9367 (#9946)

* Fix light client informant while syncing (#9932)

* Add `is_idle` to LightSync to check importing status

* Use SyncStateWrapper to make sure is_idle gets updates

* Update is_major_import to use verified queue size as well

* Add comment for `is_idle`

* Add Debug to `SyncStateWrapper`

* `fn get` -> `fn into_inner`

*  ci: rearrange pipeline by logic (#9970)

* ci: rearrange pipeline by logic

* ci: rename docs script

* Add readiness check for docker container (#9804)

* Update Dockerfile

Since parity is built for "mission critical use", I thought other operators may see the need for this.

Adding the `curl` and `jq` commands allows for an extremely simple health check to be usable in container orchestrators.

For example. Here is a health check for a parity docker container running in Kubernetes.

This can be setup as a readiness Probe that would prevent clustered nodes that aren't ready from serving traffic.

```bash
#!/bin/bash

ETH_SYNCING=$(curl -X POST --data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' http://localhost:8545 -H 'Content-Type: application/json')
RESULT=$(echo "$ETH_SYNCING | jq -r .result)

if [ "$RESULT" == "false" ]; then
  echo "Parity is ready to start accepting traffic"
  exit 0
else
  echo "Parity is still syncing the blockchain"
  exit 1
fi
```

* add sync check script

* Fix docker script (#9854)


* Dockerfile: change source path of the newly added check_sync.sh (#9869)

* Do not use the home directory as the working dir in docker (#9834)

* Do not create a home directory.

* Re-add -m flag

* fix docker build (#9971)

* bump smallvec to 0.6 in ethcore-light, ethstore and whisper (#9588)

* bump smallvec to 0.6 in ethcore-light, ethstore and whisper

* bump transaction-pool

* Fix test.

* patch cargo to use tokio-proto from git repo

this makes sure we no longer depend on smallvec 0.2.1 which is
affected by https://github.com/servo/rust-smallvec/issues/96

* use patched version of untrusted 0.5.1

* ci: allow audit to fail

.gitlab-ci.yml

@@ -43,6 +43,14 @@ test-linux:
   tags:
     - rust-stable
 
+test-audit:
+  stage:                           test
+  script:
+    - scripts/gitlab/cargo-audit.sh
+  tags:
+    - rust-stable
+  allow_failure:                   true
+
 build-linux:
   stage:                           build
   only:                            *releaseable_branches
@@ -103,25 +111,18 @@ publish-awss3:
   tags:
     - shell
 
-docs-jsonrpc:
+publish-docs:
-  stage:                            optional
+  stage:                            publish
   only:
     - tags
   except:
     - nightly
   cache: {}
   script:
-    - scripts/gitlab/docs-jsonrpc.sh
+    - scripts/gitlab/publish-docs.sh
   tags:
     - shell
 
-cargo-audit:
-  stage:                           optional
-  script:
-    - scripts/gitlab/cargo-audit.sh
-  tags:
-    - rust-stable
-
 build-android:
   stage:                           optional
   image:                           parity/rust-android:gitlab-ci
@@ -131,6 +132,7 @@ build-android:
     - scripts/gitlab/build-unix.sh
   tags:
     - rust-arm
+  allow_failure:                   true
 
 test-beta:
   stage:                           optional
@@ -140,6 +142,7 @@ test-beta:
     - scripts/gitlab/test-all.sh beta
   tags:
     - rust-beta
+  allow_failure:                   true
 
 test-nightly:
   stage:                           optional
@@ -149,3 +152,4 @@ test-nightly:
     - scripts/gitlab/test-all.sh nightly
   tags:
     - rust-nightly
+  allow_failure:                   true

Cargo.lock

@@ -642,7 +642,7 @@ dependencies = [
  "rlp_derive 0.1.0",
  "serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "stats 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
@@ -689,7 +689,7 @@ dependencies = [
  "rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "transaction-pool 1.13.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "transaction-pool 1.13.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -779,7 +779,7 @@ dependencies = [
  "serde_derive 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "transaction-pool 1.13.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "transaction-pool 1.13.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -817,7 +817,7 @@ dependencies = [
  "tokio 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-proto 0.1.1 (git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e)",
  "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -889,7 +889,6 @@ dependencies = [
  "rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "triehash-ethereum 0.2.0",
 ]
@@ -996,7 +995,7 @@ dependencies = [
  "serde 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1289,7 +1288,7 @@ dependencies = [
  "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-proto 0.1.1 (git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e)",
  "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "want 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1306,7 +1305,7 @@ dependencies = [
  "rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-proto 0.1.1 (git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e)",
  "tokio-rustls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "webpki-roots 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2008,7 +2007,7 @@ version = "1.12.0"
 dependencies = [
  "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "panic_hook 0.1.0",
- "parity-ethereum 2.1.6",
+ "parity-ethereum 2.1.7",
 ]
 
 [[package]]
@@ -2024,7 +2023,7 @@ dependencies = [
 
 [[package]]
 name = "parity-ethereum"
-version = "2.1.6"
+version = "2.1.7"
 dependencies = [
  "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2074,7 +2073,7 @@ dependencies = [
  "parity-rpc 1.12.0",
  "parity-rpc-client 1.4.0",
  "parity-updater 1.12.0",
- "parity-version 2.1.6",
+ "parity-version 2.1.7",
  "parity-whisper 0.1.0",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2239,7 +2238,7 @@ dependencies = [
  "parity-crypto 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-reactor 0.1.0",
  "parity-updater 1.12.0",
- "parity-version 2.1.6",
+ "parity-version 2.1.7",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "patricia-trie 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2254,7 +2253,7 @@ dependencies = [
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "transaction-pool 1.13.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "transaction-pool 1.13.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "vm 0.1.0",
 ]
@@ -2329,7 +2328,7 @@ dependencies = [
  "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-hash-fetch 1.12.0",
  "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "parity-version 2.1.6",
+ "parity-version 2.1.7",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2339,7 +2338,7 @@ dependencies = [
 
 [[package]]
 name = "parity-version"
-version = "2.1.6"
+version = "2.1.7"
 dependencies = [
  "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2381,7 +2380,7 @@ dependencies = [
  "serde_derive 1.0.78 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2807,7 +2806,7 @@ dependencies = [
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)",
 ]
 
 [[package]]
@@ -2914,7 +2913,7 @@ dependencies = [
  "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "ring 0.12.1 (git+https://github.com/paritytech/ring)",
  "sct 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)",
  "webpki 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2952,7 +2951,7 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "ring 0.12.1 (git+https://github.com/paritytech/ring)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)",
 ]
 
 [[package]]
@@ -3049,18 +3048,10 @@ name = "slab"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "smallvec"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "smallvec"
 version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "smallvec"
@@ -3137,11 +3128,6 @@ dependencies = [
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "take"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "target_info"
 version = "0.1.0"
@@ -3364,15 +3350,12 @@ dependencies = [
 [[package]]
 name = "tokio-proto"
 version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e#56c720ea3c74efa8f39e36c24e609628222b16a1"
 dependencies = [
  "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3414,7 +3397,7 @@ dependencies = [
  "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-proto 0.1.1 (git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e)",
 ]
 
 [[package]]
@@ -3536,12 +3519,12 @@ dependencies = [
 
 [[package]]
 name = "transaction-pool"
-version = "1.13.2"
+version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -3668,7 +3651,7 @@ dependencies = [
 [[package]]
 name = "untrusted"
 version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/paritytech/untrusted?branch=0.5#6e8e92cd31b23272685f340cfde481cb68ac1243"
 
 [[package]]
 name = "url"
@@ -3781,7 +3764,7 @@ version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "ring 0.12.1 (git+https://github.com/paritytech/ring)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)",
 ]
 
 [[package]]
@@ -3789,7 +3772,7 @@ name = "webpki-roots"
 version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)",
  "webpki 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -4135,7 +4118,6 @@ dependencies = [
 "checksum slab 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6dbdd334bd28d328dad1c41b0ea662517883d8880d8533895ef96c8003dec9c4"
 "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23"
 "checksum slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5f9776d6b986f77b35c6cf846c11ad986ff128fe0b2b63a3628e3755e8d3102d"
-"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013"
 "checksum smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f90c5e5fe535e48807ab94fc611d323935f39d4660c52b26b96446a7b33aef10"
 "checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d"
 "checksum socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c4d11a52082057d87cb5caa31ad812f4504b97ab44732cd8359df2e9ff9f48e7"
@@ -4145,7 +4127,6 @@ dependencies = [
 "checksum syn 0.12.15 (registry+https://github.com/rust-lang/crates.io-index)" = "c97c05b8ebc34ddd6b967994d5c6e9852fa92f8b82b3858c39451f97346dcce5"
 "checksum syn 0.13.11 (registry+https://github.com/rust-lang/crates.io-index)" = "14f9bf6292f3a61d2c716723fdb789a41bbe104168e6f496dc6497e531ea1b9b"
 "checksum syn 0.15.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9056ebe7f2d6a38bc63171816fd1d3430da5a43896de21676dc5c0a4b8274a11"
-"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5"
 "checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe"
 "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
 "checksum tempfile 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "11ce2fe9db64b842314052e2421ac61a73ce41b898dc8e3750398b219c5fc1e0"
@@ -4168,7 +4149,7 @@ dependencies = [
 "checksum tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5cbe4ca6e71cb0b62a66e4e6f53a8c06a6eefe46cc5f665ad6f274c9906f135"
 "checksum tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6cc2de7725863c86ac71b0b9068476fec50834f055a243558ef1655bbd34cb"
 "checksum tokio-named-pipes 0.1.0 (git+https://github.com/nikvolf/tokio-named-pipes)" = "<none>"
-"checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389"
+"checksum tokio-proto 0.1.1 (git+https://github.com/tokio-rs/tokio-proto.git?rev=56c720e)" = "<none>"
 "checksum tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4bfbaf9f260635649ec26b6fb4aded03887295ffcd999f6e43fd2c4758f758ea"
 "checksum tokio-retry 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f05746ae87dca83a2016b4f5dba5b237b897dd12fd324f60afe282112f16969a"
 "checksum tokio-rustls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9263e472d976e4345e50c6cce4cfe6b17c71593ea593cce1df26f1efd36debb"
@@ -4182,7 +4163,7 @@ dependencies = [
 "checksum tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "424c1ed15a0132251813ccea50640b224c809d6ceafb88154c1a8775873a0e89"
 "checksum toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a0263c6c02c4db6c8f7681f9fd35e90de799ebd4cfdeab77a38f4ff6b3d8c0d9"
 "checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9"
-"checksum transaction-pool 1.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fdb8870eea404a57e2f62056ac45067a53a6207fd31866122356481d3c2e1a30"
+"checksum transaction-pool 1.13.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e5866e5126b14358f1d7af4bf51a0be677a363799b90e655edcec8254edef1d2"
 "checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e"
 "checksum trezor-sys 1.0.0 (git+https://github.com/paritytech/trezor-sys)" = "<none>"
 "checksum trie-standardmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0e26f52976a57a0859616d6fcec87092ac35d08eabbd78dc3dabee93b480ea5f"
@@ -4198,7 +4179,7 @@ dependencies = [
 "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
-"checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae"
+"checksum untrusted 0.5.1 (git+https://github.com/paritytech/untrusted?branch=0.5)" = "<none>"
 "checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6"
 "checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4"
 "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"

Cargo.toml

@@ -2,7 +2,7 @@
 description = "Parity Ethereum client"
 name = "parity-ethereum"
 # NOTE Make sure to update util/version/Cargo.toml as well
-version = "2.1.6"
+version = "2.1.7"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 
@@ -140,3 +140,5 @@ members = [
 
 [patch.crates-io]
 ring = { git = "https://github.com/paritytech/ring" }
+tokio-proto = { git = "https://github.com/tokio-rs/tokio-proto.git", rev = "56c720e" }
+untrusted = { git = "https://github.com/paritytech/untrusted", branch = "0.5" }

ethcore/light/Cargo.toml

@@ -23,7 +23,7 @@ vm = { path = "../vm" }
 fastmap = { path = "../../util/fastmap" }
 rlp = { version = "0.2.4", features = ["ethereum"] }
 rlp_derive = { path = "../../util/rlp_derive" }
-smallvec = "0.4"
+smallvec = "0.6"
 futures = "0.1"
 rand = "0.4"
 itertools = "0.5"

ethcore/light/src/net/mod.rs

@@ -28,7 +28,7 @@ use parking_lot::{Mutex, RwLock};
 use provider::Provider;
 use request::{Request, NetworkRequests as Requests, Response};
 use rlp::{RlpStream, Rlp};
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::fmt;
 use std::ops::{BitOr, BitAnd, Not};
 use std::sync::Arc;
@@ -38,7 +38,7 @@ use std::time::{Duration, Instant};
 use self::request_credits::{Credits, FlowParams};
 use self::context::{Ctx, TickCtx};
 use self::error::Punishment;
-use self::load_timer::{LoadDistribution, NullStore};
+use self::load_timer::{LoadDistribution, NullStore, MOVING_SAMPLE_SIZE};
 use self::request_set::RequestSet;
 use self::id_guard::IdGuard;
 
@@ -70,6 +70,16 @@ const PROPAGATE_TIMEOUT_INTERVAL: Duration = Duration::from_secs(5);
 const RECALCULATE_COSTS_TIMEOUT: TimerToken = 3;
 const RECALCULATE_COSTS_INTERVAL: Duration = Duration::from_secs(60 * 60);
 
+const STATISTICS_TIMEOUT: TimerToken = 4;
+const STATISTICS_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Maximum load share for the light server
+pub const MAX_LIGHTSERV_LOAD: f64 = 0.5;
+
+/// Factor to multiply leecher count to cater for
+/// extra sudden connections (should be >= 1.0)
+pub const LEECHER_COUNT_FACTOR: f64 = 1.25;
+
 // minimum interval between updates.
 const UPDATE_INTERVAL: Duration = Duration::from_millis(5000);
 
@@ -256,18 +266,18 @@ pub trait Handler: Send + Sync {
 pub struct Config {
 	/// How many stored seconds of credits peers should be able to accumulate.
 	pub max_stored_seconds: u64,
-	/// How much of the total load capacity each peer should be allowed to take.
+	/// The network config median peers (used as default peer count)
-	pub load_share: f64,
+	pub median_peers: f64,
 }
 
 impl Default for Config {
 	fn default() -> Self {
-		const LOAD_SHARE: f64 = 1.0 / 25.0;
+		const MEDIAN_PEERS: f64 = 25.0;
 		const MAX_ACCUMULATED: u64 = 60 * 5; // only charge for 5 minutes.
 
 		Config {
 			max_stored_seconds: MAX_ACCUMULATED,
-			load_share: LOAD_SHARE,
+			median_peers: MEDIAN_PEERS,
 		}
 	}
 }
@@ -335,6 +345,42 @@ mod id_guard {
 	}
 }
 
+/// Provides various statistics that could
+/// be used to compute costs
+pub struct Statistics {
+	/// Samples of peer count
+	peer_counts: VecDeque<usize>,
+}
+
+impl Statistics {
+	/// Create a new Statistics instance
+	pub fn new() -> Self {
+		Statistics {
+			peer_counts: VecDeque::with_capacity(MOVING_SAMPLE_SIZE),
+		}
+	}
+
+	/// Add a new peer_count sample
+	pub fn add_peer_count(&mut self, peer_count: usize) {
+		while self.peer_counts.len() >= MOVING_SAMPLE_SIZE {
+			self.peer_counts.pop_front();
+		}
+		self.peer_counts.push_back(peer_count);
+	}
+
+	/// Get the average peer count from previous samples. Is always >= 1.0
+	pub fn avg_peer_count(&self) -> f64 {
+		let len = self.peer_counts.len();
+		if len == 0 {
+			return 1.0;
+		}
+		let avg = self.peer_counts.iter()
+			.fold(0, |sum: u32, &v| sum.saturating_add(v as u32)) as f64
+			/ len as f64;
+		avg.max(1.0)
+	}
+}
+
 /// This is an implementation of the light ethereum network protocol, abstracted
 /// over a `Provider` of data and a p2p network.
 ///
@@ -359,6 +405,7 @@ pub struct LightProtocol {
 	req_id: AtomicUsize,
 	sample_store: Box<SampleStore>,
 	load_distribution: LoadDistribution,
+	statistics: RwLock<Statistics>,
 }
 
 impl LightProtocol {
@@ -369,9 +416,11 @@ impl LightProtocol {
 		let genesis_hash = provider.chain_info().genesis_hash;
 		let sample_store = params.sample_store.unwrap_or_else(|| Box::new(NullStore));
 		let load_distribution = LoadDistribution::load(&*sample_store);
+		// Default load share relative to median peers
+		let load_share = MAX_LIGHTSERV_LOAD / params.config.median_peers;
 		let flow_params = FlowParams::from_request_times(
 			|kind| load_distribution.expected_time(kind),
-			params.config.load_share,
+			load_share,
 			Duration::from_secs(params.config.max_stored_seconds),
 		);
 
@@ -389,6 +438,7 @@ impl LightProtocol {
 			req_id: AtomicUsize::new(0),
 			sample_store,
 			load_distribution,
+			statistics: RwLock::new(Statistics::new()),
 		}
 	}
 
@@ -408,6 +458,16 @@ impl LightProtocol {
 		)
 	}
 
+	/// Get the number of active light peers downloading from the
+	/// node
+	pub fn leecher_count(&self) -> usize {
+		let credit_limit = *self.flow_params.read().limit();
+		// Count the number of peers that used some credit
+		self.peers.read().iter()
+			.filter(|(_, p)| p.lock().local_credits.current() < credit_limit)
+			.count()
+	}
+
 	/// Make a request to a peer.
 	///
 	/// Fails on: nonexistent peer, network error, peer not server,
@@ -772,12 +832,16 @@ impl LightProtocol {
 	fn begin_new_cost_period(&self, io: &IoContext) {
 		self.load_distribution.end_period(&*self.sample_store);
 
+		let avg_peer_count = self.statistics.read().avg_peer_count();
+		// Load share relative to average peer count +LEECHER_COUNT_FACTOR%
+		let load_share = MAX_LIGHTSERV_LOAD / (avg_peer_count * LEECHER_COUNT_FACTOR);
 		let new_params = Arc::new(FlowParams::from_request_times(
 			|kind| self.load_distribution.expected_time(kind),
-			self.config.load_share,
+			load_share,
 			Duration::from_secs(self.config.max_stored_seconds),
 		));
 		*self.flow_params.write() = new_params.clone();
+		trace!(target: "pip", "New cost period: avg_peers={} ; cost_table:{:?}", avg_peer_count, new_params.cost_table());
 
 		let peers = self.peers.read();
 		let now = Instant::now();
@@ -797,6 +861,11 @@ impl LightProtocol {
 			peer_info.awaiting_acknowledge = Some((now, new_params.clone()));
 		}
 	}
+
+	fn tick_statistics(&self) {
+		let leecher_count = self.leecher_count();
+		self.statistics.write().add_peer_count(leecher_count);
+	}
 }
 
 impl LightProtocol {
@@ -1099,6 +1168,8 @@ impl NetworkProtocolHandler for LightProtocol {
 			.expect("Error registering sync timer.");
 		io.register_timer(RECALCULATE_COSTS_TIMEOUT, RECALCULATE_COSTS_INTERVAL)
 			.expect("Error registering request timer interval token.");
+		io.register_timer(STATISTICS_TIMEOUT, STATISTICS_INTERVAL)
+			.expect("Error registering statistics timer.");
 	}
 
 	fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) {
@@ -1119,6 +1190,7 @@ impl NetworkProtocolHandler for LightProtocol {
 			TICK_TIMEOUT => self.tick_handlers(&io),
 			PROPAGATE_TIMEOUT => self.propagate_transactions(&io),
 			RECALCULATE_COSTS_TIMEOUT => self.begin_new_cost_period(&io),
+			STATISTICS_TIMEOUT => self.tick_statistics(),
 			_ => warn!(target: "pip", "received timeout on unknown token {}", timer),
 		}
 	}

ethcore/light/src/net/tests/mod.rs

@@ -22,9 +22,10 @@ use ethcore::client::{EachBlockWith, TestBlockChainClient};
 use ethcore::encoded;
 use ethcore::ids::BlockId;
 use ethereum_types::{H256, U256, Address};
-use net::{LightProtocol, Params, packet, Peer};
+use net::{LightProtocol, Params, packet, Peer, Statistics};
 use net::context::IoContext;
 use net::status::{Capabilities, Status};
+use net::load_timer::MOVING_SAMPLE_SIZE;
 use network::{PeerId, NodeId};
 use provider::Provider;
 use request;
@@ -780,3 +781,34 @@ fn get_transaction_index() {
 	let expected = Expect::Respond(packet::RESPONSE, response);
 	proto.handle_packet(&expected, 1, packet::REQUEST, &request_body);
 }
+
+#[test]
+fn sync_statistics() {
+	let mut stats = Statistics::new();
+
+	// Empty set should return 1.0
+	assert_eq!(stats.avg_peer_count(), 1.0);
+
+	// Average < 1.0 should return 1.0
+	stats.add_peer_count(0);
+	assert_eq!(stats.avg_peer_count(), 1.0);
+
+	stats = Statistics::new();
+
+	const N: f64 = 50.0;
+
+	for i in 1..(N as usize + 1) {
+		stats.add_peer_count(i);
+	}
+
+	// Compute the average for the sum 1..N
+	assert_eq!(stats.avg_peer_count(), N * (N + 1.0) / 2.0 / N);
+
+	for _ in 1..(MOVING_SAMPLE_SIZE + 1) {
+		stats.add_peer_count(40);
+	}
+
+	// Test that it returns the average of the last
+	// `MOVING_SAMPLE_SIZE` values
+	assert_eq!(stats.avg_peer_count(), 40.0);
+}

ethcore/src/client/client.rs

@@ -2252,24 +2252,33 @@ impl ImportSealedBlock for Client {
 	fn import_sealed_block(&self, block: SealedBlock) -> ImportResult {
 		let h = block.header().hash();
 		let start = Instant::now();
+		let header = block.header().clone();
 		let route = {
+			// Do a super duper basic verification to detect potential bugs
+			if let Err(e) = self.engine.verify_block_basic(&header) {
+				self.importer.bad_blocks.report(
+					block.rlp_bytes(),
+					format!("Detected an issue with locally sealed block: {}", e),
+				);
+				return Err(e.into());
+			}
+
 			// scope for self.import_lock
 			let _import_lock = self.importer.import_lock.lock();
 			trace_time!("import_sealed_block");
 
-			let number = block.header().number();
 			let block_data = block.rlp_bytes();
-			let header = block.header().clone();
 
 			let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self);
-			trace!(target: "client", "Imported sealed block #{} ({})", number, h);
+			trace!(target: "client", "Imported sealed block #{} ({})", header.number(), header.hash());
 			self.state_db.write().sync_cache(&route.enacted, &route.retracted, false);
 			route
 		};
+		let h = header.hash();
 		let route = ChainRoute::from([route].as_ref());
 		self.importer.miner.chain_new_blocks(
 			self,
-			&[h.clone()],
+			&[h],
 			&[],
 			route.enacted(),
 			route.retracted(),
@@ -2277,10 +2286,10 @@ impl ImportSealedBlock for Client {
 		);
 		self.notify(|notify| {
 			notify.new_blocks(
-				vec![h.clone()],
+				vec![h],
 				vec![],
 				route.clone(),
-				vec![h.clone()],
+				vec![h],
 				vec![],
 				start.elapsed(),
 			);

ethcore/src/engines/authority_round/mod.rs

@@ -16,8 +16,8 @@
 
 //! A blockchain engine that supports a non-instant BFT proof-of-authority.
 
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
-use std::fmt;
+use std::{cmp, fmt};
 use std::iter::FromIterator;
 use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
@@ -122,10 +122,10 @@ struct Step {
 }
 
 impl Step {
-	fn load(&self) -> usize { self.inner.load(AtomicOrdering::SeqCst) }
+	fn load(&self) -> u64 { self.inner.load(AtomicOrdering::SeqCst) as u64 }
 	fn duration_remaining(&self) -> Duration {
 		let now = unix_now();
-		let expected_seconds = (self.load() as u64)
+		let expected_seconds = self.load()
 			.checked_add(1)
 			.and_then(|ctr| ctr.checked_mul(self.duration as u64))
 			.map(Duration::from_secs);
@@ -161,8 +161,8 @@ impl Step {
 		}
 	}
 
-	fn check_future(&self, given: usize) -> Result<(), Option<OutOfBounds<u64>>> {
+	fn check_future(&self, given: u64) -> Result<(), Option<OutOfBounds<u64>>> {
-		const REJECTED_STEP_DRIFT: usize = 4;
+		const REJECTED_STEP_DRIFT: u64 = 4;
 
 		// Verify if the step is correct.
 		if given <= self.load() {
@@ -181,8 +181,8 @@ impl Step {
 			let d = self.duration as u64;
 			Err(Some(OutOfBounds {
 				min: None,
-				max: Some(d * current as u64),
+				max: Some(d * current),
-				found: d * given as u64,
+				found: d * given,
 			}))
 		} else {
 			Ok(())
@@ -191,8 +191,8 @@ impl Step {
 }
 
 // Chain scoring: total weight is sqrt(U256::max_value())*height - step
-fn calculate_score(parent_step: U256, current_step: U256, current_empty_steps: U256) -> U256 {
+fn calculate_score(parent_step: u64, current_step: u64, current_empty_steps: usize) -> U256 {
-	U256::from(U128::max_value()) + parent_step - current_step + current_empty_steps
+	U256::from(U128::max_value()) + U256::from(parent_step) - U256::from(current_step) + U256::from(current_empty_steps)
 }
 
 struct EpochManager {
@@ -283,13 +283,26 @@ impl EpochManager {
 /// A message broadcast by authorities when it's their turn to seal a block but there are no
 /// transactions. Other authorities accumulate these messages and later include them in the seal as
 /// proof.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 struct EmptyStep {
 	signature: H520,
-	step: usize,
+	step: u64,
 	parent_hash: H256,
 }
 
+impl PartialOrd for EmptyStep {
+	fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
+		Some(self.cmp(other))
+	}
+}
+impl Ord for EmptyStep {
+	fn cmp(&self, other: &Self) -> cmp::Ordering {
+		self.step.cmp(&other.step)
+			.then_with(|| self.parent_hash.cmp(&other.parent_hash))
+			.then_with(|| self.signature.cmp(&other.signature))
+	}
+}
+
 impl EmptyStep {
 	fn from_sealed(sealed_empty_step: SealedEmptyStep, parent_hash: &H256) -> EmptyStep {
 		let signature = sealed_empty_step.signature;
@@ -352,7 +365,7 @@ pub fn empty_step_full_rlp(signature: &H520, empty_step_rlp: &[u8]) -> Vec<u8> {
 	s.out()
 }
 
-pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec<u8> {
+pub fn empty_step_rlp(step: u64, parent_hash: &H256) -> Vec<u8> {
 	let mut s = RlpStream::new_list(2);
 	s.append(&step).append(parent_hash);
 	s.out()
@@ -364,7 +377,7 @@ pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec<u8> {
 /// empty message is included.
 struct SealedEmptyStep {
 	signature: H520,
-	step: usize,
+	step: u64,
 }
 
 impl Encodable for SealedEmptyStep {
@@ -398,7 +411,7 @@ pub struct AuthorityRound {
 	validators: Box<ValidatorSet>,
 	validate_score_transition: u64,
 	validate_step_transition: u64,
-	empty_steps: Mutex<Vec<EmptyStep>>,
+	empty_steps: Mutex<BTreeSet<EmptyStep>>,
 	epoch_manager: Mutex<EpochManager>,
 	immediate_transitions: bool,
 	block_reward: U256,
@@ -493,7 +506,7 @@ fn header_expected_seal_fields(header: &Header, empty_steps_transition: u64) ->
 	}
 }
 
-fn header_step(header: &Header, empty_steps_transition: u64) -> Result<usize, ::rlp::DecoderError> {
+fn header_step(header: &Header, empty_steps_transition: u64) -> Result<u64, ::rlp::DecoderError> {
 	let expected_seal_fields = header_expected_seal_fields(header, empty_steps_transition);
 	Rlp::new(&header.seal().get(0).expect(
 		&format!("was either checked with verify_block_basic or is genesis; has {} fields; qed (Make sure the spec file has a correct genesis seal)", expected_seal_fields))).as_val()
@@ -532,17 +545,17 @@ fn header_empty_steps_signers(header: &Header, empty_steps_transition: u64) -> R
 	}
 }
 
-fn step_proposer(validators: &ValidatorSet, bh: &H256, step: usize) -> Address {
+fn step_proposer(validators: &ValidatorSet, bh: &H256, step: u64) -> Address {
-	let proposer = validators.get(bh, step);
+	let proposer = validators.get(bh, step as usize);
 	trace!(target: "engine", "Fetched proposer for step {}: {}", step, proposer);
 	proposer
 }
 
-fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: usize, address: &Address) -> bool {
+fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: u64, address: &Address) -> bool {
 	step_proposer(validators, bh, step) == *address
 }
 
-fn verify_timestamp(step: &Step, header_step: usize) -> Result<(), BlockError> {
+fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> {
 	match step.check_future(header_step) {
 		Err(None) => {
 			trace!(target: "engine", "verify_timestamp: block from the future");
@@ -563,7 +576,7 @@ fn verify_external(header: &Header, validators: &ValidatorSet, empty_steps_trans
 	let header_step = header_step(header, empty_steps_transition)?;
 
 	let proposer_signature = header_signature(header, empty_steps_transition)?;
-	let correct_proposer = validators.get(header.parent_hash(), header_step);
+	let correct_proposer = validators.get(header.parent_hash(), header_step as usize);
 	let is_invalid_proposer = *header.author() != correct_proposer || {
 		let empty_steps_rlp = if header.number() >= empty_steps_transition {
 			Some(header_empty_steps_raw(header))
@@ -633,13 +646,13 @@ impl AuthorityRound {
 			panic!("authority_round: step duration can't be zero")
 		}
 		let should_timeout = our_params.start_step.is_none();
-		let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))) as usize;
+		let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64)));
 		let engine = Arc::new(
 			AuthorityRound {
 				transition_service: IoService::<()>::start()?,
 				step: Arc::new(PermissionedStep {
 					inner: Step {
-						inner: AtomicUsize::new(initial_step),
+						inner: AtomicUsize::new(initial_step as usize),
 						calibrate: our_params.start_step.is_none(),
 						duration: our_params.step_duration,
 					},
@@ -650,7 +663,7 @@ impl AuthorityRound {
 				validators: our_params.validators,
 				validate_score_transition: our_params.validate_score_transition,
 				validate_step_transition: our_params.validate_step_transition,
-				empty_steps: Mutex::new(Vec::new()),
+				empty_steps: Default::default(),
 				epoch_manager: Mutex::new(EpochManager::blank()),
 				immediate_transitions: our_params.immediate_transitions,
 				block_reward: our_params.block_reward,
@@ -698,22 +711,41 @@ impl AuthorityRound {
 		})
 	}
 
-	fn empty_steps(&self, from_step: U256, to_step: U256, parent_hash: H256) -> Vec<EmptyStep> {
+	fn empty_steps(&self, from_step: u64, to_step: u64, parent_hash: H256) -> Vec<EmptyStep> {
-		self.empty_steps.lock().iter().filter(|e| {
+		let from = EmptyStep {
-			U256::from(e.step) > from_step &&
+			step: from_step + 1,
-				U256::from(e.step) < to_step &&
+			parent_hash,
-				e.parent_hash == parent_hash
+			signature: Default::default(),
-		}).cloned().collect()
+		};
+		let to = EmptyStep {
+			step: to_step,
+			parent_hash: Default::default(),
+			signature: Default::default(),
+		};
+
+		if from >= to {
+			return vec![];
+		}
+
+		self.empty_steps.lock()
+			.range(from..to)
+			.filter(|e| e.parent_hash == parent_hash)
+			.cloned()
+			.collect()
 	}
 
-	fn clear_empty_steps(&self, step: U256) {
+	fn clear_empty_steps(&self, step: u64) {
 		// clear old `empty_steps` messages
-		self.empty_steps.lock().retain(|e| U256::from(e.step) > step);
+		let mut empty_steps = self.empty_steps.lock();
+		*empty_steps = empty_steps.split_off(&EmptyStep {
+			step: step + 1,
+			parent_hash: Default::default(),
+			signature: Default::default(),
+		});
 	}
 
 	fn handle_empty_step_message(&self, empty_step: EmptyStep) {
-		let mut empty_steps = self.empty_steps.lock();
+		self.empty_steps.lock().insert(empty_step);
-		empty_steps.push(empty_step);
 	}
 
 	fn generate_empty_step(&self, parent_hash: &H256) {
@@ -743,7 +775,7 @@ impl AuthorityRound {
 		}
 	}
 
-	fn report_skipped(&self, header: &Header, current_step: usize, parent_step: usize, validators: &ValidatorSet, set_number: u64) {
+	fn report_skipped(&self, header: &Header, current_step: u64, parent_step: u64, validators: &ValidatorSet, set_number: u64) {
 		// we're building on top of the genesis block so don't report any skipped steps
 		if header.number() == 1 {
 			return;
@@ -873,12 +905,12 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		let current_step = self.step.inner.load();
 
 		let current_empty_steps_len = if header.number() >= self.empty_steps_transition {
-			self.empty_steps(parent_step.into(), current_step.into(), parent.hash()).len()
+			self.empty_steps(parent_step, current_step, parent.hash()).len()
 		} else {
 			0
 		};
 
-		let score = calculate_score(parent_step.into(), current_step.into(), current_empty_steps_len.into());
+		let score = calculate_score(parent_step, current_step, current_empty_steps_len);
 		header.set_difficulty(score);
 	}
 
@@ -922,8 +954,8 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		}
 
 		let header = block.header();
-		let parent_step: U256 = header_step(parent, self.empty_steps_transition)
+		let parent_step = header_step(parent, self.empty_steps_transition)
-			.expect("Header has been verified; qed").into();
+			.expect("Header has been verified; qed");
 
 		let step = self.step.inner.load();
 
@@ -958,7 +990,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		if is_step_proposer(&*validators, header.parent_hash(), step, header.author()) {
 			// this is guarded against by `can_propose` unless the block was signed
 			// on the same step (implies same key) and on a different node.
-			if parent_step == step.into() {
+			if parent_step == step {
 				warn!("Attempted to seal block on the same step as parent. Is this authority sealing with more than one node?");
 				return Seal::None;
 			}
@@ -970,7 +1002,10 @@ impl Engine<EthereumMachine> for AuthorityRound {
 				block.transactions().is_empty() &&
 				empty_steps.len() < self.maximum_empty_steps {
 
-				self.generate_empty_step(header.parent_hash());
+				if self.step.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) {
+					self.generate_empty_step(header.parent_hash());
+				}
+
 				return Seal::None;
 			}
 
@@ -994,7 +1029,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 					// report any skipped primaries between the parent block and
 					// the block we're sealing, unless we have empty steps enabled
 					if header.number() < self.empty_steps_transition {
-						self.report_skipped(header, step, u64::from(parent_step) as usize, &*validators, set_number);
+						self.report_skipped(header, step, parent_step, &*validators, set_number);
 					}
 
 					let mut fields = vec![
@@ -1534,12 +1569,12 @@ mod tests {
 
 		// Two validators.
 		// Spec starts with step 2.
-		header.set_difficulty(calculate_score(U256::from(0), U256::from(2), U256::zero()));
+		header.set_difficulty(calculate_score(0, 2, 0));
 		let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
 		header.set_seal(vec![encode(&2usize).into_vec(), encode(&(&*signature as &[u8])).into_vec()]);
 		assert!(engine.verify_block_family(&header, &parent_header).is_ok());
 		assert!(engine.verify_block_external(&header).is_err());
-		header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero()));
+		header.set_difficulty(calculate_score(0, 1, 0));
 		let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
 		header.set_seal(vec![encode(&1usize).into_vec(), encode(&(&*signature as &[u8])).into_vec()]);
 		assert!(engine.verify_block_family(&header, &parent_header).is_ok());
@@ -1563,7 +1598,7 @@ mod tests {
 
 		// Two validators.
 		// Spec starts with step 2.
-		header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero()));
+		header.set_difficulty(calculate_score(0, 1, 0));
 		let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap();
 		header.set_seal(vec![encode(&1usize).into_vec(), encode(&(&*signature as &[u8])).into_vec()]);
 		assert!(engine.verify_block_family(&header, &parent_header).is_ok());
@@ -1591,10 +1626,10 @@ mod tests {
 		// Two validators.
 		// Spec starts with step 2.
 		header.set_seal(vec![encode(&5usize).into_vec(), encode(&(&*signature as &[u8])).into_vec()]);
-		header.set_difficulty(calculate_score(U256::from(4), U256::from(5), U256::zero()));
+		header.set_difficulty(calculate_score(4, 5, 0));
 		assert!(engine.verify_block_family(&header, &parent_header).is_ok());
 		header.set_seal(vec![encode(&3usize).into_vec(), encode(&(&*signature as &[u8])).into_vec()]);
-		header.set_difficulty(calculate_score(U256::from(4), U256::from(3), U256::zero()));
+		header.set_difficulty(calculate_score(4, 3, 0));
 		assert!(engine.verify_block_family(&header, &parent_header).is_err());
 	}
 
@@ -1628,7 +1663,7 @@ mod tests {
 		parent_header.set_seal(vec![encode(&1usize).into_vec()]);
 		parent_header.set_gas_limit("222222".parse::<U256>().unwrap());
 		let mut header: Header = Header::default();
-		header.set_difficulty(calculate_score(U256::from(1), U256::from(3), U256::zero()));
+		header.set_difficulty(calculate_score(1, 3, 0));
 		header.set_gas_limit("222222".parse::<U256>().unwrap());
 		header.set_seal(vec![encode(&3usize).into_vec()]);
 
@@ -1742,14 +1777,14 @@ mod tests {
 		(spec, tap, accounts)
 	}
 
-	fn empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> EmptyStep {
+	fn empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> EmptyStep {
 		let empty_step_rlp = super::empty_step_rlp(step, parent_hash);
 		let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into();
 		let parent_hash = parent_hash.clone();
 		EmptyStep { step, signature, parent_hash }
 	}
 
-	fn sealed_empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> SealedEmptyStep {
+	fn sealed_empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> SealedEmptyStep {
 		let empty_step_rlp = super::empty_step_rlp(step, parent_hash);
 		let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into();
 		SealedEmptyStep { signature, step }
@@ -1785,6 +1820,11 @@ mod tests {
 
 		// we've received the message
 		assert!(notify.messages.read().contains(&empty_step_rlp));
+		let len = notify.messages.read().len();
+
+		// make sure that we don't generate empty step for the second time
+		assert_eq!(engine.generate_seal(b1.block(), &genesis_header), Seal::None);
+		assert_eq!(len, notify.messages.read().len());
 	}
 
 	#[test]
@@ -1999,7 +2039,7 @@ mod tests {
 		let empty_step3 = sealed_empty_step(engine, 3, &parent_header.hash());
 
 		let empty_steps = vec![empty_step2, empty_step3];
-		header.set_difficulty(calculate_score(U256::from(0), U256::from(4), U256::from(2)));
+		header.set_difficulty(calculate_score(0, 4, 2));
 		let signature = tap.sign(addr1, Some("1".into()), header.bare_hash()).unwrap();
 		header.set_seal(vec![
 			encode(&4usize).into_vec(),
@@ -2114,4 +2154,52 @@ mod tests {
 			BTreeMap::default(),
 		);
 	}
+
+	#[test]
+	fn test_empty_steps() {
+		let last_benign = Arc::new(AtomicUsize::new(0));
+		let params = AuthorityRoundParams {
+			step_duration: 4,
+			start_step: Some(1),
+			validators: Box::new(TestSet::new(Default::default(), last_benign.clone())),
+			validate_score_transition: 0,
+			validate_step_transition: 0,
+			immediate_transitions: true,
+			maximum_uncle_count_transition: 0,
+			maximum_uncle_count: 0,
+			empty_steps_transition: 0,
+			maximum_empty_steps: 10,
+			block_reward: Default::default(),
+			block_reward_contract_transition: 0,
+			block_reward_contract: Default::default(),
+		};
+
+		let mut c_params = ::spec::CommonParams::default();
+		c_params.gas_limit_bound_divisor = 5.into();
+		let machine = ::machine::EthereumMachine::regular(c_params, Default::default());
+		let engine = AuthorityRound::new(params, machine).unwrap();
+
+
+		let parent_hash: H256 = 1.into();
+		let signature = H520::default();
+		let step = |step: u64| EmptyStep {
+			step,
+			parent_hash,
+			signature,
+		};
+
+		engine.handle_empty_step_message(step(1));
+		engine.handle_empty_step_message(step(3));
+		engine.handle_empty_step_message(step(2));
+		engine.handle_empty_step_message(step(1));
+
+		assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(1), step(2), step(3)]);
+		assert_eq!(engine.empty_steps(2, 3, parent_hash), vec![]);
+		assert_eq!(engine.empty_steps(2, 4, parent_hash), vec![step(3)]);
+
+		engine.clear_empty_steps(2);
+
+		assert_eq!(engine.empty_steps(0, 3, parent_hash), vec![]);
+		assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(3)]);
+	}
 }

ethcore/sync/Cargo.toml

@@ -31,7 +31,6 @@ env_logger = "0.5"
 rand = "0.4"
 heapsize = "0.4"
 semver = "0.9"
-smallvec = { version = "0.4", features = ["heapsizeof"] }
 parking_lot = "0.6"
 trace-time = "0.1"
 ipnetwork = "0.12.6"

ethcore/sync/src/api.rs

@@ -264,12 +264,10 @@ pub struct EthSync {
 
 fn light_params(
 	network_id: u64,
-	max_peers: u32,
+	median_peers: f64,
 	pruning_info: PruningInfo,
 	sample_store: Option<Box<SampleStore>>,
 ) -> LightParams {
-	const MAX_LIGHTSERV_LOAD: f64 = 0.5;
-
 	let mut light_params = LightParams {
 		network_id: network_id,
 		config: Default::default(),
@@ -282,9 +280,7 @@ fn light_params(
 		sample_store: sample_store,
 	};
 
-	let max_peers = ::std::cmp::max(max_peers, 1);
+	light_params.config.median_peers = median_peers;
-	light_params.config.load_share = MAX_LIGHTSERV_LOAD / max_peers as f64;
-
 	light_params
 }
 
@@ -301,9 +297,10 @@ impl EthSync {
 					.map(|mut p| { p.push("request_timings"); light_net::FileStore(p) })
 					.map(|store| Box::new(store) as Box<_>);
 
+				let median_peers = (params.network_config.min_peers + params.network_config.max_peers) as f64 / 2.0;
 				let light_params = light_params(
 					params.config.network_id,
-					params.network_config.max_peers,
+					median_peers,
 					pruning_info,
 					sample_store,
 				);
@@ -940,19 +937,3 @@ impl LightSyncProvider for LightSync {
 		Default::default() // TODO
 	}
 }
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-
-	#[test]
-	fn light_params_load_share_depends_on_max_peers() {
-		let pruning_info = PruningInfo {
-			earliest_chain: 0,
-			earliest_state: 0,
-		};
-		let params1 = light_params(0, 10, pruning_info.clone(), None);
-		let params2 = light_params(0, 20, pruning_info, None);
-		assert!(params1.config.load_share > params2.config.load_share)
-	}
-}

ethcore/sync/src/blocks.rs

@@ -15,7 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{HashSet, HashMap, hash_map};
-use smallvec::SmallVec;
 use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP};
 use heapsize::HeapSizeOf;
 use ethereum_types::H256;
@@ -29,8 +28,6 @@ use transaction::UnverifiedTransaction;
 
 known_heap_size!(0, HeaderId);
 
-type SmallHashVec = SmallVec<[H256; 1]>;
-
 #[derive(PartialEq, Debug, Clone)]
 pub struct SyncHeader {
 	pub bytes: Bytes,
@@ -157,7 +154,7 @@ pub struct BlockCollection {
 	/// Used to map body to header.
 	header_ids: HashMap<HeaderId, H256>,
 	/// Used to map receipts root to headers.
-	receipt_ids: HashMap<H256, SmallHashVec>,
+	receipt_ids: HashMap<H256, Vec<H256>>,
 	/// First block in `blocks`.
 	head: Option<H256>,
 	/// Set of block header hashes being downloaded
@@ -519,7 +516,7 @@ impl BlockCollection {
 				let receipts_stream = RlpStream::new_list(0);
 				(Some(receipts_stream.out()), receipt_root)
 			} else {
-				self.receipt_ids.entry(receipt_root).or_insert_with(|| SmallHashVec::new()).push(hash);
+				self.receipt_ids.entry(receipt_root).or_insert_with(Vec::new).push(hash);
 				(None, receipt_root)
 			}
 		} else {

ethcore/sync/src/lib.rs

@@ -35,7 +35,6 @@ extern crate fastmap;
 extern crate rand;
 extern crate semver;
 extern crate parking_lot;
-extern crate smallvec;
 extern crate rlp;
 extern crate ipnetwork;
 extern crate keccak_hash as hash;

ethcore/sync/src/light_sync/mod.rs

@@ -34,6 +34,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::mem;
+use std::ops::Deref;
 use std::sync::Arc;
 use std::time::{Instant, Duration};
 
@@ -213,6 +214,44 @@ enum SyncState {
 	Rounds(SyncRound),
 }
 
+/// A wrapper around the SyncState that makes sure to
+/// update the giving reference to `is_idle`
+#[derive(Debug)]
+struct SyncStateWrapper {
+	state: SyncState,
+}
+
+impl SyncStateWrapper {
+	/// Create a new wrapper for SyncState::Idle
+	pub fn idle() -> Self {
+		SyncStateWrapper {
+			state: SyncState::Idle,
+		}
+	}
+
+	/// Set the new state's value, making sure `is_idle` gets updated
+	pub fn set(&mut self, state: SyncState, is_idle_handle: &mut bool) {
+		*is_idle_handle = match state {
+			SyncState::Idle => true,
+			_ => false,
+		};
+		self.state = state;
+	}
+
+	/// Returns the internal state's value
+	pub fn into_inner(self) -> SyncState {
+		self.state
+	}
+}
+
+impl Deref for SyncStateWrapper {
+	type Target = SyncState;
+
+	fn deref(&self) -> &SyncState {
+		&self.state
+	}
+}
+
 struct ResponseCtx<'a> {
 	peer: PeerId,
 	req_id: ReqId,
@@ -235,7 +274,9 @@ pub struct LightSync<L: AsLightClient> {
 	pending_reqs: Mutex<HashMap<ReqId, PendingReq>>, // requests from this handler
 	client: Arc<L>,
 	rng: Mutex<OsRng>,
-	state: Mutex<SyncState>,
+	state: Mutex<SyncStateWrapper>,
+	// We duplicate this state tracking to avoid deadlocks in `is_major_importing`.
+	is_idle: Mutex<bool>,
 }
 
 #[derive(Debug, Clone)]
@@ -309,16 +350,17 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 
 		if new_best.is_none() {
 			debug!(target: "sync", "No peers remain. Reverting to idle");
-			*self.state.lock() = SyncState::Idle;
+			self.set_state(&mut self.state.lock(), SyncState::Idle);
 		} else {
 			let mut state = self.state.lock();
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Idle => SyncState::Idle,
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.requests_abandoned(unfulfilled)),
 				SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(unfulfilled)),
 			};
+			self.set_state(&mut state, next_state);
 		}
 
 		self.maintain_sync(ctx.as_basic());
@@ -390,12 +432,13 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 				data: headers,
 			};
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Idle => SyncState::Idle,
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.process_response(&ctx, &*self.client)),
 				SyncState::Rounds(round) => SyncState::Rounds(round.process_response(&ctx)),
 			};
+			self.set_state(&mut state, next_state);
 		}
 
 		self.maintain_sync(ctx.as_basic());
@@ -408,12 +451,18 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 
 // private helpers
 impl<L: AsLightClient> LightSync<L> {
+	/// Sets the LightSync's state, and update
+	/// `is_idle`
+	fn set_state(&self, state: &mut SyncStateWrapper, next_state: SyncState) {
+		state.set(next_state, &mut self.is_idle.lock());
+	}
+
 	// Begins a search for the common ancestor and our best block.
 	// does not lock state, instead has a mutable reference to it passed.
-	fn begin_search(&self, state: &mut SyncState) {
+	fn begin_search(&self, state: &mut SyncStateWrapper) {
 		if let None =  *self.best_seen.lock() {
 			// no peers.
-			*state = SyncState::Idle;
+			self.set_state(state, SyncState::Idle);
 			return;
 		}
 
@@ -422,7 +471,8 @@ impl<L: AsLightClient> LightSync<L> {
 
 		trace!(target: "sync", "Beginning search for common ancestor from {:?}",
 			(chain_info.best_block_number, chain_info.best_block_hash));
-		*state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number));
+		let next_state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number));
+		self.set_state(state, next_state);
 	}
 
 	// handles request dispatch, block import, state machine transitions, and timeouts.
@@ -435,7 +485,7 @@ impl<L: AsLightClient> LightSync<L> {
 		let chain_info = client.chain_info();
 
 		let mut state = self.state.lock();
-		debug!(target: "sync", "Maintaining sync ({:?})", &*state);
+		debug!(target: "sync", "Maintaining sync ({:?})", **state);
 
 		// drain any pending blocks into the queue.
 		{
@@ -445,11 +495,12 @@ impl<L: AsLightClient> LightSync<L> {
 			loop {
 				if client.queue_info().is_full() { break }
 
-				*state = match mem::replace(&mut *state, SyncState::Idle) {
+				let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 					SyncState::Rounds(round)
 						=> SyncState::Rounds(round.drain(&mut sink, Some(DRAIN_AMOUNT))),
 					other => other,
 				};
+				self.set_state(&mut state, next_state);
 
 				if sink.is_empty() { break }
 				trace!(target: "sync", "Drained {} headers to import", sink.len());
@@ -483,15 +534,15 @@ impl<L: AsLightClient> LightSync<L> {
 					let network_score = other.as_ref().map(|target| target.head_td);
 					trace!(target: "sync", "No target to sync to. Network score: {:?}, Local score: {:?}",
 						network_score, best_td);
-					*state = SyncState::Idle;
+					self.set_state(&mut state, SyncState::Idle);
 					return;
 				}
 			};
 
-			match mem::replace(&mut *state, SyncState::Idle) {
+			match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Rounds(SyncRound::Abort(reason, remaining)) => {
 					if remaining.len() > 0 {
-						*state = SyncState::Rounds(SyncRound::Abort(reason, remaining));
+						self.set_state(&mut state, SyncState::Rounds(SyncRound::Abort(reason, remaining)));
 						return;
 					}
 
@@ -505,7 +556,7 @@ impl<L: AsLightClient> LightSync<L> {
 						AbortReason::NoResponses => {}
 						AbortReason::TargetReached => {
 							debug!(target: "sync", "Sync target reached. Going idle");
-							*state = SyncState::Idle;
+							self.set_state(&mut state, SyncState::Idle);
 							return;
 						}
 					}
@@ -514,15 +565,15 @@ impl<L: AsLightClient> LightSync<L> {
 					self.begin_search(&mut state);
 				}
 				SyncState::AncestorSearch(AncestorSearch::FoundCommon(num, hash)) => {
-					*state = SyncState::Rounds(SyncRound::begin((num, hash), sync_target));
+					self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((num, hash), sync_target)));
 				}
 				SyncState::AncestorSearch(AncestorSearch::Genesis) => {
 					// Same here.
 					let g_hash = chain_info.genesis_hash;
-					*state = SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target));
+					self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target)));
 				}
 				SyncState::Idle => self.begin_search(&mut state),
-				other => *state = other, // restore displaced state.
+				other => self.set_state(&mut state, other), // restore displaced state.
 			}
 		}
 
@@ -543,12 +594,13 @@ impl<L: AsLightClient> LightSync<L> {
 				}
 				drop(pending_reqs);
 
-				*state = match mem::replace(&mut *state, SyncState::Idle) {
+				let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 					SyncState::Idle => SyncState::Idle,
 					SyncState::AncestorSearch(search) =>
 						SyncState::AncestorSearch(search.requests_abandoned(&unfulfilled)),
 					SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(&unfulfilled)),
 				};
+				self.set_state(&mut state, next_state);
 			}
 		}
 
@@ -605,34 +657,14 @@ impl<L: AsLightClient> LightSync<L> {
 				None
 			};
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Rounds(round) =>
 					SyncState::Rounds(round.dispatch_requests(dispatcher)),
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.dispatch_request(dispatcher)),
 				other => other,
 			};
-		}
+			self.set_state(&mut state, next_state);
-	}
-
-	fn is_major_importing_do_wait(&self, wait: bool) -> bool {
-		const EMPTY_QUEUE: usize = 3;
-
-		if self.client.as_light_client().queue_info().unverified_queue_size > EMPTY_QUEUE {
-			return true;
-		}
-		let mg_state = if wait {
-			self.state.lock()
-		} else {
-			if let Some(mg_state) = self.state.try_lock() {
-				mg_state
-			} else {
-				return false;
-			}
-		};
-		match *mg_state {
-			SyncState::Idle => false,
-			_ => true,
 		}
 	}
 }
@@ -651,7 +683,8 @@ impl<L: AsLightClient> LightSync<L> {
 			pending_reqs: Mutex::new(HashMap::new()),
 			client: client,
 			rng: Mutex::new(OsRng::new()?),
-			state: Mutex::new(SyncState::Idle),
+			state: Mutex::new(SyncStateWrapper::idle()),
+			is_idle: Mutex::new(true),
 		})
 	}
 }
@@ -666,9 +699,6 @@ pub trait SyncInfo {
 
 	/// Whether major sync is underway.
 	fn is_major_importing(&self) -> bool;
-
-	/// Whether major sync is underway, skipping some synchronization.
-	fn is_major_importing_no_sync(&self) -> bool;
 }
 
 impl<L: AsLightClient> SyncInfo for LightSync<L> {
@@ -681,11 +711,13 @@ impl<L: AsLightClient> SyncInfo for LightSync<L> {
 	}
 
 	fn is_major_importing(&self) -> bool {
-		self.is_major_importing_do_wait(true)
+		const EMPTY_QUEUE: usize = 3;
-	}
 
-	fn is_major_importing_no_sync(&self) -> bool {
+		let queue_info = self.client.as_light_client().queue_info();
-		self.is_major_importing_do_wait(false)
+		let is_verifying = queue_info.unverified_queue_size + queue_info.verified_queue_size > EMPTY_QUEUE;
+		let is_syncing = !*self.is_idle.lock();
+
+		is_verifying || is_syncing
 	}
 
 }

ethstore/Cargo.toml

@@ -19,7 +19,7 @@ parking_lot = "0.6"
 parity-crypto = "0.1"
 ethereum-types = "0.4"
 dir = { path = "../util/dir" }
-smallvec = "0.4"
+smallvec = "0.6"
 parity-wordlist = "1.0"
 tempdir = "0.3"
 

miner/src/pool/tests/mod.rs

@@ -76,6 +76,7 @@ fn should_return_correct_nonces_when_dropped_because_of_limit() {
 	// when
 	let tx1 = Tx::gas_price(2).signed();
 	let tx2 = Tx::gas_price(2).signed();
+	let sender = tx2.sender();
 	let tx3 = Tx::gas_price(1).signed();
 	let tx4 = Tx::gas_price(3).signed();
 	let res = txq.import(TestClient::new(), vec![tx1, tx2].retracted());
@@ -90,7 +91,8 @@ fn should_return_correct_nonces_when_dropped_because_of_limit() {
 			   Ok(())
 	]);
 	assert_eq!(txq.status().status.transaction_count, 3);
-	// First inserted transacton got dropped because of limit
+	// tx2 transacton got dropped because of limit
+	// tx1 and tx1' are kept, because they have lower insertion_ids so they are preferred.
 	assert_eq!(txq.next_nonce(TestClient::new(), &sender), None);
 }
 

parity/informant.rs

@@ -184,7 +184,7 @@ impl InformantData for LightNodeInformantData {
 	fn executes_transactions(&self) -> bool { false }
 
 	fn is_major_importing(&self) -> bool {
-		self.sync.is_major_importing_no_sync()
+		self.sync.is_major_importing()
 	}
 
 	fn report(&self) -> Report {

parity/run.rs

@@ -396,11 +396,6 @@ fn execute_impl<Cr, Rr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq:
 	// create dirs used by parity
 	cmd.dirs.create_dirs(cmd.acc_conf.unlocked_accounts.len() == 0, cmd.secretstore_conf.enabled)?;
 
-	// run in daemon mode
-	if let Some(pid_file) = cmd.daemon {
-		daemonize(pid_file)?;
-	}
-
 	//print out running parity environment
 	print_running_environment(&spec.data_dir, &cmd.dirs, &db_dirs);
 
@@ -800,6 +795,12 @@ fn execute_impl<Cr, Rr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq:
 	client.set_exit_handler(on_client_rq);
 	updater.set_exit_handler(on_updater_rq);
 
+	// run in daemon mode
+	if let Some(pid_file) = cmd.daemon {
+		info!("Running as a daemon process!");
+		daemonize(pid_file)?;
+	}
+
 	Ok(RunningClient {
 		inner: RunningClientInner::Full {
 			rpc: rpc_direct,

scripts/docker/hub/Dockerfile

@@ -5,7 +5,7 @@ ARG TARGET
 ENV TARGET ${TARGET}
 
 # install tools and dependencies
-RUN apt update && apt install -y --no-install-recommends openssl libudev-dev file
+RUN apt update && apt install -y --no-install-recommends openssl libudev-dev file curl jq
 
 # show backtraces
 ENV RUST_BACKTRACE 1
@@ -18,19 +18,19 @@ RUN rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/*
 RUN groupadd -g 1000 parity \
   && useradd -m -u 1000 -g parity -s /bin/sh parity
 
-USER parity
-
-WORKDIR /home/parity
-
-ENV PATH "~/bin:${PATH}"
 
 #add TARGET to docker image
-COPY artifacts/x86_64-unknown-linux-gnu/$TARGET ./bin/$TARGET
+COPY artifacts/x86_64-unknown-linux-gnu/$TARGET /bin/$TARGET
 
 # Build a shell script because the ENTRYPOINT command doesn't like using ENV
 RUN echo "#!/bin/bash \n ${TARGET} \$@" > ./entrypoint.sh
 RUN chmod +x ./entrypoint.sh
 
+COPY scripts/docker/hub/check_sync.sh /check_sync.sh
+
+# switch to user parity here
+USER parity
+
 # setup ENTRYPOINT
 EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp
 ENTRYPOINT ["./entrypoint.sh"]

scripts/docker/hub/check_sync.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# checks if parity has a fully synced blockchain
+
+ETH_SYNCING=$(curl -X POST --data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' http://localhost:8545 -H 'Content-Type: application/json')
+RESULT=$(echo "$ETH_SYNCING" | jq -r .result)
+
+if [ "$RESULT" == "false" ]; then
+  echo "Parity is ready to start accepting traffic"
+  exit 0
+else
+  echo "Parity is still syncing the blockchain"
+  exit 1
+fi

util/version/Cargo.toml

@@ -3,7 +3,7 @@
 [package]
 name = "parity-version"
 # NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION)
-version = "2.1.6"
+version = "2.1.7"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"
 

whisper/Cargo.toml

@@ -22,7 +22,7 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 slab = "0.3"
-smallvec = "0.4"
+smallvec = "0.6"
 tiny-keccak = "1.4"
 
 jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" }