Backports for stable 2.1.7 (#9975)

* version: bump stable to 2.1.7

* Adjust requests costs for light client (#9925)

* PIP Table Cost relative to average peers instead of max peers

* Add tracing in PIP new_cost_table

* Update stat peer_count

* Use number of leeching peers for Light serve costs

* Fix test::light_params_load_share_depends_on_max_peers (wrong type)

* Remove (now) useless test

* Remove `load_share` from LightParams.Config
Prevent div. by 0

* Add LEECHER_COUNT_FACTOR

* PR Grumble: u64 to u32 for f64 casting

* Prevent u32 overflow for avg_peer_count

* Add tests for LightSync::Statistics

* Fix empty steps (#9939)

* Don't send empty step twice or empty step then block.

* Perform basic validation of locally sealed blocks.

* Don't include empty step twice.

* prevent silent errors in daemon mode, closes #9367 (#9946)

* Fix light client informant while syncing (#9932)

* Add `is_idle` to LightSync to check importing status

* Use SyncStateWrapper to make sure is_idle gets updates

* Update is_major_import to use verified queue size as well

* Add comment for `is_idle`

* Add Debug to `SyncStateWrapper`

* `fn get` -> `fn into_inner`

*  ci: rearrange pipeline by logic (#9970)

* ci: rearrange pipeline by logic

* ci: rename docs script

* Add readiness check for docker container (#9804)

* Update Dockerfile

Since parity is built for "mission critical use", I thought other operators may see the need for this.

Adding the `curl` and `jq` commands allows for an extremely simple health check to be usable in container orchestrators.

For example. Here is a health check for a parity docker container running in Kubernetes.

This can be setup as a readiness Probe that would prevent clustered nodes that aren't ready from serving traffic.

```bash
#!/bin/bash

ETH_SYNCING=$(curl -X POST --data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' http://localhost:8545 -H 'Content-Type: application/json')
RESULT=$(echo "$ETH_SYNCING | jq -r .result)

if [ "$RESULT" == "false" ]; then
  echo "Parity is ready to start accepting traffic"
  exit 0
else
  echo "Parity is still syncing the blockchain"
  exit 1
fi
```

* add sync check script

* Fix docker script (#9854)


* Dockerfile: change source path of the newly added check_sync.sh (#9869)

* Do not use the home directory as the working dir in docker (#9834)

* Do not create a home directory.

* Re-add -m flag

* fix docker build (#9971)

* bump smallvec to 0.6 in ethcore-light, ethstore and whisper (#9588)

* bump smallvec to 0.6 in ethcore-light, ethstore and whisper

* bump transaction-pool

* Fix test.

* patch cargo to use tokio-proto from git repo

this makes sure we no longer depend on smallvec 0.2.1 which is
affected by https://github.com/servo/rust-smallvec/issues/96

* use patched version of untrusted 0.5.1

* ci: allow audit to fail

ethcore/sync/src/api.rs

@@ -264,12 +264,10 @@ pub struct EthSync {
 
 fn light_params(
 	network_id: u64,
-	max_peers: u32,
+	median_peers: f64,
 	pruning_info: PruningInfo,
 	sample_store: Option<Box<SampleStore>>,
 ) -> LightParams {
-	const MAX_LIGHTSERV_LOAD: f64 = 0.5;
-
 	let mut light_params = LightParams {
 		network_id: network_id,
 		config: Default::default(),
@@ -282,9 +280,7 @@ fn light_params(
 		sample_store: sample_store,
 	};
 
-	let max_peers = ::std::cmp::max(max_peers, 1);
-	light_params.config.load_share = MAX_LIGHTSERV_LOAD / max_peers as f64;
-
+	light_params.config.median_peers = median_peers;
 	light_params
 }
 
@@ -301,9 +297,10 @@ impl EthSync {
 					.map(|mut p| { p.push("request_timings"); light_net::FileStore(p) })
 					.map(|store| Box::new(store) as Box<_>);
 
+				let median_peers = (params.network_config.min_peers + params.network_config.max_peers) as f64 / 2.0;
 				let light_params = light_params(
 					params.config.network_id,
-					params.network_config.max_peers,
+					median_peers,
 					pruning_info,
 					sample_store,
 				);
@@ -940,19 +937,3 @@ impl LightSyncProvider for LightSync {
 		Default::default() // TODO
 	}
 }
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-
-	#[test]
-	fn light_params_load_share_depends_on_max_peers() {
-		let pruning_info = PruningInfo {
-			earliest_chain: 0,
-			earliest_state: 0,
-		};
-		let params1 = light_params(0, 10, pruning_info.clone(), None);
-		let params2 = light_params(0, 20, pruning_info, None);
-		assert!(params1.config.load_share > params2.config.load_share)
-	}
-}

ethcore/sync/src/blocks.rs

@@ -15,7 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{HashSet, HashMap, hash_map};
-use smallvec::SmallVec;
 use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP};
 use heapsize::HeapSizeOf;
 use ethereum_types::H256;
@@ -29,8 +28,6 @@ use transaction::UnverifiedTransaction;
 
 known_heap_size!(0, HeaderId);
 
-type SmallHashVec = SmallVec<[H256; 1]>;
-
 #[derive(PartialEq, Debug, Clone)]
 pub struct SyncHeader {
 	pub bytes: Bytes,
@@ -157,7 +154,7 @@ pub struct BlockCollection {
 	/// Used to map body to header.
 	header_ids: HashMap<HeaderId, H256>,
 	/// Used to map receipts root to headers.
-	receipt_ids: HashMap<H256, SmallHashVec>,
+	receipt_ids: HashMap<H256, Vec<H256>>,
 	/// First block in `blocks`.
 	head: Option<H256>,
 	/// Set of block header hashes being downloaded
@@ -519,7 +516,7 @@ impl BlockCollection {
 				let receipts_stream = RlpStream::new_list(0);
 				(Some(receipts_stream.out()), receipt_root)
 			} else {
-				self.receipt_ids.entry(receipt_root).or_insert_with(|| SmallHashVec::new()).push(hash);
+				self.receipt_ids.entry(receipt_root).or_insert_with(Vec::new).push(hash);
 				(None, receipt_root)
 			}
 		} else {

ethcore/sync/src/lib.rs

@@ -35,7 +35,6 @@ extern crate fastmap;
 extern crate rand;
 extern crate semver;
 extern crate parking_lot;
-extern crate smallvec;
 extern crate rlp;
 extern crate ipnetwork;
 extern crate keccak_hash as hash;

ethcore/sync/src/light_sync/mod.rs

@@ -34,6 +34,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::mem;
+use std::ops::Deref;
 use std::sync::Arc;
 use std::time::{Instant, Duration};
 
@@ -213,6 +214,44 @@ enum SyncState {
 	Rounds(SyncRound),
 }
 
+/// A wrapper around the SyncState that makes sure to
+/// update the giving reference to `is_idle`
+#[derive(Debug)]
+struct SyncStateWrapper {
+	state: SyncState,
+}
+
+impl SyncStateWrapper {
+	/// Create a new wrapper for SyncState::Idle
+	pub fn idle() -> Self {
+		SyncStateWrapper {
+			state: SyncState::Idle,
+		}
+	}
+
+	/// Set the new state's value, making sure `is_idle` gets updated
+	pub fn set(&mut self, state: SyncState, is_idle_handle: &mut bool) {
+		*is_idle_handle = match state {
+			SyncState::Idle => true,
+			_ => false,
+		};
+		self.state = state;
+	}
+
+	/// Returns the internal state's value
+	pub fn into_inner(self) -> SyncState {
+		self.state
+	}
+}
+
+impl Deref for SyncStateWrapper {
+	type Target = SyncState;
+
+	fn deref(&self) -> &SyncState {
+		&self.state
+	}
+}
+
 struct ResponseCtx<'a> {
 	peer: PeerId,
 	req_id: ReqId,
@@ -235,7 +274,9 @@ pub struct LightSync<L: AsLightClient> {
 	pending_reqs: Mutex<HashMap<ReqId, PendingReq>>, // requests from this handler
 	client: Arc<L>,
 	rng: Mutex<OsRng>,
-	state: Mutex<SyncState>,
+	state: Mutex<SyncStateWrapper>,
+	// We duplicate this state tracking to avoid deadlocks in `is_major_importing`.
+	is_idle: Mutex<bool>,
 }
 
 #[derive(Debug, Clone)]
@@ -309,16 +350,17 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 
 		if new_best.is_none() {
 			debug!(target: "sync", "No peers remain. Reverting to idle");
-			*self.state.lock() = SyncState::Idle;
+			self.set_state(&mut self.state.lock(), SyncState::Idle);
 		} else {
 			let mut state = self.state.lock();
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Idle => SyncState::Idle,
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.requests_abandoned(unfulfilled)),
 				SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(unfulfilled)),
 			};
+			self.set_state(&mut state, next_state);
 		}
 
 		self.maintain_sync(ctx.as_basic());
@@ -390,12 +432,13 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 				data: headers,
 			};
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Idle => SyncState::Idle,
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.process_response(&ctx, &*self.client)),
 				SyncState::Rounds(round) => SyncState::Rounds(round.process_response(&ctx)),
 			};
+			self.set_state(&mut state, next_state);
 		}
 
 		self.maintain_sync(ctx.as_basic());
@@ -408,12 +451,18 @@ impl<L: AsLightClient + Send + Sync> Handler for LightSync<L> {
 
 // private helpers
 impl<L: AsLightClient> LightSync<L> {
+	/// Sets the LightSync's state, and update
+	/// `is_idle`
+	fn set_state(&self, state: &mut SyncStateWrapper, next_state: SyncState) {
+		state.set(next_state, &mut self.is_idle.lock());
+	}
+
 	// Begins a search for the common ancestor and our best block.
 	// does not lock state, instead has a mutable reference to it passed.
-	fn begin_search(&self, state: &mut SyncState) {
+	fn begin_search(&self, state: &mut SyncStateWrapper) {
 		if let None =  *self.best_seen.lock() {
 			// no peers.
-			*state = SyncState::Idle;
+			self.set_state(state, SyncState::Idle);
 			return;
 		}
 
@@ -422,7 +471,8 @@ impl<L: AsLightClient> LightSync<L> {
 
 		trace!(target: "sync", "Beginning search for common ancestor from {:?}",
 			(chain_info.best_block_number, chain_info.best_block_hash));
-		*state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number));
+		let next_state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number));
+		self.set_state(state, next_state);
 	}
 
 	// handles request dispatch, block import, state machine transitions, and timeouts.
@@ -435,7 +485,7 @@ impl<L: AsLightClient> LightSync<L> {
 		let chain_info = client.chain_info();
 
 		let mut state = self.state.lock();
-		debug!(target: "sync", "Maintaining sync ({:?})", &*state);
+		debug!(target: "sync", "Maintaining sync ({:?})", **state);
 
 		// drain any pending blocks into the queue.
 		{
@@ -445,11 +495,12 @@ impl<L: AsLightClient> LightSync<L> {
 			loop {
 				if client.queue_info().is_full() { break }
 
-				*state = match mem::replace(&mut *state, SyncState::Idle) {
+				let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 					SyncState::Rounds(round)
 						=> SyncState::Rounds(round.drain(&mut sink, Some(DRAIN_AMOUNT))),
 					other => other,
 				};
+				self.set_state(&mut state, next_state);
 
 				if sink.is_empty() { break }
 				trace!(target: "sync", "Drained {} headers to import", sink.len());
@@ -483,15 +534,15 @@ impl<L: AsLightClient> LightSync<L> {
 					let network_score = other.as_ref().map(|target| target.head_td);
 					trace!(target: "sync", "No target to sync to. Network score: {:?}, Local score: {:?}",
 						network_score, best_td);
-					*state = SyncState::Idle;
+					self.set_state(&mut state, SyncState::Idle);
 					return;
 				}
 			};
 
-			match mem::replace(&mut *state, SyncState::Idle) {
+			match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Rounds(SyncRound::Abort(reason, remaining)) => {
 					if remaining.len() > 0 {
-						*state = SyncState::Rounds(SyncRound::Abort(reason, remaining));
+						self.set_state(&mut state, SyncState::Rounds(SyncRound::Abort(reason, remaining)));
 						return;
 					}
 
@@ -505,7 +556,7 @@ impl<L: AsLightClient> LightSync<L> {
 						AbortReason::NoResponses => {}
 						AbortReason::TargetReached => {
 							debug!(target: "sync", "Sync target reached. Going idle");
-							*state = SyncState::Idle;
+							self.set_state(&mut state, SyncState::Idle);
 							return;
 						}
 					}
@@ -514,15 +565,15 @@ impl<L: AsLightClient> LightSync<L> {
 					self.begin_search(&mut state);
 				}
 				SyncState::AncestorSearch(AncestorSearch::FoundCommon(num, hash)) => {
-					*state = SyncState::Rounds(SyncRound::begin((num, hash), sync_target));
+					self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((num, hash), sync_target)));
 				}
 				SyncState::AncestorSearch(AncestorSearch::Genesis) => {
 					// Same here.
 					let g_hash = chain_info.genesis_hash;
-					*state = SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target));
+					self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target)));
 				}
 				SyncState::Idle => self.begin_search(&mut state),
-				other => *state = other, // restore displaced state.
+				other => self.set_state(&mut state, other), // restore displaced state.
 			}
 		}
 
@@ -543,12 +594,13 @@ impl<L: AsLightClient> LightSync<L> {
 				}
 				drop(pending_reqs);
 
-				*state = match mem::replace(&mut *state, SyncState::Idle) {
+				let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 					SyncState::Idle => SyncState::Idle,
 					SyncState::AncestorSearch(search) =>
 						SyncState::AncestorSearch(search.requests_abandoned(&unfulfilled)),
 					SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(&unfulfilled)),
 				};
+				self.set_state(&mut state, next_state);
 			}
 		}
 
@@ -605,34 +657,14 @@ impl<L: AsLightClient> LightSync<L> {
 				None
 			};
 
-			*state = match mem::replace(&mut *state, SyncState::Idle) {
+			let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() {
 				SyncState::Rounds(round) =>
 					SyncState::Rounds(round.dispatch_requests(dispatcher)),
 				SyncState::AncestorSearch(search) =>
 					SyncState::AncestorSearch(search.dispatch_request(dispatcher)),
 				other => other,
 			};
-		}
-	}
-
-	fn is_major_importing_do_wait(&self, wait: bool) -> bool {
-		const EMPTY_QUEUE: usize = 3;
-
-		if self.client.as_light_client().queue_info().unverified_queue_size > EMPTY_QUEUE {
-			return true;
-		}
-		let mg_state = if wait {
-			self.state.lock()
-		} else {
-			if let Some(mg_state) = self.state.try_lock() {
-				mg_state
-			} else {
-				return false;
-			}
-		};
-		match *mg_state {
-			SyncState::Idle => false,
-			_ => true,
+			self.set_state(&mut state, next_state);
 		}
 	}
 }
@@ -651,7 +683,8 @@ impl<L: AsLightClient> LightSync<L> {
 			pending_reqs: Mutex::new(HashMap::new()),
 			client: client,
 			rng: Mutex::new(OsRng::new()?),
-			state: Mutex::new(SyncState::Idle),
+			state: Mutex::new(SyncStateWrapper::idle()),
+			is_idle: Mutex::new(true),
 		})
 	}
 }
@@ -666,9 +699,6 @@ pub trait SyncInfo {
 
 	/// Whether major sync is underway.
 	fn is_major_importing(&self) -> bool;
-
-	/// Whether major sync is underway, skipping some synchronization.
-	fn is_major_importing_no_sync(&self) -> bool;
 }
 
 impl<L: AsLightClient> SyncInfo for LightSync<L> {
@@ -681,11 +711,13 @@ impl<L: AsLightClient> SyncInfo for LightSync<L> {
 	}
 
 	fn is_major_importing(&self) -> bool {
-		self.is_major_importing_do_wait(true)
-	}
+		const EMPTY_QUEUE: usize = 3;
 
-	fn is_major_importing_no_sync(&self) -> bool {
-		self.is_major_importing_do_wait(false)
+		let queue_info = self.client.as_light_client().queue_info();
+		let is_verifying = queue_info.unverified_queue_size + queue_info.verified_queue_size > EMPTY_QUEUE;
+		let is_syncing = !*self.is_idle.lock();
+
+		is_verifying || is_syncing
 	}
 
 }