Sync reorg up to history size (#3874)

* Allow sync reorg up to pruning history size

* Peer difficulty tracking

* Abort downloading block if received with NewBlock

* Set pruning history to 1200

* Renamed history size field

ethcore/src/verification/queue/kind.rs

@@ -19,18 +19,21 @@
 use engines::Engine;
 use error::Error;
 
-use util::{HeapSizeOf, H256};
+use util::{HeapSizeOf, H256, U256};
 
 pub use self::blocks::Blocks;
 pub use self::headers::Headers;
 
 /// Something which can produce a hash and a parent hash.
-pub trait HasHash {
+pub trait BlockLike {
 	/// Get the hash of this item.
 	fn hash(&self) -> H256;
 
 	/// Get the hash of this item's parent.
 	fn parent_hash(&self) -> H256;
+
+	/// Get the difficulty of this item.
+	fn difficulty(&self) -> U256;
 }
 
 /// Defines transitions between stages of verification.
@@ -45,13 +48,13 @@ pub trait HasHash {
 /// consistent.
 pub trait Kind: 'static + Sized + Send + Sync {
 	/// The first stage: completely unverified.
-	type Input: Sized + Send + HasHash + HeapSizeOf;
+	type Input: Sized + Send + BlockLike + HeapSizeOf;
 
 	/// The second stage: partially verified.
-	type Unverified: Sized + Send + HasHash + HeapSizeOf;
+	type Unverified: Sized + Send + BlockLike + HeapSizeOf;
 
 	/// The third stage: completely verified.
-	type Verified: Sized + Send + HasHash + HeapSizeOf;
+	type Verified: Sized + Send + BlockLike + HeapSizeOf;
 
 	/// Attempt to create the `Unverified` item from the input.
 	fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error>;
@@ -62,14 +65,14 @@ pub trait Kind: 'static + Sized + Send + Sync {
 
 /// The blocks verification module.
 pub mod blocks {
-	use super::{Kind, HasHash};
+	use super::{Kind, BlockLike};
 
 	use engines::Engine;
 	use error::Error;
 	use header::Header;
 	use verification::{PreverifiedBlock, verify_block_basic, verify_block_unordered};
 
-	use util::{Bytes, HeapSizeOf, H256};
+	use util::{Bytes, HeapSizeOf, H256, U256};
 
 	/// A mode for verifying blocks.
 	pub struct Blocks;
@@ -126,7 +129,7 @@ pub mod blocks {
 		}
 	}
 
-	impl HasHash for Unverified {
+	impl BlockLike for Unverified {
 		fn hash(&self) -> H256 {
 			self.header.hash()
 		}
@@ -134,9 +137,13 @@ pub mod blocks {
 		fn parent_hash(&self) -> H256 {
 			self.header.parent_hash().clone()
 		}
+
+		fn difficulty(&self) -> U256 {
+			self.header.difficulty().clone()
+		}
 	}
 
-	impl HasHash for PreverifiedBlock {
+	impl BlockLike for PreverifiedBlock {
 		fn hash(&self) -> H256 {
 			self.header.hash()
 		}
@@ -144,12 +151,16 @@ pub mod blocks {
 		fn parent_hash(&self) -> H256 {
 			self.header.parent_hash().clone()
 		}
+
+		fn difficulty(&self) -> U256 {
+			self.header.difficulty().clone()
+		}
 	}
 }
 
 /// Verification for headers.
 pub mod headers {
-	use super::{Kind, HasHash};
+	use super::{Kind, BlockLike};
 
 	use engines::Engine;
 	use error::Error;
@@ -157,10 +168,12 @@ pub mod headers {
 	use verification::verify_header_params;
 
 	use util::hash::H256;
+	use util::U256;
 
-	impl HasHash for Header {
+	impl BlockLike for Header {
 		fn hash(&self) -> H256 { self.hash() }
 		fn parent_hash(&self) -> H256 { self.parent_hash().clone() }
+		fn difficulty(&self) -> U256 { self.difficulty().clone() }
 	}
 
 	/// A mode for verifying headers.

ethcore/src/verification/queue/mod.rs

@@ -26,7 +26,7 @@ use error::*;
 use engines::Engine;
 use service::*;
 
-use self::kind::{HasHash, Kind};
+use self::kind::{BlockLike, Kind};
 
 pub use types::verification_queue_info::VerificationQueueInfo as QueueInfo;
 
@@ -132,13 +132,14 @@ pub struct VerificationQueue<K: Kind> {
 	deleting: Arc<AtomicBool>,
 	ready_signal: Arc<QueueSignal>,
 	empty: Arc<SCondvar>,
-	processing: RwLock<HashSet<H256>>,
+	processing: RwLock<HashMap<H256, U256>>, // hash to difficulty
 	ticks_since_adjustment: AtomicUsize,
 	max_queue_size: usize,
 	max_mem_use: usize,
 	scale_verifiers: bool,
 	verifier_handles: Vec<JoinHandle<()>>,
 	state: Arc<(Mutex<State>, Condvar)>,
+	total_difficulty: RwLock<U256>,
 }
 
 struct QueueSignal {
@@ -269,7 +270,7 @@ impl<K: Kind> VerificationQueue<K> {
 			more_to_verify: more_to_verify,
 			verification: verification,
 			deleting: deleting,
-			processing: RwLock::new(HashSet::new()),
+			processing: RwLock::new(HashMap::new()),
 			empty: empty,
 			ticks_since_adjustment: AtomicUsize::new(0),
 			max_queue_size: max(config.max_queue_size, MIN_QUEUE_LIMIT),
@@ -277,6 +278,7 @@ impl<K: Kind> VerificationQueue<K> {
 			scale_verifiers: scale_verifiers,
 			verifier_handles: verifier_handles,
 			state: state,
+			total_difficulty: RwLock::new(0.into()),
 		}
 	}
 
@@ -434,6 +436,7 @@ impl<K: Kind> VerificationQueue<K> {
 		sizes.unverified.store(0, AtomicOrdering::Release);
 		sizes.verifying.store(0, AtomicOrdering::Release);
 		sizes.verified.store(0, AtomicOrdering::Release);
+		*self.total_difficulty.write() = 0.into();
 
 		self.processing.write().clear();
 	}
@@ -448,7 +451,7 @@ impl<K: Kind> VerificationQueue<K> {
 
 	/// Check if the item is currently in the queue
 	pub fn status(&self, hash: &H256) -> Status {
-		if self.processing.read().contains(hash) {
+		if self.processing.read().contains_key(hash) {
 			return Status::Queued;
 		}
 		if self.verification.bad.lock().contains(hash) {
@@ -461,7 +464,7 @@ impl<K: Kind> VerificationQueue<K> {
 	pub fn import(&self, input: K::Input) -> ImportResult {
 		let h = input.hash();
 		{
-			if self.processing.read().contains(&h) {
+			if self.processing.read().contains_key(&h) {
 				return Err(ImportError::AlreadyQueued.into());
 			}
 
@@ -480,7 +483,11 @@ impl<K: Kind> VerificationQueue<K> {
 			Ok(item) => {
 				self.verification.sizes.unverified.fetch_add(item.heap_size_of_children(), AtomicOrdering::SeqCst);
 
-				self.processing.write().insert(h.clone());
+				self.processing.write().insert(h.clone(), item.difficulty());
+				{
+					let mut td = self.total_difficulty.write();
+					*td = *td + item.difficulty();
+				}
 				self.verification.unverified.lock().push_back(item);
 				self.more_to_verify.notify_all();
 				Ok(h)
@@ -511,7 +518,10 @@ impl<K: Kind> VerificationQueue<K> {
 		bad.reserve(hashes.len());
 		for hash in hashes {
 			bad.insert(hash.clone());
-			processing.remove(hash);
+			if let Some(difficulty) = processing.remove(hash) {
+				let mut td = self.total_difficulty.write();
+				*td = *td - difficulty;
+			}
 		}
 
 		let mut new_verified = VecDeque::new();
@@ -520,7 +530,10 @@ impl<K: Kind> VerificationQueue<K> {
 			if bad.contains(&output.parent_hash()) {
 				removed_size += output.heap_size_of_children();
 				bad.insert(output.hash());
-				processing.remove(&output.hash());
+				if let Some(difficulty) = processing.remove(&output.hash()) {
+					let mut td = self.total_difficulty.write();
+					*td = *td - difficulty;
+				}
 			} else {
 				new_verified.push_back(output);
 			}
@@ -538,7 +551,10 @@ impl<K: Kind> VerificationQueue<K> {
 		}
 		let mut processing = self.processing.write();
 		for hash in hashes {
-			processing.remove(hash);
+			if let Some(difficulty) = processing.remove(hash) {
+				let mut td = self.total_difficulty.write();
+				*td = *td - difficulty;
+			}
 		}
 		processing.is_empty()
 	}
@@ -592,6 +608,11 @@ impl<K: Kind> VerificationQueue<K> {
 		}
 	}
 
+	/// Get the total difficulty of all the blocks in the queue.
+	pub fn total_difficulty(&self) -> U256 {
+		self.total_difficulty.read().clone()
+	}
+
 	/// Get the current number of working verifiers.
 	pub fn num_verifiers(&self) -> usize {
 		match *self.state.0.lock() {
@@ -760,6 +781,22 @@ mod tests {
 		}
 	}
 
+	#[test]
+	fn returns_total_difficulty() {
+		let queue = get_test_queue(false);
+		let block = get_good_dummy_block();
+		let hash = BlockView::new(&block).header().hash().clone();
+		if let Err(e) = queue.import(Unverified::new(block)) {
+			panic!("error importing block that is valid by definition({:?})", e);
+		}
+		queue.flush();
+		assert_eq!(queue.total_difficulty(), 131072.into());
+		queue.drain(10);
+		assert_eq!(queue.total_difficulty(), 131072.into());
+		queue.mark_as_good(&[ hash ]);
+		assert_eq!(queue.total_difficulty(), 0.into());
+	}
+
 	#[test]
 	fn returns_ok_for_drained_duplicates() {
 		let queue = get_test_queue(false);