Sync reorg up to history size (#3874)

* Allow sync reorg up to pruning history size

* Peer difficulty tracking

* Abort downloading block if received with NewBlock

* Set pruning history to 1200

* Renamed history size field

ethcore/src/verification/queue/mod.rs

@@ -26,7 +26,7 @@ use error::*;
 use engines::Engine;
 use service::*;
 
-use self::kind::{HasHash, Kind};
+use self::kind::{BlockLike, Kind};
 
 pub use types::verification_queue_info::VerificationQueueInfo as QueueInfo;
 
@@ -132,13 +132,14 @@ pub struct VerificationQueue<K: Kind> {
 	deleting: Arc<AtomicBool>,
 	ready_signal: Arc<QueueSignal>,
 	empty: Arc<SCondvar>,
-	processing: RwLock<HashSet<H256>>,
+	processing: RwLock<HashMap<H256, U256>>, // hash to difficulty
 	ticks_since_adjustment: AtomicUsize,
 	max_queue_size: usize,
 	max_mem_use: usize,
 	scale_verifiers: bool,
 	verifier_handles: Vec<JoinHandle<()>>,
 	state: Arc<(Mutex<State>, Condvar)>,
+	total_difficulty: RwLock<U256>,
 }
 
 struct QueueSignal {
@@ -269,7 +270,7 @@ impl<K: Kind> VerificationQueue<K> {
 			more_to_verify: more_to_verify,
 			verification: verification,
 			deleting: deleting,
-			processing: RwLock::new(HashSet::new()),
+			processing: RwLock::new(HashMap::new()),
 			empty: empty,
 			ticks_since_adjustment: AtomicUsize::new(0),
 			max_queue_size: max(config.max_queue_size, MIN_QUEUE_LIMIT),
@@ -277,6 +278,7 @@ impl<K: Kind> VerificationQueue<K> {
 			scale_verifiers: scale_verifiers,
 			verifier_handles: verifier_handles,
 			state: state,
+			total_difficulty: RwLock::new(0.into()),
 		}
 	}
 
@@ -434,6 +436,7 @@ impl<K: Kind> VerificationQueue<K> {
 		sizes.unverified.store(0, AtomicOrdering::Release);
 		sizes.verifying.store(0, AtomicOrdering::Release);
 		sizes.verified.store(0, AtomicOrdering::Release);
+		*self.total_difficulty.write() = 0.into();
 
 		self.processing.write().clear();
 	}
@@ -448,7 +451,7 @@ impl<K: Kind> VerificationQueue<K> {
 
 	/// Check if the item is currently in the queue
 	pub fn status(&self, hash: &H256) -> Status {
-		if self.processing.read().contains(hash) {
+		if self.processing.read().contains_key(hash) {
 			return Status::Queued;
 		}
 		if self.verification.bad.lock().contains(hash) {
@@ -461,7 +464,7 @@ impl<K: Kind> VerificationQueue<K> {
 	pub fn import(&self, input: K::Input) -> ImportResult {
 		let h = input.hash();
 		{
-			if self.processing.read().contains(&h) {
+			if self.processing.read().contains_key(&h) {
 				return Err(ImportError::AlreadyQueued.into());
 			}
 
@@ -480,7 +483,11 @@ impl<K: Kind> VerificationQueue<K> {
 			Ok(item) => {
 				self.verification.sizes.unverified.fetch_add(item.heap_size_of_children(), AtomicOrdering::SeqCst);
 
-				self.processing.write().insert(h.clone());
+				self.processing.write().insert(h.clone(), item.difficulty());
+				{
+					let mut td = self.total_difficulty.write();
+					*td = *td + item.difficulty();
+				}
 				self.verification.unverified.lock().push_back(item);
 				self.more_to_verify.notify_all();
 				Ok(h)
@@ -511,7 +518,10 @@ impl<K: Kind> VerificationQueue<K> {
 		bad.reserve(hashes.len());
 		for hash in hashes {
 			bad.insert(hash.clone());
-			processing.remove(hash);
+			if let Some(difficulty) = processing.remove(hash) {
+				let mut td = self.total_difficulty.write();
+				*td = *td - difficulty;
+			}
 		}
 
 		let mut new_verified = VecDeque::new();
@@ -520,7 +530,10 @@ impl<K: Kind> VerificationQueue<K> {
 			if bad.contains(&output.parent_hash()) {
 				removed_size += output.heap_size_of_children();
 				bad.insert(output.hash());
-				processing.remove(&output.hash());
+				if let Some(difficulty) = processing.remove(&output.hash()) {
+					let mut td = self.total_difficulty.write();
+					*td = *td - difficulty;
+				}
 			} else {
 				new_verified.push_back(output);
 			}
@@ -538,7 +551,10 @@ impl<K: Kind> VerificationQueue<K> {
 		}
 		let mut processing = self.processing.write();
 		for hash in hashes {
-			processing.remove(hash);
+			if let Some(difficulty) = processing.remove(hash) {
+				let mut td = self.total_difficulty.write();
+				*td = *td - difficulty;
+			}
 		}
 		processing.is_empty()
 	}
@@ -592,6 +608,11 @@ impl<K: Kind> VerificationQueue<K> {
 		}
 	}
 
+	/// Get the total difficulty of all the blocks in the queue.
+	pub fn total_difficulty(&self) -> U256 {
+		self.total_difficulty.read().clone()
+	}
+
 	/// Get the current number of working verifiers.
 	pub fn num_verifiers(&self) -> usize {
 		match *self.state.0.lock() {
@@ -760,6 +781,22 @@ mod tests {
 		}
 	}
 
+	#[test]
+	fn returns_total_difficulty() {
+		let queue = get_test_queue(false);
+		let block = get_good_dummy_block();
+		let hash = BlockView::new(&block).header().hash().clone();
+		if let Err(e) = queue.import(Unverified::new(block)) {
+			panic!("error importing block that is valid by definition({:?})", e);
+		}
+		queue.flush();
+		assert_eq!(queue.total_difficulty(), 131072.into());
+		queue.drain(10);
+		assert_eq!(queue.total_difficulty(), 131072.into());
+		queue.mark_as_good(&[ hash ]);
+		assert_eq!(queue.total_difficulty(), 0.into());
+	}
+
 	#[test]
 	fn returns_ok_for_drained_duplicates() {
 		let queue = get_test_queue(false);