Merge branch 'master' into lightsync

ethcore/light/src/client/mod.rs

@@ -215,31 +215,31 @@ impl Provider for Client {
 		None
 	}
 
-	fn block_headers(&self, _req: request::Headers) -> Vec<Bytes> {
+	fn block_header(&self, id: BlockId) -> Option<Bytes> {
+		self.chain.get_header(id)
+	}
+
+	fn block_body(&self, _id: BlockId) -> Option<Bytes> {
+		None
+	}
+
+	fn block_receipts(&self, _hash: &H256) -> Option<Bytes> {
+		None
+	}
+
+	fn state_proof(&self, _req: request::StateProof) -> Vec<Bytes> {
 		Vec::new()
 	}
 
-	fn block_bodies(&self, _req: request::Bodies) -> Vec<Bytes> {
+	fn contract_code(&self, _req: request::ContractCode) -> Bytes {
 		Vec::new()
 	}
 
-	fn receipts(&self, _req: request::Receipts) -> Vec<Bytes> {
-		Vec::new()
-	}
-
-	fn proofs(&self, _req: request::StateProofs) -> Vec<Bytes> {
-		Vec::new()
-	}
-
-	fn contract_code(&self, _req: request::ContractCodes) -> Vec<Bytes> {
-		Vec::new()
-	}
-
-	fn header_proofs(&self, _req: request::HeaderProofs) -> Vec<Bytes> {
-		Vec::new()
+	fn header_proof(&self, _req: request::HeaderProof) -> Option<(Bytes, Vec<Bytes>)> {
+		None
 	}
 
 	fn ready_transactions(&self) -> Vec<PendingTransaction> {
-		Client::ready_transactions(self)
+		Vec::new()
 	}
 }

ethcore/light/src/net/mod.rs

@@ -264,7 +264,7 @@ impl LightProtocol {
 	/// Check the maximum amount of requests of a specific type
 	/// which a peer would be able to serve. Returns zero if the
 	/// peer is unknown or has no buffer flow parameters.
-	pub fn max_requests(&self, peer: PeerId, kind: request::Kind) -> usize {
+	fn max_requests(&self, peer: PeerId, kind: request::Kind) -> usize {
 		self.peers.read().get(&peer).and_then(|peer| {
 			let mut peer = peer.lock();
 			let idle = peer.idle;
@@ -989,7 +989,7 @@ impl LightProtocol {
 
 		let max_cost = try!(peer.deduct_max(&self.flow_params, request::Kind::Codes, req.code_requests.len()));
 
-		let response = self.provider.contract_code(req);
+		let response = self.provider.contract_codes(req);
 		let response_len = response.iter().filter(|x| !x.is_empty()).count();
 		let actual_cost = self.flow_params.compute_cost(request::Kind::Codes, response_len);
 		assert!(max_cost >= actual_cost, "Actual cost exceeded maximum computed cost.");

ethcore/light/src/net/tests/mod.rs

@@ -94,79 +94,36 @@ impl Provider for TestProvider {
 		None
 	}
 
-	fn block_headers(&self, req: request::Headers) -> Vec<Bytes> {
-		use request::HashOrNumber;
-		use ethcore::views::HeaderView;
+	fn block_header(&self, id: BlockId) -> Option<Bytes> {
+		self.0.client.block_header(id)
+	}
 
-		let best_num = self.chain_info().best_block_number;
-		let start_num = match req.start {
-			HashOrNumber::Number(start_num) => start_num,
-			HashOrNumber::Hash(hash) => match self.0.client.block_header(BlockId::Hash(hash)) {
-				None => {
-					return Vec::new();
-				}
-				Some(header) => {
-					let num = HeaderView::new(&header).number();
-					if req.max == 1 || self.0.client.block_hash(BlockId::Number(num)) != Some(hash) {
-						// Non-canonical header or single header requested.
-						return vec![header];
-					}
+	fn block_body(&self, id: BlockId) -> Option<Bytes> {
+		self.0.client.block_body(id)
+	}
 
-					num
-				}
+	fn block_receipts(&self, hash: &H256) -> Option<Bytes> {
+		self.0.client.block_receipts(&hash)
+	}
+
+	fn state_proof(&self, req: request::StateProof) -> Vec<Bytes> {
+		match req.key2 {
+			Some(_) => vec![::util::sha3::SHA3_NULL_RLP.to_vec()],
+			None => {
+				// sort of a leaf node
+				let mut stream = RlpStream::new_list(2);
+				stream.append(&req.key1).append_empty_data();
+				vec![stream.out()]
 			}
-		};
-
-		(0u64..req.max as u64)
-			.map(|x: u64| x.saturating_mul(req.skip + 1))
-			.take_while(|x| if req.reverse { x < &start_num } else { best_num - start_num >= *x })
-			.map(|x| if req.reverse { start_num - x } else { start_num + x })
-			.map(|x| self.0.client.block_header(BlockId::Number(x)))
-			.take_while(|x| x.is_some())
-			.flat_map(|x| x)
-			.collect()
+		}
 	}
 
-	fn block_bodies(&self, req: request::Bodies) -> Vec<Bytes> {
-		req.block_hashes.into_iter()
-			.map(|hash| self.0.client.block_body(BlockId::Hash(hash)))
-			.map(|body| body.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
-			.collect()
+	fn contract_code(&self, req: request::ContractCode) -> Bytes {
+		req.account_key.iter().chain(req.account_key.iter()).cloned().collect()
 	}
 
-	fn receipts(&self, req: request::Receipts) -> Vec<Bytes> {
-		req.block_hashes.into_iter()
-			.map(|hash| self.0.client.block_receipts(&hash))
-			.map(|receipts| receipts.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
-			.collect()
-	}
-
-	fn proofs(&self, req: request::StateProofs) -> Vec<Bytes> {
-		req.requests.into_iter()
-			.map(|req| {
-				match req.key2 {
-					Some(_) => ::util::sha3::SHA3_NULL_RLP.to_vec(),
-					None => {
-						// sort of a leaf node
-						let mut stream = RlpStream::new_list(2);
-						stream.append(&req.key1).append_empty_data();
-						stream.out()
-					}
-				}
-			})
-			.collect()
-	}
-
-	fn contract_code(&self, req: request::ContractCodes) -> Vec<Bytes> {
-		req.code_requests.into_iter()
-			.map(|req| {
-                req.account_key.iter().chain(req.account_key.iter()).cloned().collect()
-			 })
-			.collect()
-	}
-
-	fn header_proofs(&self, req: request::HeaderProofs) -> Vec<Bytes> {
-		req.requests.into_iter().map(|_| ::rlp::EMPTY_LIST_RLP.to_vec()).collect()
+	fn header_proof(&self, _req: request::HeaderProof) -> Option<(Bytes, Vec<Bytes>)> {
+		None
 	}
 
 	fn ready_transactions(&self) -> Vec<PendingTransaction> {
@@ -455,8 +412,8 @@ fn get_state_proofs() {
 	let request_body = encode_request(&request, req_id);
 	let response = {
 		let proofs = vec![
-			{ let mut stream = RlpStream::new_list(2); stream.append(&key1).append_empty_data(); stream.out() },
-			::util::sha3::SHA3_NULL_RLP.to_vec(),
+			{ let mut stream = RlpStream::new_list(2); stream.append(&key1).append_empty_data(); vec![stream.out()] },
+			vec![::util::sha3::SHA3_NULL_RLP.to_vec()],
 		];
 
 		let new_buf = *flow_params.limit() - flow_params.compute_cost(request::Kind::StateProofs, 2);
@@ -465,7 +422,10 @@ fn get_state_proofs() {
 
 		response_stream.append(&req_id).append(&new_buf).begin_list(2);
 		for proof in proofs {
-			response_stream.append_raw(&proof, 1);
+			response_stream.begin_list(proof.len());
+			for node in proof {
+				response_stream.append_raw(&node, 1);
+			}
 		}
 
 		response_stream.out()

ethcore/light/src/provider.rs

@@ -52,31 +52,139 @@ pub trait Provider: Send + Sync {
 	///
 	/// The returned vector may have any length in the range [0, `max`], but the
 	/// results within must adhere to the `skip` and `reverse` parameters.
-	fn block_headers(&self, req: request::Headers) -> Vec<Bytes>;
+	fn block_headers(&self, req: request::Headers) -> Vec<Bytes> {
+		use request::HashOrNumber;
+		use ethcore::views::HeaderView;
+
+		if req.max == 0 { return Vec::new() }
+
+		let best_num = self.chain_info().best_block_number;
+		let start_num = match req.start {
+			HashOrNumber::Number(start_num) => start_num,
+			HashOrNumber::Hash(hash) => match self.block_header(BlockId::Hash(hash)) {
+				None => {
+					trace!(target: "les_provider", "Unknown block hash {} requested", hash);
+					return Vec::new();
+				}
+				Some(header) => {
+					let num = HeaderView::new(&header).number();
+					let canon_hash = self.block_header(BlockId::Number(num))
+						.map(|h| HeaderView::new(&h).hash());
+
+					if req.max == 1 || canon_hash != Some(hash) {
+						// Non-canonical header or single header requested.
+						return vec![header];
+					}
+
+					num
+				}
+			}
+		};
+
+		(0u64..req.max as u64)
+			.map(|x: u64| x.saturating_mul(req.skip + 1))
+			.take_while(|x| if req.reverse { x < &start_num } else { best_num - start_num >= *x })
+			.map(|x| if req.reverse { start_num - x } else { start_num + x })
+			.map(|x| self.block_header(BlockId::Number(x)))
+			.take_while(|x| x.is_some())
+			.flat_map(|x| x)
+			.collect()
+	}
+
+	/// Get a block header by id.
+	fn block_header(&self, id: BlockId) -> Option<Bytes>;
 
 	/// Provide as many as possible of the requested blocks (minus the headers) encoded
 	/// in RLP format.
-	fn block_bodies(&self, req: request::Bodies) -> Vec<Bytes>;
+	fn block_bodies(&self, req: request::Bodies) -> Vec<Bytes> {
+		req.block_hashes.into_iter()
+			.map(|hash| self.block_body(BlockId::Hash(hash)))
+			.map(|body| body.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
+			.collect()
+	}
+
+	/// Get a block body by id.
+	fn block_body(&self, id: BlockId) -> Option<Bytes>;
 
 	/// Provide the receipts as many as possible of the requested blocks.
 	/// Returns a vector of RLP-encoded lists of receipts.
-	fn receipts(&self, req: request::Receipts) -> Vec<Bytes>;
+	fn receipts(&self, req: request::Receipts) -> Vec<Bytes> {
+		req.block_hashes.into_iter()
+			.map(|hash| self.block_receipts(&hash))
+			.map(|receipts| receipts.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
+			.collect()
+	}
+
+	/// Get a block's receipts as an RLP-encoded list by block hash.
+	fn block_receipts(&self, hash: &H256) -> Option<Bytes>;
 
 	/// Provide a set of merkle proofs, as requested. Each request is a
 	/// block hash and request parameters.
 	///
 	/// Returns a vector of RLP-encoded lists satisfying the requests.
-	fn proofs(&self, req: request::StateProofs) -> Vec<Bytes>;
+	fn proofs(&self, req: request::StateProofs) -> Vec<Bytes> {
+		use rlp::{RlpStream, Stream};
+
+		let mut results = Vec::with_capacity(req.requests.len());
+
+		for request in req.requests {
+			let proof = self.state_proof(request);
+
+			let mut stream = RlpStream::new_list(proof.len());
+			for node in proof {
+				stream.append_raw(&node, 1);
+			}
+
+			results.push(stream.out());
+		}
+
+		results
+	}
+
+	/// Get a state proof from a request. Each proof should be a vector
+	/// of rlp-encoded trie nodes, in ascending order by distance from the root.
+	fn state_proof(&self, req: request::StateProof) -> Vec<Bytes>;
 
 	/// Provide contract code for the specified (block_hash, account_hash) pairs.
 	/// Each item in the resulting vector is either the raw bytecode or empty.
-	fn contract_code(&self, req: request::ContractCodes) -> Vec<Bytes>;
+	fn contract_codes(&self, req: request::ContractCodes) -> Vec<Bytes> {
+		req.code_requests.into_iter()
+			.map(|req| self.contract_code(req))
+			.collect()
+	}
 
-	/// Provide header proofs from the Canonical Hash Tries as well as the headers 
+	/// Get contract code by request. Either the raw bytecode or empty.
+	fn contract_code(&self, req: request::ContractCode) -> Bytes;
+
+	/// Provide header proofs from the Canonical Hash Tries as well as the headers
 	/// they correspond to -- each element in the returned vector is a 2-tuple.
-	/// The first element is a block header and the second a merkle proof of 
+	/// The first element is a block header and the second a merkle proof of
 	/// the header in a requested CHT.
-	fn header_proofs(&self, req: request::HeaderProofs) -> Vec<Bytes>;
+	fn header_proofs(&self, req: request::HeaderProofs) -> Vec<Bytes> {
+		use rlp::{self, RlpStream, Stream};
+
+		req.requests.into_iter()
+			.map(|req| self.header_proof(req))
+			.map(|maybe_proof| match maybe_proof {
+				None => rlp::EMPTY_LIST_RLP.to_vec(),
+				Some((header, proof)) => {
+					let mut stream = RlpStream::new_list(2);
+					stream.append_raw(&header, 1).begin_list(proof.len());
+
+					for node in proof {
+						stream.append_raw(&node, 1);
+					}
+
+					stream.out()
+				}
+			})
+			.collect()
+	}
+
+	/// Provide a header proof from a given Canonical Hash Trie as well as the
+	/// corresponding header. The first element is the block header and the
+	/// second is a merkle proof of the CHT.
+	fn header_proof(&self, req: request::HeaderProof) -> Option<(Bytes, Vec<Bytes>)>;
 
 	/// Provide pending transactions.
 	fn ready_transactions(&self) -> Vec<PendingTransaction>;
@@ -96,86 +204,31 @@ impl<T: ProvingBlockChainClient + ?Sized> Provider for T {
 		Some(self.pruning_info().earliest_state)
 	}
 
-	fn block_headers(&self, req: request::Headers) -> Vec<Bytes> {
-		use request::HashOrNumber;
-		use ethcore::views::HeaderView;
-
-		let best_num = self.chain_info().best_block_number;
-		let start_num = match req.start {
-			HashOrNumber::Number(start_num) => start_num,
-			HashOrNumber::Hash(hash) => match self.block_header(BlockId::Hash(hash)) {
-				None => {
-					trace!(target: "les_provider", "Unknown block hash {} requested", hash);
-					return Vec::new();
-				}
-				Some(header) => {
-					let num = HeaderView::new(&header).number();
-					if req.max == 1 || self.block_hash(BlockId::Number(num)) != Some(hash) {
-						// Non-canonical header or single header requested.
-						return vec![header];
-					}
-
-					num
-				}
-			}
-		};
-		
-		(0u64..req.max as u64)
-			.map(|x: u64| x.saturating_mul(req.skip + 1))
-			.take_while(|x| if req.reverse { x < &start_num } else { best_num - start_num >= *x })
-			.map(|x| if req.reverse { start_num - x } else { start_num + x })
-			.map(|x| self.block_header(BlockId::Number(x)))
-			.take_while(|x| x.is_some())
-			.flat_map(|x| x)
-			.collect()
+	fn block_header(&self, id: BlockId) -> Option<Bytes> {
+		BlockChainClient::block_header(self, id)
 	}
 
-	fn block_bodies(&self, req: request::Bodies) -> Vec<Bytes> {
-		req.block_hashes.into_iter()
-			.map(|hash| self.block_body(BlockId::Hash(hash)))
-			.map(|body| body.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
-			.collect()
+	fn block_body(&self, id: BlockId) -> Option<Bytes> {
+		BlockChainClient::block_body(self, id)
 	}
 
-	fn receipts(&self, req: request::Receipts) -> Vec<Bytes> {
-		req.block_hashes.into_iter()
-			.map(|hash| self.block_receipts(&hash))
-			.map(|receipts| receipts.unwrap_or_else(|| ::rlp::EMPTY_LIST_RLP.to_vec()))
-			.collect()
+	fn block_receipts(&self, hash: &H256) -> Option<Bytes> {
+		BlockChainClient::block_receipts(self, hash)
 	}
 
-	fn proofs(&self, req: request::StateProofs) -> Vec<Bytes> {
-		use rlp::{RlpStream, Stream};
-
-		let mut results = Vec::with_capacity(req.requests.len());
-
-		for request in req.requests {
-			let proof = match request.key2 {
-				Some(key2) => self.prove_storage(request.key1, key2, request.from_level, BlockId::Hash(request.block)),
-				None => self.prove_account(request.key1, request.from_level, BlockId::Hash(request.block)),
-			};
-
-			let mut stream = RlpStream::new_list(proof.len());
-			for node in proof {
-				stream.append_raw(&node, 1);
-			}
-
-			results.push(stream.out());
+	fn state_proof(&self, req: request::StateProof) -> Vec<Bytes> {
+		match req.key2 {
+			Some(key2) => self.prove_storage(req.key1, key2, req.from_level, BlockId::Hash(req.block)),
+			None => self.prove_account(req.key1, req.from_level, BlockId::Hash(req.block)),
 		}
-
-		results
 	}
 
-	fn contract_code(&self, req: request::ContractCodes) -> Vec<Bytes> {
-		req.code_requests.into_iter()
-			.map(|req| {
-				self.code_by_hash(req.account_key, BlockId::Hash(req.block_hash))
-			})
-			.collect()
+	fn contract_code(&self, req: request::ContractCode) -> Bytes {
+		self.code_by_hash(req.account_key, BlockId::Hash(req.block_hash))
 	}
 
-	fn header_proofs(&self, req: request::HeaderProofs) -> Vec<Bytes> {
-		req.requests.into_iter().map(|_| ::rlp::EMPTY_LIST_RLP.to_vec()).collect()
+	fn header_proof(&self, _req: request::HeaderProof) -> Option<(Bytes, Vec<Bytes>)> {
+		None
 	}
 
 	fn ready_transactions(&self) -> Vec<PendingTransaction> {

ethcore/src/verification/queue/mod.rs

@@ -137,7 +137,7 @@ pub struct VerificationQueue<K: Kind> {
 	max_queue_size: usize,
 	max_mem_use: usize,
 	scale_verifiers: bool,
-	verifier_handles: Vec<JoinHandle<()>>,	
+	verifier_handles: Vec<JoinHandle<()>>,
 	state: Arc<(Mutex<State>, Condvar)>,
 }
 
@@ -225,8 +225,8 @@ impl<K: Kind> VerificationQueue<K> {
 
 		let num_cpus = ::num_cpus::get();
 		let max_verifiers = min(num_cpus, MAX_VERIFIERS);
-		let default_amount = max(1, min(max_verifiers, config.verifier_settings.num_verifiers));		
-		let state = Arc::new((Mutex::new(State::Work(default_amount)), Condvar::new()));		
+		let default_amount = max(1, min(max_verifiers, config.verifier_settings.num_verifiers));
+		let state = Arc::new((Mutex::new(State::Work(default_amount)), Condvar::new()));
 		let mut verifier_handles = Vec::with_capacity(max_verifiers);
 
 		debug!(target: "verification", "Allocating {} verifiers, {} initially active", max_verifiers, default_amount);
@@ -248,11 +248,11 @@ impl<K: Kind> VerificationQueue<K> {
 				.spawn(move || {
 					panic_handler.catch_panic(move || {
 						VerificationQueue::verify(
-							verification, 
-							engine, 
-							wait, 
-							ready, 
-							empty, 
+							verification,
+							engine,
+							wait,
+							ready,
+							empty,
 							state,
 							i,
 						)
@@ -299,11 +299,11 @@ impl<K: Kind> VerificationQueue<K> {
 
 					debug!(target: "verification", "verifier {} sleeping", id);
 					state.1.wait(&mut cur_state);
-					debug!(target: "verification", "verifier {} waking up", id);					
+					debug!(target: "verification", "verifier {} waking up", id);
 				}
 
-				if let State::Exit = *cur_state { 
-					debug!(target: "verification", "verifier {} exiting", id);										
+				if let State::Exit = *cur_state {
+					debug!(target: "verification", "verifier {} exiting", id);
 					break;
 				}
 			}
@@ -326,7 +326,7 @@ impl<K: Kind> VerificationQueue<K> {
 				}
 
 				if let State::Exit = *state.0.lock() {
-					debug!(target: "verification", "verifier {} exiting", id);										
+					debug!(target: "verification", "verifier {} exiting", id);
 					return;
 				}
 			}
@@ -687,8 +687,12 @@ impl<K: Kind> Drop for VerificationQueue<K> {
 		*self.state.0.lock() = State::Exit;
 		self.state.1.notify_all();
 
-		// wake up all threads waiting for more work.
-		self.more_to_verify.notify_all();
+		// acquire this lock to force threads to reach the waiting point
+		// if they're in-between the exit check and the more_to_verify wait.
+		{
+			let _more = self.verification.more_to_verify.lock().unwrap();
+			self.more_to_verify.notify_all();
+		}
 
 		// wait for all verifier threads to join.
 		for thread in self.verifier_handles.drain(..) {
@@ -817,7 +821,7 @@ mod tests {
 	fn readjust_verifiers() {
 		let queue = get_test_queue(true);
 
-		// put all the verifiers to sleep to ensure 
+		// put all the verifiers to sleep to ensure
 		// the test isn't timing sensitive.
 		*queue.state.0.lock() = State::Work(0);