Merge pull request #5337 from paritytech/kovan-receipt-fix

Kovan warp sync fixed

Cargo.lock

@@ -569,7 +569,7 @@ dependencies = [
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.1.0",
- "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "stats 0.1.0",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -805,7 +805,7 @@ dependencies = [
  "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -832,6 +832,7 @@ dependencies = [
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.1.0",
  "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2217,8 +2218,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "smallvec"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "spmc"
@@ -2811,7 +2815,7 @@ dependencies = [
 "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23"
 "checksum smallvec 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fcc8d19212aacecf95e4a7a2179b26f7aeb9732a915cf01f05b0d3e044865410"
 "checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013"
-"checksum smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3c84984c278afe61a46e19868e8b23e2ee3be5b3cc6dea6edad4893bc6c841"
+"checksum smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dca03f2f42500a9ef8ac0d16183dff8bed40e3dcf98f9d4147928548d5c4236e"
 "checksum spmc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "93bdab61c1a413e591c4d17388ffa859eaff2df27f1e13a5ec8b716700605adf"
 "checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b"
 "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"

ethcore/src/block.rs

@@ -492,6 +492,16 @@ impl LockedBlock {
 			_ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }),
 		}
 	}
+
+	/// Remove state root from transaction receipts to make them EIP-98 compatible.
+	pub fn strip_receipts(self) -> LockedBlock {
+		let mut block = self;
+		for receipt in &mut block.block.receipts {
+			receipt.state_root = None;
+		}
+		block.block.header.set_receipts_root(ordered_trie_root(block.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
+		block
+	}
 }
 
 impl Drain for LockedBlock {

ethcore/src/client/client.rs

@@ -388,12 +388,16 @@ impl Client {
 			let db = self.state_db.lock().boxed_clone_canon(header.parent_hash());
 
 			let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
-			let locked_block = enact_result.map_err(|e| {
+			let mut locked_block = enact_result.map_err(|e| {
 				warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			})?;
 
+			if header.number() < self.engine().params().validate_receipts_transition && header.receipts_root() != locked_block.block().header().receipts_root() {
+				locked_block = locked_block.strip_receipts();
+			}
+	
 			// Final Verification
-			if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header(), self.engine().params().validate_receipts_transition) {
+			if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header()) {
 				warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 				return Err(());
 			}

ethcore/src/state/mod.rs

@@ -601,7 +601,7 @@ impl<B: Backend> State<B> {
 
 		let e = self.execute(env_info, engine, t, tracing)?;
 //		trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod()));
-		let state_root = if env_info.number < engine.params().eip98_transition {
+		let state_root = if env_info.number < engine.params().eip98_transition || env_info.number < engine.params().validate_receipts_transition {
 			self.commit()?;
 			Some(self.root().clone())
 		} else {

ethcore/src/verification/canon_verifier.rs

@@ -31,7 +31,7 @@ impl Verifier for CanonVerifier {
 		verification::verify_block_family(header, bytes, engine, bc)
 	}
 
-	fn verify_block_final(&self, expected: &Header, got: &Header, receipts: u64) -> Result<(), Error> {
+	fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error> {
-		verification::verify_block_final(expected, got, receipts)
+		verification::verify_block_final(expected, got)
 	}
 }

ethcore/src/verification/noop_verifier.rs

@@ -31,7 +31,7 @@ impl Verifier for NoopVerifier {
 		Ok(())
 	}
 
-	fn verify_block_final(&self, _expected: &Header, _got: &Header, _receipts: u64) -> Result<(), Error> {
+	fn verify_block_final(&self, _expected: &Header, _got: &Header) -> Result<(), Error> {
 		Ok(())
 	}
 }

ethcore/src/verification/verification.rs

@@ -178,7 +178,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &
 }
 
 /// Phase 4 verification. Check block information against transaction enactment results,
-pub fn verify_block_final(expected: &Header, got: &Header, check_receipts: u64) -> Result<(), Error> {
+pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> {
 	if expected.gas_used() != got.gas_used() {
 		return Err(From::from(BlockError::InvalidGasUsed(Mismatch { expected: expected.gas_used().clone(), found: got.gas_used().clone() })))
 	}
@@ -188,7 +188,7 @@ pub fn verify_block_final(expected: &Header, got: &Header, check_receipts: u64)
 	if expected.state_root() != got.state_root() {
 		return Err(From::from(BlockError::InvalidStateRoot(Mismatch { expected: expected.state_root().clone(), found: got.state_root().clone() })))
 	}
-	if got.number() >= check_receipts && expected.receipts_root() != got.receipts_root() {
+	if expected.receipts_root() != got.receipts_root() {
 		return Err(From::from(BlockError::InvalidReceiptsRoot(Mismatch { expected: expected.receipts_root().clone(), found: got.receipts_root().clone() })))
 	}
 	Ok(())

ethcore/src/verification/verifier.rs

@@ -26,5 +26,5 @@ pub trait Verifier: Send + Sync {
 	/// Verify a block relative to its parent and uncles.
 	fn verify_block_family(&self, header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>;
 	/// Do a final verification check for an enacted header vs its expected counterpart.
-	fn verify_block_final(&self, expected: &Header, got: &Header, receipts: u64) -> Result<(), Error>;
+	fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error>;
 }

sync/Cargo.toml

@@ -26,6 +26,7 @@ rand = "0.3.13"
 heapsize = "0.3"
 ethcore-ipc = { path = "../ipc/rpc" }
 semver = "0.6"
+smallvec = { version = "0.3", features = ["heapsizeof"] }
 ethcore-ipc-nano = { path = "../ipc/nano" }
 ethcore-devtools = { path = "../devtools" }
 ethkey = { path = "../ethkey" }

sync/src/blocks.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::collections::hash_map::Entry;
+use smallvec::SmallVec;
 use util::*;
 use rlp::*;
 use network::NetworkError;
@@ -21,11 +23,14 @@ use ethcore::header::Header as BlockHeader;
 
 known_heap_size!(0, HeaderId);
 
+type SmallHashVec = SmallVec<[H256; 1]>;
+
 /// Block data with optional body.
 struct SyncBlock {
 	header: Bytes,
 	body: Option<Bytes>,
 	receipts: Option<Bytes>,
+	receipts_root: H256,
 }
 
 /// Block with optional receipt
@@ -64,15 +69,15 @@ pub struct BlockCollection {
 	parents: HashMap<H256, H256>,
 	/// Used to map body to header.
 	header_ids: HashMap<HeaderId, H256>,
-	/// Used to map receipts root to header.
+	/// Used to map receipts root to headers.
-	receipt_ids: HashMap<H256, H256>,
+	receipt_ids: HashMap<H256, SmallHashVec>,
 	/// First block in `blocks`.
 	head: Option<H256>,
 	/// Set of block header hashes being downloaded
 	downloading_headers: HashSet<H256>,
 	/// Set of block bodies being downloaded identified by block hash.
 	downloading_bodies: HashSet<H256>,
-	/// Set of block receipts being downloaded identified by block hash.
+	/// Set of block receipts being downloaded identified by receipt root.
 	downloading_receipts: HashSet<H256>,
 }
 
@@ -194,21 +199,24 @@ impl BlockCollection {
 			head = self.parents.get(&head.unwrap()).cloned();
 			if let Some(head) = head {
 				match self.blocks.get(&head) {
-					Some(block) if block.receipts.is_none() && !self.downloading_receipts.contains(&head) => {
+					Some(block) => {
-						self.downloading_receipts.insert(head.clone());
+						if block.receipts.is_none() && !self.downloading_receipts.contains(&block.receipts_root) {
-						needed_receipts.push(head.clone());
+							self.downloading_receipts.insert(block.receipts_root);
+							needed_receipts.push(head.clone());
+						}
 					}
 					_ => (),
 				}
 			}
 		}
-		for h in self.receipt_ids.values() {
+		// If there are multiple blocks per receipt, only request one of them.
+		for (root, h) in self.receipt_ids.iter().map(|(root, hashes)| (root, hashes[0])) {
 			if needed_receipts.len() >= count {
 				break;
 			}
-			if !self.downloading_receipts.contains(h) {
+			if !self.downloading_receipts.contains(root) {
 				needed_receipts.push(h.clone());
-				self.downloading_receipts.insert(h.clone());
+				self.downloading_receipts.insert(*root);
 			}
 		}
 		needed_receipts
@@ -245,7 +253,9 @@ impl BlockCollection {
 	/// Unmark block receipt as being downloaded.
 	pub fn clear_receipt_download(&mut self, hashes: &[H256]) {
 		for h in hashes {
-			self.downloading_receipts.remove(h);
+			if let Some(ref block) = self.blocks.get(h) {
+				self.downloading_receipts.remove(&block.receipts_root);
+			}
 		}
 	}
 
@@ -366,23 +376,24 @@ impl BlockCollection {
 			let receipts = UntrustedRlp::new(&r);
 			ordered_trie_root(receipts.iter().map(|r| r.as_raw().to_vec())) //TODO: get rid of vectors here
 		};
-		match self.receipt_ids.get(&receipt_root).cloned() {
+		self.downloading_receipts.remove(&receipt_root);
-			Some(h) => {
+		match self.receipt_ids.entry(receipt_root) {
-				self.receipt_ids.remove(&receipt_root);
+			Entry::Occupied(entry) => {
-				self.downloading_receipts.remove(&h);
+				for h in entry.remove() {
-				match self.blocks.get_mut(&h) {
+					match self.blocks.get_mut(&h) {
-					Some(ref mut block) => {
+						Some(ref mut block) => {
-						trace!(target: "sync", "Got receipt {}", h);
+							trace!(target: "sync", "Got receipt {}", h);
-						block.receipts = Some(r);
+							block.receipts = Some(r.clone());
-						Ok(())
+						},
-					},
+						None => {
-					None => {
+							warn!("Got receipt with no header {}", h);
-						warn!("Got receipt with no header {}", h);
+							return Err(NetworkError::BadProtocol)
-						Err(NetworkError::BadProtocol)
+						}
 					}
 				}
+				Ok(())
 			}
-			None => {
+			_ => {
 				trace!(target: "sync", "Ignored unknown/stale block receipt {:?}", receipt_root);
 				Err(NetworkError::BadProtocol)
 			}
@@ -407,6 +418,7 @@ impl BlockCollection {
 			header: header,
 			body: None,
 			receipts: None,
+			receipts_root: H256::new(),
 		};
 		let header_id = HeaderId {
 			transactions_root: info.transactions_root().clone(),
@@ -429,11 +441,9 @@ impl BlockCollection {
 				let receipts_stream = RlpStream::new_list(0);
 				block.receipts = Some(receipts_stream.out());
 			} else {
-				if self.receipt_ids.contains_key(&receipt_root) {
+				self.receipt_ids.entry(receipt_root).or_insert_with(|| SmallHashVec::new()).push(hash.clone());
-					warn!(target: "sync", "Duplicate receipt root {:?}, block: {:?}", receipt_root, hash);
-				}
-				self.receipt_ids.insert(receipt_root, hash.clone());
 			}
+			block.receipts_root = receipt_root;
 		}
 
 		self.parents.insert(info.parent_hash().clone(), hash.clone());

sync/src/chain.rs

@@ -379,6 +379,8 @@ pub struct ChainSync {
 	transactions_stats: TransactionsStats,
 	/// Enable ancient block downloading
 	download_old_blocks: bool,
+	/// Enable warp sync.
+	enable_warp_sync: bool,
 }
 
 type RlpResponseResult = Result<Option<(PacketId, RlpStream)>, PacketDecodeError>;
@@ -403,6 +405,7 @@ impl ChainSync {
 			snapshot: Snapshot::new(),
 			sync_start_time: None,
 			transactions_stats: TransactionsStats::default(),
+			enable_warp_sync: config.warp_sync,
 		};
 		sync.update_targets(chain);
 		sync
@@ -501,6 +504,9 @@ impl ChainSync {
 	}
 
 	fn maybe_start_snapshot_sync(&mut self, io: &mut SyncIo) {
+		if !self.enable_warp_sync {
+			return;
+		}
 		if self.state != SyncState::WaitingPeers && self.state != SyncState::Blocks && self.state != SyncState::Waiting {
 			return;
 		}

sync/src/lib.rs

@@ -35,6 +35,7 @@ extern crate time;
 extern crate rand;
 extern crate semver;
 extern crate parking_lot;
+extern crate smallvec;
 extern crate rlp;
 
 extern crate ethcore_light as light;