From 3915943f572ed01262ce0efaa76dcb35e5218702 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 29 Mar 2017 19:59:20 +0200 Subject: [PATCH 1/2] Kovan warp sync fixed --- Cargo.lock | 12 ++++++--- ethcore/src/block.rs | 10 ++++++++ ethcore/src/client/client.rs | 8 ++++-- ethcore/src/state/mod.rs | 2 +- ethcore/src/verification/canon_verifier.rs | 4 +-- ethcore/src/verification/noop_verifier.rs | 2 +- ethcore/src/verification/verification.rs | 4 +-- ethcore/src/verification/verifier.rs | 2 +- sync/Cargo.toml | 1 + sync/src/blocks.rs | 30 ++++++++++++---------- sync/src/chain.rs | 6 +++++ sync/src/lib.rs | 1 + 12 files changed, 56 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8845c4f64..40d73d5ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,7 +558,7 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.1.0", - "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -794,7 +794,7 @@ dependencies = [ "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -821,6 +821,7 @@ dependencies = [ "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.1.0", "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2196,8 +2197,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "smallvec" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "spmc" @@ -2788,7 +2792,7 @@ dependencies = [ "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum smallvec 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fcc8d19212aacecf95e4a7a2179b26f7aeb9732a915cf01f05b0d3e044865410" "checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" -"checksum smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3c84984c278afe61a46e19868e8b23e2ee3be5b3cc6dea6edad4893bc6c841" +"checksum smallvec 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dca03f2f42500a9ef8ac0d16183dff8bed40e3dcf98f9d4147928548d5c4236e" "checksum spmc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "93bdab61c1a413e591c4d17388ffa859eaff2df27f1e13a5ec8b716700605adf" "checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 38278af05..12e77d41b 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -492,6 +492,16 @@ impl LockedBlock { _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } } + + /// Remove state root from transaction receipts to make them EIP-98 compatible. + pub fn strip_receipts(self) -> LockedBlock { + let mut block = self; + for receipt in &mut block.block.receipts { + receipt.state_root = None; + } + block.block.header.set_receipts_root(ordered_trie_root(block.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()))); + block + } } impl Drain for LockedBlock { diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 12f92177c..9da044573 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -388,12 +388,16 @@ impl Client { let db = self.state_db.lock().boxed_clone_canon(header.parent_hash()); let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone()); - let locked_block = enact_result.map_err(|e| { + let mut locked_block = enact_result.map_err(|e| { warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); })?; + if header.number() < self.engine().params().validate_receipts_transition && header.receipts_root() != locked_block.block().header().receipts_root() { + locked_block = locked_block.strip_receipts(); + } + // Final Verification - if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header(), self.engine().params().validate_receipts_transition) { + if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header()) { warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 26d33d152..b1594dd9b 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -601,7 +601,7 @@ impl State { let e = self.execute(env_info, engine, t, tracing)?; // trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod())); - let state_root = if env_info.number < engine.params().eip98_transition { + let state_root = if env_info.number < engine.params().eip98_transition || env_info.number < engine.params().validate_receipts_transition { self.commit()?; Some(self.root().clone()) } else { diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs index 188d94a10..849f7caad 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/src/verification/canon_verifier.rs @@ -31,7 +31,7 @@ impl Verifier for CanonVerifier { verification::verify_block_family(header, bytes, engine, bc) } - fn verify_block_final(&self, expected: &Header, got: &Header, receipts: u64) -> Result<(), Error> { - verification::verify_block_final(expected, got, receipts) + fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error> { + verification::verify_block_final(expected, got) } } diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index 0512c8208..2fcd877f5 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -31,7 +31,7 @@ impl Verifier for NoopVerifier { Ok(()) } - fn verify_block_final(&self, _expected: &Header, _got: &Header, _receipts: u64) -> Result<(), Error> { + fn verify_block_final(&self, _expected: &Header, _got: &Header) -> Result<(), Error> { Ok(()) } } diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 2e913a305..096b348c3 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -178,7 +178,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: & } /// Phase 4 verification. Check block information against transaction enactment results, -pub fn verify_block_final(expected: &Header, got: &Header, check_receipts: u64) -> Result<(), Error> { +pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> { if expected.gas_used() != got.gas_used() { return Err(From::from(BlockError::InvalidGasUsed(Mismatch { expected: expected.gas_used().clone(), found: got.gas_used().clone() }))) } @@ -188,7 +188,7 @@ pub fn verify_block_final(expected: &Header, got: &Header, check_receipts: u64) if expected.state_root() != got.state_root() { return Err(From::from(BlockError::InvalidStateRoot(Mismatch { expected: expected.state_root().clone(), found: got.state_root().clone() }))) } - if got.number() >= check_receipts && expected.receipts_root() != got.receipts_root() { + if expected.receipts_root() != got.receipts_root() { return Err(From::from(BlockError::InvalidReceiptsRoot(Mismatch { expected: expected.receipts_root().clone(), found: got.receipts_root().clone() }))) } Ok(()) diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs index 08ec8c25a..e5dabd392 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/src/verification/verifier.rs @@ -26,5 +26,5 @@ pub trait Verifier: Send + Sync { /// Verify a block relative to its parent and uncles. fn verify_block_family(&self, header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>; /// Do a final verification check for an enacted header vs its expected counterpart. - fn verify_block_final(&self, expected: &Header, got: &Header, receipts: u64) -> Result<(), Error>; + fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error>; } diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 778563367..3869b9885 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -26,6 +26,7 @@ rand = "0.3.13" heapsize = "0.3" ethcore-ipc = { path = "../ipc/rpc" } semver = "0.6" +smallvec = { version = "0.3", features = ["heapsizeof"] } ethcore-ipc-nano = { path = "../ipc/nano" } ethcore-devtools = { path = "../devtools" } ethkey = { path = "../ethkey" } diff --git a/sync/src/blocks.rs b/sync/src/blocks.rs index d7b0062da..b1daf11e8 100644 --- a/sync/src/blocks.rs +++ b/sync/src/blocks.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::collections::hash_map::Entry; +use smallvec::SmallVec; use util::*; use rlp::*; use network::NetworkError; @@ -21,6 +23,8 @@ use ethcore::header::Header as BlockHeader; known_heap_size!(0, HeaderId); +type SmallHashVec = SmallVec<[H256; 1]>; + /// Block data with optional body. struct SyncBlock { header: Bytes, @@ -64,8 +68,8 @@ pub struct BlockCollection { parents: HashMap, /// Used to map body to header. header_ids: HashMap, - /// Used to map receipts root to header. - receipt_ids: HashMap, + /// Used to map receipts root to headers. + receipt_ids: HashMap, /// First block in `blocks`. head: Option, /// Set of block header hashes being downloaded @@ -202,7 +206,7 @@ impl BlockCollection { } } } - for h in self.receipt_ids.values() { + for h in self.receipt_ids.values().flat_map(|hashes| hashes) { if needed_receipts.len() >= count { break; } @@ -366,23 +370,26 @@ impl BlockCollection { let receipts = UntrustedRlp::new(&r); ordered_trie_root(receipts.iter().map(|r| r.as_raw().to_vec())) //TODO: get rid of vectors here }; - match self.receipt_ids.get(&receipt_root).cloned() { - Some(h) => { - self.receipt_ids.remove(&receipt_root); + match self.receipt_ids.entry(receipt_root) { + Entry::Occupied(mut entry) => { + let h = entry.get_mut().pop().expect("Empty vectors are not allowed in insert_receipt; qed"); + if entry.get().is_empty() { + entry.remove(); + } self.downloading_receipts.remove(&h); match self.blocks.get_mut(&h) { Some(ref mut block) => { trace!(target: "sync", "Got receipt {}", h); - block.receipts = Some(r); + block.receipts = Some(r.clone()); Ok(()) }, None => { - warn!("Got receipt with no header {}", h); + debug!("Got receipt with no header {}", h); Err(NetworkError::BadProtocol) } } } - None => { + _ => { trace!(target: "sync", "Ignored unknown/stale block receipt {:?}", receipt_root); Err(NetworkError::BadProtocol) } @@ -429,10 +436,7 @@ impl BlockCollection { let receipts_stream = RlpStream::new_list(0); block.receipts = Some(receipts_stream.out()); } else { - if self.receipt_ids.contains_key(&receipt_root) { - warn!(target: "sync", "Duplicate receipt root {:?}, block: {:?}", receipt_root, hash); - } - self.receipt_ids.insert(receipt_root, hash.clone()); + self.receipt_ids.entry(receipt_root).or_insert_with(|| SmallHashVec::new()).push(hash.clone()); } } diff --git a/sync/src/chain.rs b/sync/src/chain.rs index ca5172fc2..3c1717620 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -379,6 +379,8 @@ pub struct ChainSync { transactions_stats: TransactionsStats, /// Enable ancient block downloading download_old_blocks: bool, + /// Enable warp sync. + enable_warp_sync: bool, } type RlpResponseResult = Result, PacketDecodeError>; @@ -403,6 +405,7 @@ impl ChainSync { snapshot: Snapshot::new(), sync_start_time: None, transactions_stats: TransactionsStats::default(), + enable_warp_sync: config.warp_sync, }; sync.update_targets(chain); sync @@ -501,6 +504,9 @@ impl ChainSync { } fn maybe_start_snapshot_sync(&mut self, io: &mut SyncIo) { + if !self.enable_warp_sync { + return; + } if self.state != SyncState::WaitingPeers && self.state != SyncState::Blocks && self.state != SyncState::Waiting { return; } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 6cd4fade5..b51c157be 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -35,6 +35,7 @@ extern crate time; extern crate rand; extern crate semver; extern crate parking_lot; +extern crate smallvec; extern crate rlp; extern crate ethcore_light as light; From deef600cafded0fe3ba41eef44a4eb34ef7ff9c6 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 31 Mar 2017 14:12:15 +0200 Subject: [PATCH 2/2] Download unique receipts only --- sync/src/blocks.rs | 52 ++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/sync/src/blocks.rs b/sync/src/blocks.rs index b1daf11e8..3dcc912a7 100644 --- a/sync/src/blocks.rs +++ b/sync/src/blocks.rs @@ -30,6 +30,7 @@ struct SyncBlock { header: Bytes, body: Option, receipts: Option, + receipts_root: H256, } /// Block with optional receipt @@ -76,7 +77,7 @@ pub struct BlockCollection { downloading_headers: HashSet, /// Set of block bodies being downloaded identified by block hash. downloading_bodies: HashSet, - /// Set of block receipts being downloaded identified by block hash. + /// Set of block receipts being downloaded identified by receipt root. downloading_receipts: HashSet, } @@ -198,21 +199,24 @@ impl BlockCollection { head = self.parents.get(&head.unwrap()).cloned(); if let Some(head) = head { match self.blocks.get(&head) { - Some(block) if block.receipts.is_none() && !self.downloading_receipts.contains(&head) => { - self.downloading_receipts.insert(head.clone()); - needed_receipts.push(head.clone()); + Some(block) => { + if block.receipts.is_none() && !self.downloading_receipts.contains(&block.receipts_root) { + self.downloading_receipts.insert(block.receipts_root); + needed_receipts.push(head.clone()); + } } _ => (), } } } - for h in self.receipt_ids.values().flat_map(|hashes| hashes) { + // If there are multiple blocks per receipt, only request one of them. + for (root, h) in self.receipt_ids.iter().map(|(root, hashes)| (root, hashes[0])) { if needed_receipts.len() >= count { break; } - if !self.downloading_receipts.contains(h) { + if !self.downloading_receipts.contains(root) { needed_receipts.push(h.clone()); - self.downloading_receipts.insert(h.clone()); + self.downloading_receipts.insert(*root); } } needed_receipts @@ -249,7 +253,9 @@ impl BlockCollection { /// Unmark block receipt as being downloaded. pub fn clear_receipt_download(&mut self, hashes: &[H256]) { for h in hashes { - self.downloading_receipts.remove(h); + if let Some(ref block) = self.blocks.get(h) { + self.downloading_receipts.remove(&block.receipts_root); + } } } @@ -370,24 +376,22 @@ impl BlockCollection { let receipts = UntrustedRlp::new(&r); ordered_trie_root(receipts.iter().map(|r| r.as_raw().to_vec())) //TODO: get rid of vectors here }; + self.downloading_receipts.remove(&receipt_root); match self.receipt_ids.entry(receipt_root) { - Entry::Occupied(mut entry) => { - let h = entry.get_mut().pop().expect("Empty vectors are not allowed in insert_receipt; qed"); - if entry.get().is_empty() { - entry.remove(); - } - self.downloading_receipts.remove(&h); - match self.blocks.get_mut(&h) { - Some(ref mut block) => { - trace!(target: "sync", "Got receipt {}", h); - block.receipts = Some(r.clone()); - Ok(()) - }, - None => { - debug!("Got receipt with no header {}", h); - Err(NetworkError::BadProtocol) + Entry::Occupied(entry) => { + for h in entry.remove() { + match self.blocks.get_mut(&h) { + Some(ref mut block) => { + trace!(target: "sync", "Got receipt {}", h); + block.receipts = Some(r.clone()); + }, + None => { + warn!("Got receipt with no header {}", h); + return Err(NetworkError::BadProtocol) + } } } + Ok(()) } _ => { trace!(target: "sync", "Ignored unknown/stale block receipt {:?}", receipt_root); @@ -414,6 +418,7 @@ impl BlockCollection { header: header, body: None, receipts: None, + receipts_root: H256::new(), }; let header_id = HeaderId { transactions_root: info.transactions_root().clone(), @@ -438,6 +443,7 @@ impl BlockCollection { } else { self.receipt_ids.entry(receipt_root).or_insert_with(|| SmallHashVec::new()).push(hash.clone()); } + block.receipts_root = receipt_root; } self.parents.insert(info.parent_hash().clone(), hash.clone());