From a254b2098fd029ee03724e3898a69562e2787bcb Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 11 Apr 2017 17:07:04 +0200 Subject: [PATCH 01/22] more useful Engine::verify_seal --- ethcore/src/block.rs | 11 +++++--- ethcore/src/engines/authority_round.rs | 4 +-- ethcore/src/engines/basic_authority.rs | 2 +- ethcore/src/engines/instant_seal.rs | 2 +- ethcore/src/engines/mod.rs | 36 ++++++++++++++++++++++---- ethcore/src/engines/tendermint/mod.rs | 2 +- ethcore/src/miner/miner.rs | 2 +- ethcore/src/snapshot/mod.rs | 2 +- 8 files changed, 46 insertions(+), 15 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 12e77d41b..6ec2584ec 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -26,7 +26,7 @@ use util::error::{Mismatch, OutOfBounds}; use basic_types::{LogBloom, Seal}; use env_info::{EnvInfo, LastHashes}; -use engines::Engine; +use engines::{Engine, ValidationProof}; use error::{Error, BlockError, TransactionError}; use factory::Factories; use header::Header; @@ -484,10 +484,15 @@ impl LockedBlock { /// Provide a valid seal in order to turn this into a `SealedBlock`. /// This does check the validity of `seal` with the engine. /// Returns the `ClosedBlock` back again if the seal is no good. - pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { + pub fn try_seal( + self, + engine: &Engine, + seal: Vec, + proof: Option, + ) -> Result { let mut s = self; s.block.header.set_seal(seal); - match engine.verify_block_seal(&s.block.header) { + match engine.verify_block_seal(&s.block.header, proof) { Err(e) => Err((e, s)), _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index e6cbdb531..acc276d21 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -464,14 +464,14 @@ mod tests { engine.set_signer(tap.clone(), addr1, "1".into()); if let Seal::Regular(seal) = engine.generate_seal(b1.block()) { - assert!(b1.clone().try_seal(engine, seal).is_ok()); + assert!(b1.clone().try_seal(engine, seal, None).is_ok()); // Second proposal is forbidden. assert!(engine.generate_seal(b1.block()) == Seal::None); } engine.set_signer(tap, addr2, "2".into()); if let Seal::Regular(seal) = engine.generate_seal(b2.block()) { - assert!(b2.clone().try_seal(engine, seal).is_ok()); + assert!(b2.clone().try_seal(engine, seal, None).is_ok()); // Second proposal is forbidden. assert!(engine.generate_seal(b2.block()) == Seal::None); } diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index e5a53d4e9..fe27994e4 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -258,7 +258,7 @@ mod tests { let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); if let Seal::Regular(seal) = engine.generate_seal(b.block()) { - assert!(b.try_seal(engine, seal).is_ok()); + assert!(b.try_seal(engine, seal, None).is_ok()); } } diff --git a/ethcore/src/engines/instant_seal.rs b/ethcore/src/engines/instant_seal.rs index 45bede9f4..5d05dd83f 100644 --- a/ethcore/src/engines/instant_seal.rs +++ b/ethcore/src/engines/instant_seal.rs @@ -89,7 +89,7 @@ mod tests { let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::default(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); if let Seal::Regular(seal) = engine.generate_seal(b.block()) { - assert!(b.try_seal(engine, seal).is_ok()); + assert!(b.try_seal(engine, seal, None).is_ok()); } } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 438b9bda0..76a0d790b 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -33,18 +33,20 @@ pub use self::authority_round::AuthorityRound; pub use self::tendermint::Tendermint; use std::sync::Weak; -use util::*; -use ethkey::Signature; + use account_provider::AccountProvider; use block::ExecutedBlock; use builtin::Builtin; +use client::Client; use env_info::EnvInfo; use error::{Error, TransactionError}; -use spec::CommonParams; use evm::Schedule; use header::Header; +use spec::CommonParams; use transaction::{UnverifiedTransaction, SignedTransaction}; -use client::Client; + +use ethkey::Signature; +use util::*; /// Voting errors. #[derive(Debug)] @@ -59,6 +61,8 @@ pub enum EngineError { UnexpectedMessage, /// Seal field has an unexpected size. BadSealFieldSize(OutOfBounds), + /// Needs a validation proof for the given block hash before verification can continue. + NeedsValidationProof(H256), } impl fmt::Display for EngineError { @@ -70,6 +74,7 @@ impl fmt::Display for EngineError { NotAuthorized(ref address) => format!("Signer {} is not authorized.", address), UnexpectedMessage => "This Engine should not be fed messages.".into(), BadSealFieldSize(ref oob) => format!("Seal field has an unexpected length: {}", oob), + NeedsValidationProof(ref hash) => format!("Needs validation proof of block {} to verify seal.", hash), }; f.write_fmt(format_args!("Engine error ({})", msg)) @@ -87,6 +92,12 @@ pub enum Seal { None, } +/// A validation proof, required for validation of a block header. +pub type ValidationProof = Vec; + +/// Type alias for a function we can make calls through synchronously. +pub type Call = Fn(Address, Bytes) -> Result; + /// A consensus mechanism for the chain. Generally either proof-of-work or proof-of-stake-based. /// Provides hooks into each of the major parts of block import. pub trait Engine : Sync + Send { @@ -180,10 +191,25 @@ pub trait Engine : Sync + Send { /// Verify the seal of a block. This is an auxilliary method that actually just calls other `verify_` methods /// to get the job done. By default it must pass `verify_basic` and `verify_block_unordered`. If more or fewer /// methods are needed for an Engine, this may be overridden. - fn verify_block_seal(&self, header: &Header) -> Result<(), Error> { + fn verify_block_seal(&self, header: &Header, _proof: Option) -> Result<(), Error> { self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) } + /// Generate a validation proof for the given block header. + /// + /// All values queried during execution of given will go into the proof. + /// This may only be called for blocks indicated in "needs validation proof" + /// errors. + /// + /// Engines which don't draw consensus information from the state (e.g. PoW) + /// don't need to change anything here. + /// + /// Engines which do draw consensus information from the state may only do so + /// here. + fn generate_validation_proof(&self, _call: &Call) -> ValidationProof { + ValidationProof::default() + } + /// Populate a header's fields based on its parent's header. /// Usually implements the chain scoring rule based on weight. /// The gas floor target must not be lower than the engine's minimum gas limit. diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index 8c8094117..e88803ce2 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -864,7 +864,7 @@ mod tests { let proposer = insert_and_register(&tap, spec.engine.as_ref(), "1"); let (b, seal) = propose_default(&spec, proposer); - assert!(b.lock().try_seal(spec.engine.as_ref(), seal).is_ok()); + assert!(b.lock().try_seal(spec.engine.as_ref(), seal, None).is_ok()); } #[test] diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index ac1695b52..8337ef2d8 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -1133,7 +1133,7 @@ impl MinerService for Miner { |b| &b.hash() == &block_hash ) { trace!(target: "miner", "Submitted block {}={}={} with seal {:?}", block_hash, b.hash(), b.header().bare_hash(), seal); - b.lock().try_seal(&*self.engine, seal).or_else(|(e, _)| { + b.lock().try_seal(&*self.engine, seal, None).or_else(|(e, _)| { warn!(target: "miner", "Mined solution rejected: {}", e); Err(Error::PowInvalid) }) diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 69dbc943d..0ef263476 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -567,7 +567,7 @@ pub fn verify_old_block(rng: &mut OsRng, header: &Header, engine: &Engine, chain if always || rng.gen::() <= POW_VERIFY_RATE { match chain.block_header(header.parent_hash()) { Some(parent) => engine.verify_block_family(header, &parent, body), - None => engine.verify_block_seal(header), + None => engine.verify_block_seal(header, None), // TODO: fetch validation proof as necessary. } } else { engine.verify_block_basic(header, body) From 7723d6281bcd11c7bd4da101c02fc70e78bbf303 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 12:46:25 +0200 Subject: [PATCH 02/22] starting memoized validatorset --- ethcore/src/engines/mod.rs | 2 +- ethcore/src/engines/validator_set/mod.rs | 41 +++++++++++++++++++ ethcore/src/engines/validator_set/multi.rs | 19 ++++++++- .../src/engines/validator_set/simple_list.rs | 9 ++-- 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 76a0d790b..0ae353a56 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -206,7 +206,7 @@ pub trait Engine : Sync + Send { /// /// Engines which do draw consensus information from the state may only do so /// here. - fn generate_validation_proof(&self, _call: &Call) -> ValidationProof { + fn generate_validation_proof(&self, header: &Header, _call: &Call) -> ValidationProof { ValidationProof::default() } diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index cbbedfb33..a2f6d1075 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -25,11 +25,15 @@ use std::sync::Weak; use util::{Address, H256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; +use receipt::Receipt; + use self::simple_list::SimpleList; use self::contract::ValidatorContract; use self::safe_contract::ValidatorSafeContract; use self::multi::Multi; +pub type Call = Fn(Address, Vec) -> Result, String>; + /// Creates a validator set from spec. pub fn new_validator_set(spec: ValidatorSpec) -> Box { match spec { @@ -42,7 +46,12 @@ pub fn new_validator_set(spec: ValidatorSpec) -> Box { } } +/// A validator set. pub trait ValidatorSet: Send + Sync { + /// Get this validator set as a flexible validator set. + /// Returning `None` indicates this is only usable for + /// full nodes. + fn as_memoized(&self) -> Option<&Memoized> { None } /// Checks if a given address is a validator. fn contains(&self, parent_block_hash: &H256, address: &Address) -> bool; /// Draws an validator nonce modulo number of validators. @@ -56,3 +65,35 @@ pub trait ValidatorSet: Send + Sync { /// Allows blockchain state access. fn register_contract(&self, _client: Weak) {} } + +/// A flexible validator set can track its changes. +pub trait FlexibleValidatorSet: Send + Sync { + /// Whether a validator set may have changed at this header. + fn has_possibly_changed(&self, header: &Header) -> bool; + + /// Whether a validator set has changed at this header, given the block receipts. + /// Called after `has_possibly_changed`. + /// `Some` indicates the validator set has changed at this header and the new + /// expected validator set. + /// `None` indicates no change. + fn has_changed(&self, header: &Header, receipts: &[Receipt]) -> Option; + + /// Fetch validators at a block synchronously. + fn fetch_validators(&self, header: &Header, call: &Call) -> Result; +} + +/// A memoized flexible validator set +pub struct Memoized { + inner: Box, + memo: Mutex<(SimpleList, (H256, u64))>, +} + +impl Memoized { + /// Denote that the + pub fn use_memo_at(&self, list: SimpleList) +} + +impl ValidatorSet for Memoized { + fn as_memoized(&self) -> Option<&Memoized> { Some(self) } +} + diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index 5027f23cd..8527a8aa9 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -40,7 +40,7 @@ impl Multi { } } - fn correct_set(&self, bh: &H256) -> Option<&Box> { + fn correct_set(&self, bh: &H256) -> Option<&ValidatorSet> { match self .block_number .read()(bh) @@ -55,7 +55,7 @@ impl Multi { ) { Ok((block, set)) => { trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); - Some(set) + Some(&*set) }, Err(e) => { debug!(target: "engine", "ValidatorSet could not be recovered: {}", e); @@ -66,6 +66,21 @@ impl Multi { } impl ValidatorSet for Multi { + fn has_possibly_changed(&self, header: &Header) -> bool { + // if the sets are the same for each header, compare those. + // otherwise, the sets have almost certainly changed. + match (self.correct_set(&header.hash()), self.correct_set(header.parent_hash())) { + (Some(a), Some(b)) if a as *const _ == b as *const _ => { a.has_possibly_changed(header) }, + _ => true, + } + } + + fn has_changed(&self, header: &Header, receipts: &[Receipt]) -> Option> { + + } + + fn fetch(&self) -> + fn contains(&self, bh: &H256, address: &Address) -> bool { self.correct_set(bh).map_or(false, |set| set.contains(bh, address)) } diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index 2d7687979..cc4259fac 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -22,13 +22,11 @@ use super::ValidatorSet; #[derive(Debug, PartialEq, Eq, Default)] pub struct SimpleList { validators: Vec
, - validator_n: usize, } impl SimpleList { pub fn new(validators: Vec
) -> Self { SimpleList { - validator_n: validators.len(), validators: validators, } } @@ -36,7 +34,7 @@ impl SimpleList { impl HeapSizeOf for SimpleList { fn heap_size_of_children(&self) -> usize { - self.validators.heap_size_of_children() + self.validator_n.heap_size_of_children() + self.validators.heap_size_of_children() } } @@ -46,11 +44,12 @@ impl ValidatorSet for SimpleList { } fn get(&self, _bh: &H256, nonce: usize) -> Address { - self.validators.get(nonce % self.validator_n).expect("There are validator_n authorities; taking number modulo validator_n gives number in validator_n range; qed").clone() + let validator_n = self.validators.len(); + self.validators.get(nonce % validator_n).expect("There are validator_n authorities; taking number modulo validator_n gives number in validator_n range; qed").clone() } fn count(&self, _bh: &H256) -> usize { - self.validator_n + self.validators.len() } } From 2d8a8bd3e5fcd77e3c81165268e6b4729a85e5bf Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 14:41:19 +0200 Subject: [PATCH 03/22] engine changes --- ethcore/src/block.rs | 5 +-- ethcore/src/engines/authority_round.rs | 4 +- ethcore/src/engines/basic_authority.rs | 2 +- ethcore/src/engines/instant_seal.rs | 2 +- ethcore/src/engines/mod.rs | 52 ++++++++++++++-------- ethcore/src/engines/tendermint/mod.rs | 2 +- ethcore/src/engines/validator_set/mod.rs | 39 ---------------- ethcore/src/engines/validator_set/multi.rs | 17 +------ ethcore/src/miner/miner.rs | 2 +- ethcore/src/snapshot/mod.rs | 2 +- 10 files changed, 43 insertions(+), 84 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 6ec2584ec..8da7c14b2 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -26,7 +26,7 @@ use util::error::{Mismatch, OutOfBounds}; use basic_types::{LogBloom, Seal}; use env_info::{EnvInfo, LastHashes}; -use engines::{Engine, ValidationProof}; +use engines::Engine; use error::{Error, BlockError, TransactionError}; use factory::Factories; use header::Header; @@ -488,11 +488,10 @@ impl LockedBlock { self, engine: &Engine, seal: Vec, - proof: Option, ) -> Result { let mut s = self; s.block.header.set_seal(seal); - match engine.verify_block_seal(&s.block.header, proof) { + match engine.verify_block_seal(&s.block.header) { Err(e) => Err((e, s)), _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index acc276d21..e6cbdb531 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -464,14 +464,14 @@ mod tests { engine.set_signer(tap.clone(), addr1, "1".into()); if let Seal::Regular(seal) = engine.generate_seal(b1.block()) { - assert!(b1.clone().try_seal(engine, seal, None).is_ok()); + assert!(b1.clone().try_seal(engine, seal).is_ok()); // Second proposal is forbidden. assert!(engine.generate_seal(b1.block()) == Seal::None); } engine.set_signer(tap, addr2, "2".into()); if let Seal::Regular(seal) = engine.generate_seal(b2.block()) { - assert!(b2.clone().try_seal(engine, seal, None).is_ok()); + assert!(b2.clone().try_seal(engine, seal).is_ok()); // Second proposal is forbidden. assert!(engine.generate_seal(b2.block()) == Seal::None); } diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index fe27994e4..e5a53d4e9 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -258,7 +258,7 @@ mod tests { let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); if let Seal::Regular(seal) = engine.generate_seal(b.block()) { - assert!(b.try_seal(engine, seal, None).is_ok()); + assert!(b.try_seal(engine, seal).is_ok()); } } diff --git a/ethcore/src/engines/instant_seal.rs b/ethcore/src/engines/instant_seal.rs index 5d05dd83f..45bede9f4 100644 --- a/ethcore/src/engines/instant_seal.rs +++ b/ethcore/src/engines/instant_seal.rs @@ -89,7 +89,7 @@ mod tests { let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::default(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); if let Seal::Regular(seal) = engine.generate_seal(b.block()) { - assert!(b.try_seal(engine, seal, None).is_ok()); + assert!(b.try_seal(engine, seal).is_ok()); } } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 0ae353a56..e4bc514b7 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -44,6 +44,7 @@ use evm::Schedule; use header::Header; use spec::CommonParams; use transaction::{UnverifiedTransaction, SignedTransaction}; +use receipt::Receipt; use ethkey::Signature; use util::*; @@ -61,8 +62,6 @@ pub enum EngineError { UnexpectedMessage, /// Seal field has an unexpected size. BadSealFieldSize(OutOfBounds), - /// Needs a validation proof for the given block hash before verification can continue. - NeedsValidationProof(H256), } impl fmt::Display for EngineError { @@ -74,7 +73,6 @@ impl fmt::Display for EngineError { NotAuthorized(ref address) => format!("Signer {} is not authorized.", address), UnexpectedMessage => "This Engine should not be fed messages.".into(), BadSealFieldSize(ref oob) => format!("Seal field has an unexpected length: {}", oob), - NeedsValidationProof(ref hash) => format!("Needs validation proof of block {} to verify seal.", hash), }; f.write_fmt(format_args!("Engine error ({})", msg)) @@ -92,12 +90,29 @@ pub enum Seal { None, } -/// A validation proof, required for validation of a block header. -pub type ValidationProof = Vec; - /// Type alias for a function we can make calls through synchronously. pub type Call = Fn(Address, Bytes) -> Result; +/// Results of a query of whether a validation proof is necessary at a block. +pub enum RequiresProof { + /// Cannot determine until more data is passed. + Unsure(Unsure), + /// Validation proof not required. + No, + /// Validation proof required. + Yes, +} + +/// More data required to determine if a validation proof is required. +pub enum Unsure { + /// Needs the body. + NeedsBody, + /// Needs the receipts. + NeedsReceipts, + /// Needs both body and receipts. + NeedsBoth, +} + /// A consensus mechanism for the chain. Generally either proof-of-work or proof-of-stake-based. /// Provides hooks into each of the major parts of block import. pub trait Engine : Sync + Send { @@ -191,23 +206,22 @@ pub trait Engine : Sync + Send { /// Verify the seal of a block. This is an auxilliary method that actually just calls other `verify_` methods /// to get the job done. By default it must pass `verify_basic` and `verify_block_unordered`. If more or fewer /// methods are needed for an Engine, this may be overridden. - fn verify_block_seal(&self, header: &Header, _proof: Option) -> Result<(), Error> { + fn verify_block_seal(&self, header: &Header) -> Result<(), Error> { self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) } - /// Generate a validation proof for the given block header. + /// Re-do all verification for a header with the given contract-calling interface /// - /// All values queried during execution of given will go into the proof. - /// This may only be called for blocks indicated in "needs validation proof" - /// errors. - /// - /// Engines which don't draw consensus information from the state (e.g. PoW) - /// don't need to change anything here. - /// - /// Engines which do draw consensus information from the state may only do so - /// here. - fn generate_validation_proof(&self, header: &Header, _call: &Call) -> ValidationProof { - ValidationProof::default() + /// This will be used to generate proofs of validation as well as verify them. + fn verify_with_state(&self, _header: &Header, _call: &Call) -> Result<(), Error> { + Ok(()) + } + + /// Whether a proof is required for the given header. + fn proof_required(&self, _header: Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>) + -> RequiresProof + { + RequiresProof::No } /// Populate a header's fields based on its parent's header. diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index e88803ce2..8c8094117 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -864,7 +864,7 @@ mod tests { let proposer = insert_and_register(&tap, spec.engine.as_ref(), "1"); let (b, seal) = propose_default(&spec, proposer); - assert!(b.lock().try_seal(spec.engine.as_ref(), seal, None).is_ok()); + assert!(b.lock().try_seal(spec.engine.as_ref(), seal).is_ok()); } #[test] diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index a2f6d1075..2c00c43b0 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -25,15 +25,12 @@ use std::sync::Weak; use util::{Address, H256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; -use receipt::Receipt; use self::simple_list::SimpleList; use self::contract::ValidatorContract; use self::safe_contract::ValidatorSafeContract; use self::multi::Multi; -pub type Call = Fn(Address, Vec) -> Result, String>; - /// Creates a validator set from spec. pub fn new_validator_set(spec: ValidatorSpec) -> Box { match spec { @@ -48,10 +45,6 @@ pub fn new_validator_set(spec: ValidatorSpec) -> Box { /// A validator set. pub trait ValidatorSet: Send + Sync { - /// Get this validator set as a flexible validator set. - /// Returning `None` indicates this is only usable for - /// full nodes. - fn as_memoized(&self) -> Option<&Memoized> { None } /// Checks if a given address is a validator. fn contains(&self, parent_block_hash: &H256, address: &Address) -> bool; /// Draws an validator nonce modulo number of validators. @@ -65,35 +58,3 @@ pub trait ValidatorSet: Send + Sync { /// Allows blockchain state access. fn register_contract(&self, _client: Weak) {} } - -/// A flexible validator set can track its changes. -pub trait FlexibleValidatorSet: Send + Sync { - /// Whether a validator set may have changed at this header. - fn has_possibly_changed(&self, header: &Header) -> bool; - - /// Whether a validator set has changed at this header, given the block receipts. - /// Called after `has_possibly_changed`. - /// `Some` indicates the validator set has changed at this header and the new - /// expected validator set. - /// `None` indicates no change. - fn has_changed(&self, header: &Header, receipts: &[Receipt]) -> Option; - - /// Fetch validators at a block synchronously. - fn fetch_validators(&self, header: &Header, call: &Call) -> Result; -} - -/// A memoized flexible validator set -pub struct Memoized { - inner: Box, - memo: Mutex<(SimpleList, (H256, u64))>, -} - -impl Memoized { - /// Denote that the - pub fn use_memo_at(&self, list: SimpleList) -} - -impl ValidatorSet for Memoized { - fn as_memoized(&self) -> Option<&Memoized> { Some(self) } -} - diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index 8527a8aa9..3ae93b0f5 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -55,7 +55,7 @@ impl Multi { ) { Ok((block, set)) => { trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); - Some(&*set) + Some(&**set) }, Err(e) => { debug!(target: "engine", "ValidatorSet could not be recovered: {}", e); @@ -66,21 +66,6 @@ impl Multi { } impl ValidatorSet for Multi { - fn has_possibly_changed(&self, header: &Header) -> bool { - // if the sets are the same for each header, compare those. - // otherwise, the sets have almost certainly changed. - match (self.correct_set(&header.hash()), self.correct_set(header.parent_hash())) { - (Some(a), Some(b)) if a as *const _ == b as *const _ => { a.has_possibly_changed(header) }, - _ => true, - } - } - - fn has_changed(&self, header: &Header, receipts: &[Receipt]) -> Option> { - - } - - fn fetch(&self) -> - fn contains(&self, bh: &H256, address: &Address) -> bool { self.correct_set(bh).map_or(false, |set| set.contains(bh, address)) } diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 8337ef2d8..ac1695b52 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -1133,7 +1133,7 @@ impl MinerService for Miner { |b| &b.hash() == &block_hash ) { trace!(target: "miner", "Submitted block {}={}={} with seal {:?}", block_hash, b.hash(), b.header().bare_hash(), seal); - b.lock().try_seal(&*self.engine, seal, None).or_else(|(e, _)| { + b.lock().try_seal(&*self.engine, seal).or_else(|(e, _)| { warn!(target: "miner", "Mined solution rejected: {}", e); Err(Error::PowInvalid) }) diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 0ef263476..1c3b4366b 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -567,7 +567,7 @@ pub fn verify_old_block(rng: &mut OsRng, header: &Header, engine: &Engine, chain if always || rng.gen::() <= POW_VERIFY_RATE { match chain.block_header(header.parent_hash()) { Some(parent) => engine.verify_block_family(header, &parent, body), - None => engine.verify_block_seal(header, None), // TODO: fetch validation proof as necessary. + None => engine.verify_block_seal(header), // TODO: fetch validation proof as necessary. } } else { engine.verify_block_basic(header, body) From 2f5a774325f59da2f647ac336abf8aabccd58c93 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 15:54:23 +0200 Subject: [PATCH 04/22] add validator contracts to native_contracts bin --- ethcore/native_contracts/build.rs | 8 ++++++++ ethcore/native_contracts/generator/src/lib.rs | 4 ++-- ethcore/native_contracts/src/lib.rs | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ethcore/native_contracts/build.rs b/ethcore/native_contracts/build.rs index 91eaa86cd..0a080e801 100644 --- a/ethcore/native_contracts/build.rs +++ b/ethcore/native_contracts/build.rs @@ -22,9 +22,15 @@ use std::io::Write; // TODO: `include!` these from files where they're pretty-printed? const REGISTRY_ABI: &'static str = r#"[{"constant":true,"inputs":[{"name":"_data","type":"address"}],"name":"canReverse","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_new","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"},{"name":"_value","type":"bytes32"}],"name":"setData","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"string"}],"name":"confirmReverse","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"}],"name":"reserve","outputs":[{"name":"success","type":"bool"}],"payable":true,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"}],"name":"drop","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"}],"name":"getAddress","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_amount","type":"uint256"}],"name":"setFee","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_to","type":"address"}],"name":"transfer","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"}],"name":"getData","outputs":[{"name":"","type":"bytes32"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"}],"name":"reserved","outputs":[{"name":"reserved","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"drain","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"string"},{"name":"_who","type":"address"}],"name":"proposeReverse","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"}],"name":"hasReverse","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"}],"name":"getUint","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"fee","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"}],"name":"getOwner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_name","type":"bytes32"}],"name":"getReverse","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_data","type":"address"}],"name":"reverse","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"},{"name":"_value","type":"uint256"}],"name":"setUint","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"string"},{"name":"_who","type":"address"}],"name":"confirmReverseAs","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"removeReverse","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"bytes32"},{"name":"_key","type":"string"},{"name":"_value","type":"address"}],"name":"setAddress","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function"}]"#; + const SERVICE_TRANSACTION_ABI: &'static str = r#"[{"constant":false,"inputs":[{"name":"_new","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_who","type":"address"}],"name":"certify","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_who","type":"address"},{"name":"_field","type":"string"}],"name":"getAddress","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_who","type":"address"}],"name":"revoke","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"delegate","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_who","type":"address"},{"name":"_field","type":"string"}],"name":"getUint","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"_new","type":"address"}],"name":"setDelegate","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_who","type":"address"}],"name":"certified","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"_who","type":"address"},{"name":"_field","type":"string"}],"name":"get","outputs":[{"name":"","type":"bytes32"}],"payable":false,"type":"function"}]"#; + const SECRETSTORE_ACL_STORAGE_ABI: &'static str = r#"[{"constant":true,"inputs":[{"name":"user","type":"address"},{"name":"document","type":"bytes32"}],"name":"checkPermissions","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]"#; +const VALIDATOR_SET_ABI: &'static str = r#"[{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"}],"payable":false,"type":"function"}]"#; + +const VALIDATOR_REPORT_ABI: &'static str = r#"[{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportMalicious","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportBenign","outputs":[],"payable":false,"type":"function"}]"#; + fn build_file(name: &str, abi: &str, filename: &str) { let code = ::native_contract_generator::generate_module(name, abi).unwrap(); @@ -39,4 +45,6 @@ fn main() { build_file("Registry", REGISTRY_ABI, "registry.rs"); build_file("ServiceTransactionChecker", SERVICE_TRANSACTION_ABI, "service_transaction.rs"); build_file("SecretStoreAclStorage", SECRETSTORE_ACL_STORAGE_ABI, "secretstore_acl_storage.rs"); + build_file("ValidatorSet", VALIDATOR_SET_ABI, "validator_set.rs"); + build_file("ValidatorReport", VALIDATOR_REPORT_ABI, "validator_report.rs"); } diff --git a/ethcore/native_contracts/generator/src/lib.rs b/ethcore/native_contracts/generator/src/lib.rs index f49caf227..363e76ea1 100644 --- a/ethcore/native_contracts/generator/src/lib.rs +++ b/ethcore/native_contracts/generator/src/lib.rs @@ -299,10 +299,10 @@ fn detokenize(name: &str, output_type: ParamType) -> String { ParamType::Bool => format!("{}.to_bool()", name), ParamType::String => format!("{}.to_string()", name), ParamType::Array(kind) => { - let read_array = format!("x.into_iter().map(|a| {{ {} }}).collect::>()", + let read_array = format!("x.into_iter().map(|a| {{ {} }}).collect::>>()", detokenize("a", *kind)); - format!("{}.to_array().and_then(|x| {})", + format!("{}.to_array().and_then(|x| {{ {} }})", name, read_array) } ParamType::FixedArray(_, _) => panic!("Fixed-length arrays not supported.") diff --git a/ethcore/native_contracts/src/lib.rs b/ethcore/native_contracts/src/lib.rs index e894a636f..1aaaf07b1 100644 --- a/ethcore/native_contracts/src/lib.rs +++ b/ethcore/native_contracts/src/lib.rs @@ -26,7 +26,11 @@ extern crate ethcore_util as util; mod registry; mod service_transaction; mod secretstore_acl_storage; +mod validator_set; +mod validator_report; pub use self::registry::Registry; pub use self::service_transaction::ServiceTransactionChecker; pub use self::secretstore_acl_storage::SecretStoreAclStorage; +pub use self::validator_set::ValidatorSet; +pub use self::validator_report::ValidatorReport; From 0f80c57dcaac179b29b077ece375ee91d0886ba2 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 16:15:35 +0200 Subject: [PATCH 05/22] use native contracts in `ValidatorSet` --- ethcore/native_contracts/generator/src/lib.rs | 8 +- .../native_contracts/src/validator_report.rs | 22 ++++ ethcore/native_contracts/src/validator_set.rs | 22 ++++ ethcore/src/engines/validator_set/contract.rs | 104 +++++------------- .../engines/validator_set/safe_contract.rs | 96 +++++----------- 5 files changed, 107 insertions(+), 145 deletions(-) create mode 100644 ethcore/native_contracts/src/validator_report.rs create mode 100644 ethcore/native_contracts/src/validator_set.rs diff --git a/ethcore/native_contracts/generator/src/lib.rs b/ethcore/native_contracts/generator/src/lib.rs index 363e76ea1..28d658cb3 100644 --- a/ethcore/native_contracts/generator/src/lib.rs +++ b/ethcore/native_contracts/generator/src/lib.rs @@ -46,7 +46,7 @@ pub fn generate_module(struct_name: &str, abi: &str) -> Result { Ok(format!(r##" use byteorder::{{BigEndian, ByteOrder}}; -use futures::{{future, Future, BoxFuture}}; +use futures::{{future, Future, IntoFuture, BoxFuture}}; use ethabi::{{Contract, Interface, Token}}; use util::{{self, Uint}}; @@ -99,7 +99,10 @@ fn generate_functions(contract: &Contract) -> Result { /// Inputs: {abi_inputs:?} /// Outputs: {abi_outputs:?} pub fn {snake_name}(&self, call: F, {params}) -> BoxFuture<{output_type}, String> - where F: Fn(util::Address, Vec) -> U, U: Future, Error=String> + Send + 'static + where + F: Fn(util::Address, Vec) -> U, + U: IntoFuture, Error=String>, + U::Future: Send + 'static {{ let function = self.contract.function(r#"{abi_name}"#.to_string()) .expect("function existence checked at compile-time; qed"); @@ -111,6 +114,7 @@ pub fn {snake_name}(&self, call: F, {params}) -> BoxFuture<{output_type}, }}; call_future + .into_future() .and_then(move |out| function.decode_output(out).map_err(|e| format!("{{:?}}", e))) .map(::std::collections::VecDeque::from) .and_then(|mut outputs| {decode_outputs}) diff --git a/ethcore/native_contracts/src/validator_report.rs b/ethcore/native_contracts/src/validator_report.rs new file mode 100644 index 000000000..d77b07c71 --- /dev/null +++ b/ethcore/native_contracts/src/validator_report.rs @@ -0,0 +1,22 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![allow(unused_mut, unused_variables, unused_imports)] + +//! Validator reporting. +// TODO: testing. + +include!(concat!(env!("OUT_DIR"), "/validator_report.rs")); diff --git a/ethcore/native_contracts/src/validator_set.rs b/ethcore/native_contracts/src/validator_set.rs new file mode 100644 index 000000000..0a69913e0 --- /dev/null +++ b/ethcore/native_contracts/src/validator_set.rs @@ -0,0 +1,22 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![allow(unused_mut, unused_variables, unused_imports)] + +//! Validator set contract. +// TODO: testing. + +include!(concat!(env!("OUT_DIR"), "/validator_set.rs")); diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index adfdd1225..c45e8c479 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -18,27 +18,46 @@ /// It can also report validators for misbehaviour with two levels: `reportMalicious` and `reportBenign`. use std::sync::Weak; +use futures::Future; use util::*; use client::{Client, BlockChainClient}; + +use native_contracts::ValidatorReport as Provider; + use super::ValidatorSet; use super::safe_contract::ValidatorSafeContract; -/// The validator contract should have the following interface: -/// [{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"}],"payable":false,"type":"function"}] +/// A validator contract with reporting. pub struct ValidatorContract { validators: ValidatorSafeContract, - provider: RwLock>, + provider: Provider, + client: RwLock>>, // TODO [keorn]: remove } impl ValidatorContract { pub fn new(contract_address: Address) -> Self { ValidatorContract { validators: ValidatorSafeContract::new(contract_address), - provider: RwLock::new(None), + provider: Provider::new(contract_address), + client: RwLock::new(None), } } } +impl ValidatorContract { + // could be `impl Trait`. + // note: dispatches transactions to network as well as execute. + // TODO [keorn]: Make more general. + fn transact(&self) -> Box) -> Result, String>> { + let client = self.client.read().clone(); + Box::new(move |a, d| client.as_ref() + .and_then(Weak::upgrade) + .ok_or("No client!".into()) + .and_then(|c| c.transact_contract(a, d).map_err(|e| format!("Transaction import error: {}", e))) + .map(|_| Default::default())) + } +} + impl ValidatorSet for ValidatorContract { fn contains(&self, bh: &H256, address: &Address) -> bool { self.validators.contains(bh, address) @@ -53,85 +72,22 @@ impl ValidatorSet for ValidatorContract { } fn report_malicious(&self, address: &Address) { - if let Some(ref provider) = *self.provider.read() { - match provider.report_malicious(address) { - Ok(_) => warn!(target: "engine", "Reported malicious validator {}", address), - Err(s) => warn!(target: "engine", "Validator {} could not be reported {}", address, s), - } - } else { - warn!(target: "engine", "Malicious behaviour could not be reported: no provider contract.") + match self.provider.report_malicious(&*self.transact(), *address).wait() { + Ok(_) => warn!(target: "engine", "Reported malicious validator {}", address), + Err(s) => warn!(target: "engine", "Validator {} could not be reported {}", address, s), } } fn report_benign(&self, address: &Address) { - if let Some(ref provider) = *self.provider.read() { - match provider.report_benign(address) { - Ok(_) => warn!(target: "engine", "Reported benign validator misbehaviour {}", address), - Err(s) => warn!(target: "engine", "Validator {} could not be reported {}", address, s), - } - } else { - warn!(target: "engine", "Benign misbehaviour could not be reported: no provider contract.") + match self.provider.report_benign(&*self.transact(), *address).wait() { + Ok(_) => warn!(target: "engine", "Reported benign validator misbehaviour {}", address), + Err(s) => warn!(target: "engine", "Validator {} could not be reported {}", address, s), } } fn register_contract(&self, client: Weak) { self.validators.register_contract(client.clone()); - let transact = move |a, d| client - .upgrade() - .ok_or("No client!".into()) - .and_then(|c| c.transact_contract(a, d).map_err(|e| format!("Transaction import error: {}", e))) - .map(|_| Default::default()); - *self.provider.write() = Some(provider::Contract::new(self.validators.address, transact)); - } -} - -mod provider { - // Autogenerated from JSON contract definition using Rust contract convertor. - #![allow(unused_imports)] - use std::string::String; - use std::result::Result; - use std::fmt; - use {util, ethabi}; - use util::{Uint}; - - pub struct Contract { - contract: ethabi::Contract, - address: util::Address, - do_call: Box) -> Result, String> + Send + Sync + 'static>, - } - impl Contract { - pub fn new(address: util::Address, do_call: F) -> Self where F: Fn(util::Address, Vec) -> Result, String> + Send + Sync + 'static { - Contract { - contract: ethabi::Contract::new(ethabi::Interface::load(b"[{\"constant\":false,\"inputs\":[{\"name\":\"validator\",\"type\":\"address\"}],\"name\":\"reportMalicious\",\"outputs\":[],\"payable\":false,\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"validator\",\"type\":\"address\"}],\"name\":\"reportBenign\",\"outputs\":[],\"payable\":false,\"type\":\"function\"}]").expect("JSON is autogenerated; qed")), - address: address, - do_call: Box::new(do_call), - } - } - fn as_string(e: T) -> String { format!("{:?}", e) } - - /// Auto-generated from: `{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportMalicious","outputs":[],"payable":false,"type":"function"}` - #[allow(dead_code)] - pub fn report_malicious(&self, validator: &util::Address) -> Result<(), String> { - let call = self.contract.function("reportMalicious".into()).map_err(Self::as_string)?; - let data = call.encode_call( - vec![ethabi::Token::Address(validator.clone().0)] - ).map_err(Self::as_string)?; - call.decode_output((self.do_call)(self.address.clone(), data)?).map_err(Self::as_string)?; - - Ok(()) - } - - /// Auto-generated from: `{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportBenign","outputs":[],"payable":false,"type":"function"}` - #[allow(dead_code)] - pub fn report_benign(&self, validator: &util::Address) -> Result<(), String> { - let call = self.contract.function("reportBenign".into()).map_err(Self::as_string)?; - let data = call.encode_call( - vec![ethabi::Token::Address(validator.clone().0)] - ).map_err(Self::as_string)?; - call.decode_output((self.do_call)(self.address.clone(), data)?).map_err(Self::as_string)?; - - Ok(()) - } + *self.client.write() = Some(client); } } diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 0a0eaecfd..958f3e0a0 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -17,24 +17,26 @@ /// Validator set maintained in a contract, updated using `getValidators` method. use std::sync::Weak; -use ethabi; +use futures::Future; +use native_contracts::ValidatorSet as Provider; + use util::*; use util::cache::MemoryLruCache; + use types::ids::BlockId; use client::{Client, BlockChainClient}; + use super::ValidatorSet; use super::simple_list::SimpleList; const MEMOIZE_CAPACITY: usize = 500; -const CONTRACT_INTERFACE: &'static [u8] = b"[{\"constant\":true,\"inputs\":[],\"name\":\"getValidators\",\"outputs\":[{\"name\":\"\",\"type\":\"address[]\"}],\"payable\":false,\"type\":\"function\"}]"; -const GET_VALIDATORS: &'static str = "getValidators"; /// The validator contract should have the following interface: -/// [{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"}],"payable":false,"type":"function"}] pub struct ValidatorSafeContract { pub address: Address, validators: RwLock>, - provider: RwLock>, + provider: Provider, + client: RwLock>>, // TODO [keorn]: remove } impl ValidatorSafeContract { @@ -42,26 +44,30 @@ impl ValidatorSafeContract { ValidatorSafeContract { address: contract_address, validators: RwLock::new(MemoryLruCache::new(MEMOIZE_CAPACITY)), - provider: RwLock::new(None), + provider: Provider::new(contract_address), + client: RwLock::new(None), } } + fn do_call(&self, id: BlockId) -> Box) -> Result, String>> { + let client = self.client.read().clone(); + Box::new(move |addr, data| client.as_ref() + .and_then(Weak::upgrade) + .ok_or("No client!".into()) + .and_then(|c| c.call_contract(id, addr, data))) + } + /// Queries the state and gets the set of validators. fn get_list(&self, block_hash: H256) -> Option { - if let Some(ref provider) = *self.provider.read() { - match provider.get_validators(BlockId::Hash(block_hash)) { - Ok(new) => { - debug!(target: "engine", "Set of validators obtained: {:?}", new); - Some(SimpleList::new(new)) - }, - Err(s) => { - debug!(target: "engine", "Set of validators could not be updated: {}", s); - None - }, - } - } else { - warn!(target: "engine", "Set of validators could not be updated: no provider contract."); - None + match self.provider.get_validators(&*self.do_call(BlockId::Hash(block_hash))).wait() { + Ok(new) => { + debug!(target: "engine", "Set of validators obtained: {:?}", new); + Some(SimpleList::new(new)) + }, + Err(s) => { + debug!(target: "engine", "Set of validators could not be updated: {}", s); + None + }, } } } @@ -114,55 +120,7 @@ impl ValidatorSet for ValidatorSafeContract { fn register_contract(&self, client: Weak) { trace!(target: "engine", "Setting up contract caller."); - let contract = ethabi::Contract::new(ethabi::Interface::load(CONTRACT_INTERFACE).expect("JSON interface is valid; qed")); - let call = contract.function(GET_VALIDATORS.into()).expect("Method name is valid; qed"); - let data = call.encode_call(vec![]).expect("get_validators does not take any arguments; qed"); - let contract_address = self.address.clone(); - let do_call = move |id| client - .upgrade() - .ok_or("No client!".into()) - .and_then(|c| c.call_contract(id, contract_address.clone(), data.clone())) - .map(|raw_output| call.decode_output(raw_output).expect("ethabi is correct; qed")); - *self.provider.write() = Some(provider::Contract::new(do_call)); - } -} - -mod provider { - use std::string::String; - use std::result::Result; - use {util, ethabi}; - use types::ids::BlockId; - - pub struct Contract { - do_call: Box Result, String> + Send + Sync + 'static>, - } - - impl Contract { - pub fn new(do_call: F) -> Self where F: Fn(BlockId) -> Result, String> + Send + Sync + 'static { - Contract { - do_call: Box::new(do_call), - } - } - - /// Gets validators from contract with interface: `{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"}],"payable":false,"type":"function"}` - pub fn get_validators(&self, id: BlockId) -> Result, String> { - Ok((self.do_call)(id)? - .into_iter() - .rev() - .collect::>() - .pop() - .expect("get_validators returns one argument; qed") - .to_array() - .and_then(|v| v - .into_iter() - .map(|a| a.to_address()) - .collect::>>()) - .expect("get_validators returns a list of addresses; qed") - .into_iter() - .map(util::Address::from) - .collect::>() - ) - } + *self.client.write() = Some(client); } } From ec922ee5e4d954a5acffe44e62c8a6ba85954e4d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 16:42:19 +0200 Subject: [PATCH 06/22] supply optional call context to validator sets --- ethcore/src/engines/validator_set/contract.rs | 24 +++++++----- ethcore/src/engines/validator_set/mod.rs | 37 +++++++++++++++++-- ethcore/src/engines/validator_set/multi.rs | 31 ++++++++++------ .../engines/validator_set/safe_contract.rs | 33 +++++++++-------- .../src/engines/validator_set/simple_list.rs | 12 ++++-- 5 files changed, 94 insertions(+), 43 deletions(-) diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index c45e8c479..12d816246 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -18,12 +18,14 @@ /// It can also report validators for misbehaviour with two levels: `reportMalicious` and `reportBenign`. use std::sync::Weak; -use futures::Future; use util::*; -use client::{Client, BlockChainClient}; +use futures::Future; use native_contracts::ValidatorReport as Provider; +use client::{Client, BlockChainClient}; +use engines::Call; + use super::ValidatorSet; use super::safe_contract::ValidatorSafeContract; @@ -48,7 +50,7 @@ impl ValidatorContract { // could be `impl Trait`. // note: dispatches transactions to network as well as execute. // TODO [keorn]: Make more general. - fn transact(&self) -> Box) -> Result, String>> { + fn transact(&self) -> Box { let client = self.client.read().clone(); Box::new(move |a, d| client.as_ref() .and_then(Weak::upgrade) @@ -59,16 +61,20 @@ impl ValidatorContract { } impl ValidatorSet for ValidatorContract { - fn contains(&self, bh: &H256, address: &Address) -> bool { - self.validators.contains(bh, address) + fn default_caller(&self, id: ::ids::BlockId) -> Box { + self.validators.default_caller(id) } - fn get(&self, bh: &H256, nonce: usize) -> Address { - self.validators.get(bh, nonce) + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { + self.validators.contains_with_caller(bh, address, caller) } - fn count(&self, bh: &H256) -> usize { - self.validators.count(bh) + fn get_with_caller(&self, bh: &H256, nonce: usize, caller: &Call) -> Address { + self.validators.get_with_caller(bh, nonce, caller) + } + + fn count_with_caller(&self, bh: &H256, caller: &Call) -> usize { + self.validators.count_with_caller(bh, caller) } fn report_malicious(&self, address: &Address) { diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index 2c00c43b0..707b5adf5 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -22,6 +22,7 @@ mod contract; mod multi; use std::sync::Weak; +use ids::BlockId; use util::{Address, H256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; @@ -31,6 +32,8 @@ use self::contract::ValidatorContract; use self::safe_contract::ValidatorSafeContract; use self::multi::Multi; +use super::Call; + /// Creates a validator set from spec. pub fn new_validator_set(spec: ValidatorSpec) -> Box { match spec { @@ -44,13 +47,39 @@ pub fn new_validator_set(spec: ValidatorSpec) -> Box { } /// A validator set. +// TODO [keorn]: remove internal callers. pub trait ValidatorSet: Send + Sync { - /// Checks if a given address is a validator. - fn contains(&self, parent_block_hash: &H256, address: &Address) -> bool; + /// Get the default "Call" helper, for use in general operation. + fn default_caller(&self, block_id: BlockId) -> Box; + + /// Checks if a given address is a validator, + /// using underlying, default call mechanism. + fn contains(&self, parent: &H256, address: &Address) -> bool { + let default = self.default_caller(BlockId::Hash(*parent)); + self.contains_with_caller(parent, address, &*default) + } /// Draws an validator nonce modulo number of validators. - fn get(&self, parent_block_hash: &H256, nonce: usize) -> Address; + fn get(&self, parent: &H256, nonce: usize) -> Address { + let default = self.default_caller(BlockId::Hash(*parent)); + self.get_with_caller(parent, nonce, &*default) + } /// Returns the current number of validators. - fn count(&self, parent_block_hash: &H256) -> usize; + fn count(&self, parent: &H256) -> usize { + let default = self.default_caller(BlockId::Hash(*parent)); + self.count_with_caller(parent, &*default) + } + + /// Checks if a given address is a validator, with the given function + /// for executing synchronous calls to contracts. + fn contains_with_caller(&self, parent_block_hash: &H256, address: &Address, caller: &Call) -> bool; + + /// Draws an validator nonce modulo number of validators. + /// + fn get_with_caller(&self, parent_block_hash: &H256, nonce: usize, caller: &Call) -> Address; + + /// Returns the current number of validators. + fn count_with_caller(&self, parent_block_hash: &H256, caller: &Call) -> usize; + /// Notifies about malicious behaviour. fn report_malicious(&self, _validator: &Address) {} /// Notifies about benign misbehaviour. diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index 3ae93b0f5..f098ca343 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -18,13 +18,14 @@ use std::collections::BTreeMap; use std::sync::Weak; +use engines::Call; use util::{H256, Address, RwLock}; use ids::BlockId; use header::BlockNumber; use client::{Client, BlockChainClient}; use super::ValidatorSet; -type BlockNumberLookup = Box Result + Send + Sync + 'static>; +type BlockNumberLookup = Box Result + Send + Sync + 'static>; pub struct Multi { sets: BTreeMap>, @@ -40,10 +41,10 @@ impl Multi { } } - fn correct_set(&self, bh: &H256) -> Option<&ValidatorSet> { + fn correct_set(&self, id: BlockId) -> Option<&ValidatorSet> { match self .block_number - .read()(bh) + .read()(id) .map(|parent_block| self .sets .iter() @@ -66,16 +67,24 @@ impl Multi { } impl ValidatorSet for Multi { - fn contains(&self, bh: &H256, address: &Address) -> bool { - self.correct_set(bh).map_or(false, |set| set.contains(bh, address)) + fn default_caller(&self, block_id: BlockId) -> Box { + self.correct_set(block_id).map(|set| set.default_caller(block_id)) + .unwrap_or(Box::new(|_, _| Err("No validator set for given ID.".into()))) } - fn get(&self, bh: &H256, nonce: usize) -> Address { - self.correct_set(bh).map_or_else(Default::default, |set| set.get(bh, nonce)) + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { + self.correct_set(BlockId::Hash(*bh)) + .map_or(false, |set| set.contains_with_caller(bh, address, caller)) } - fn count(&self, bh: &H256) -> usize { - self.correct_set(bh).map_or_else(usize::max_value, |set| set.count(bh)) + fn get_with_caller(&self, bh: &H256, nonce: usize, caller: &Call) -> Address { + self.correct_set(BlockId::Hash(*bh)) + .map_or_else(Default::default, |set| set.get_with_caller(bh, nonce, caller)) + } + + fn count_with_caller(&self, bh: &H256, caller: &Call) -> usize { + self.correct_set(BlockId::Hash(*bh)) + .map_or_else(usize::max_value, |set| set.count_with_caller(bh, caller)) } fn report_malicious(&self, validator: &Address) { @@ -94,10 +103,10 @@ impl ValidatorSet for Multi { for set in self.sets.values() { set.register_contract(client.clone()); } - *self.block_number.write() = Box::new(move |hash| client + *self.block_number.write() = Box::new(move |id| client .upgrade() .ok_or("No client!".into()) - .and_then(|c| c.block_number(BlockId::Hash(*hash)).ok_or("Unknown block".into()))); + .and_then(|c| c.block_number(id).ok_or("Unknown block".into()))); } } diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 958f3e0a0..8a0599dde 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -23,6 +23,7 @@ use native_contracts::ValidatorSet as Provider; use util::*; use util::cache::MemoryLruCache; +use engines::Call; use types::ids::BlockId; use client::{Client, BlockChainClient}; @@ -49,17 +50,9 @@ impl ValidatorSafeContract { } } - fn do_call(&self, id: BlockId) -> Box) -> Result, String>> { - let client = self.client.read().clone(); - Box::new(move |addr, data| client.as_ref() - .and_then(Weak::upgrade) - .ok_or("No client!".into()) - .and_then(|c| c.call_contract(id, addr, data))) - } - /// Queries the state and gets the set of validators. - fn get_list(&self, block_hash: H256) -> Option { - match self.provider.get_validators(&*self.do_call(BlockId::Hash(block_hash))).wait() { + fn get_list(&self, caller: &Call) -> Option { + match self.provider.get_validators(caller).wait() { Ok(new) => { debug!(target: "engine", "Set of validators obtained: {:?}", new); Some(SimpleList::new(new)) @@ -73,14 +66,22 @@ impl ValidatorSafeContract { } impl ValidatorSet for ValidatorSafeContract { - fn contains(&self, block_hash: &H256, address: &Address) -> bool { + fn default_caller(&self, id: BlockId) -> Box { + let client = self.client.read().clone(); + Box::new(move |addr, data| client.as_ref() + .and_then(Weak::upgrade) + .ok_or("No client!".into()) + .and_then(|c| c.call_contract(id, addr, data))) + } + + fn contains_with_caller(&self, block_hash: &H256, address: &Address, caller: &Call) -> bool { let mut guard = self.validators.write(); let maybe_existing = guard .get_mut(block_hash) .map(|list| list.contains(block_hash, address)); maybe_existing .unwrap_or_else(|| self - .get_list(block_hash.clone()) + .get_list(caller) .map_or(false, |list| { let contains = list.contains(block_hash, address); guard.insert(block_hash.clone(), list); @@ -88,14 +89,14 @@ impl ValidatorSet for ValidatorSafeContract { })) } - fn get(&self, block_hash: &H256, nonce: usize) -> Address { + fn get_with_caller(&self, block_hash: &H256, nonce: usize, caller: &Call) -> Address { let mut guard = self.validators.write(); let maybe_existing = guard .get_mut(block_hash) .map(|list| list.get(block_hash, nonce)); maybe_existing .unwrap_or_else(|| self - .get_list(block_hash.clone()) + .get_list(caller) .map_or_else(Default::default, |list| { let address = list.get(block_hash, nonce); guard.insert(block_hash.clone(), list); @@ -103,14 +104,14 @@ impl ValidatorSet for ValidatorSafeContract { })) } - fn count(&self, block_hash: &H256) -> usize { + fn count_with_caller(&self, block_hash: &H256, caller: &Call) -> usize { let mut guard = self.validators.write(); let maybe_existing = guard .get_mut(block_hash) .map(|list| list.count(block_hash)); maybe_existing .unwrap_or_else(|| self - .get_list(block_hash.clone()) + .get_list(caller) .map_or_else(usize::max_value, |list| { let address = list.count(block_hash); guard.insert(block_hash.clone(), list); diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index cc4259fac..ac82cf76f 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -17,6 +17,8 @@ /// Preconfigured validator list. use util::{H256, Address, HeapSizeOf}; + +use engines::Call; use super::ValidatorSet; #[derive(Debug, PartialEq, Eq, Default)] @@ -39,16 +41,20 @@ impl HeapSizeOf for SimpleList { } impl ValidatorSet for SimpleList { - fn contains(&self, _bh: &H256, address: &Address) -> bool { + fn default_caller(&self, _block_id: ::ids::BlockId) -> Box { + Box::new(|_, _| Err("Simple list doesn't require calls.".into())) + } + + fn contains_with_caller(&self, _bh: &H256, address: &Address, _: &Call) -> bool { self.validators.contains(address) } - fn get(&self, _bh: &H256, nonce: usize) -> Address { + fn get_with_caller(&self, _bh: &H256, nonce: usize, _: &Call) -> Address { let validator_n = self.validators.len(); self.validators.get(nonce % validator_n).expect("There are validator_n authorities; taking number modulo validator_n gives number in validator_n range; qed").clone() } - fn count(&self, _bh: &H256) -> usize { + fn count_with_caller(&self, _bh: &H256, _: &Call) -> usize { self.validators.len() } } From 34a1512ff07b74993ec45206e4a5af9612425fc0 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 18:55:38 +0200 Subject: [PATCH 07/22] skeleton for proof checking --- ethcore/src/engines/authority_round.rs | 14 +++++- ethcore/src/engines/basic_authority.rs | 18 ++++++-- ethcore/src/engines/mod.rs | 17 ++++--- ethcore/src/engines/validator_set/contract.rs | 11 +++++ ethcore/src/engines/validator_set/mod.rs | 23 +++++++++- ethcore/src/engines/validator_set/multi.rs | 44 ++++++++++++------- .../engines/validator_set/safe_contract.rs | 14 ++++++ .../src/engines/validator_set/simple_list.rs | 17 +++++++ 8 files changed, 131 insertions(+), 27 deletions(-) diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index e6cbdb531..e57fb27bc 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -25,7 +25,7 @@ use rlp::{UntrustedRlp, encode}; use account_provider::AccountProvider; use block::*; use spec::CommonParams; -use engines::{Engine, Seal, EngineError}; +use engines::{Call, Engine, Seal, EngineError}; use header::Header; use error::{Error, TransactionError, BlockError}; use evm::Schedule; @@ -358,6 +358,18 @@ impl Engine for AuthorityRound { Ok(()) } + // the proofs we need just allow us to get the full validator set. + fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { + self.validators.generate_proof(header, caller) + .map_err(|e| EngineError::InsufficientProof(e).into()) + } + + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> super::RequiresProof + { + self.validators.proof_required(header, block, receipts) + } + fn verify_transaction_basic(&self, t: &UnverifiedTransaction, header: &Header) -> result::Result<(), Error> { t.check_low_s()?; diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index e5a53d4e9..a785b360a 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -23,7 +23,7 @@ use account_provider::AccountProvider; use block::*; use builtin::Builtin; use spec::CommonParams; -use engines::{Engine, Seal}; +use engines::{Engine, EngineError, Seal, Call, RequiresProof}; use env_info::EnvInfo; use error::{BlockError, Error}; use evm::Schedule; @@ -51,8 +51,7 @@ impl From for BasicAuthorityParams { } } -/// Engine using `BasicAuthority` proof-of-work consensus algorithm, suitable for Ethereum -/// mainnet chains in the Olympic, Frontier and Homestead eras. +/// Engine using `BasicAuthority`, trivial proof-of-authority consensus. pub struct BasicAuthority { params: CommonParams, gas_limit_bound_divisor: U256, @@ -139,6 +138,7 @@ impl Engine for BasicAuthority { fn verify_block_family(&self, header: &Header, parent: &Header, _block: Option<&[u8]>) -> result::Result<(), Error> { use rlp::UntrustedRlp; + // Check if the signature belongs to a validator, can depend on parent state. let sig = UntrustedRlp::new(&header.seal()[0]).as_val::()?; let signer = public_to_address(&recover(&sig.into(), &header.bare_hash())?); @@ -164,6 +164,18 @@ impl Engine for BasicAuthority { Ok(()) } + // the proofs we need just allow us to get the full validator set. + fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { + self.validators.generate_proof(header, caller) + .map_err(|e| EngineError::InsufficientProof(e).into()) + } + + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> RequiresProof + { + self.validators.proof_required(header, block, receipts) + } + fn register_client(&self, client: Weak) { self.validators.register_contract(client); } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index e4bc514b7..f1e0a4d43 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -62,6 +62,8 @@ pub enum EngineError { UnexpectedMessage, /// Seal field has an unexpected size. BadSealFieldSize(OutOfBounds), + /// Validation proof insufficient. + InsufficientProof(String), } impl fmt::Display for EngineError { @@ -73,6 +75,7 @@ impl fmt::Display for EngineError { NotAuthorized(ref address) => format!("Signer {} is not authorized.", address), UnexpectedMessage => "This Engine should not be fed messages.".into(), BadSealFieldSize(ref oob) => format!("Seal field has an unexpected length: {}", oob), + InsufficientProof(ref msg) => format!("Insufficient validation proof: {}", msg), }; f.write_fmt(format_args!("Engine error ({})", msg)) @@ -99,8 +102,9 @@ pub enum RequiresProof { Unsure(Unsure), /// Validation proof not required. No, - /// Validation proof required. - Yes, + /// Validation proof required, and the expected output + /// if it can be recovered. + Yes(Option), } /// More data required to determine if a validation proof is required. @@ -213,12 +217,15 @@ pub trait Engine : Sync + Send { /// Re-do all verification for a header with the given contract-calling interface /// /// This will be used to generate proofs of validation as well as verify them. - fn verify_with_state(&self, _header: &Header, _call: &Call) -> Result<(), Error> { - Ok(()) + /// Must be called on blocks that have already passed basic verification. + /// + /// Return the "validation proof" generated. + fn prove_with_caller(&self, _header: &Header, _caller: &Call) -> Result { + Ok(Vec::new()) } /// Whether a proof is required for the given header. - fn proof_required(&self, _header: Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>) + fn proof_required(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>) -> RequiresProof { RequiresProof::No diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index 12d816246..5c4af4ba7 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -25,6 +25,7 @@ use native_contracts::ValidatorReport as Provider; use client::{Client, BlockChainClient}; use engines::Call; +use header::Header; use super::ValidatorSet; use super::safe_contract::ValidatorSafeContract; @@ -65,6 +66,16 @@ impl ValidatorSet for ValidatorContract { self.validators.default_caller(id) } + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> ::engines::RequiresProof + { + self.validators.proof_required(header, block, receipts) + } + + fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { + self.validators.generate_proof(header, caller) + } + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { self.validators.contains_with_caller(bh, address, caller) } diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index 707b5adf5..d5abf430d 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -26,6 +26,7 @@ use ids::BlockId; use util::{Address, H256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; +use header::Header; use self::simple_list::SimpleList; use self::contract::ValidatorContract; @@ -47,9 +48,10 @@ pub fn new_validator_set(spec: ValidatorSpec) -> Box { } /// A validator set. -// TODO [keorn]: remove internal callers. pub trait ValidatorSet: Send + Sync { /// Get the default "Call" helper, for use in general operation. + // TODO [keorn]: this is a hack intended to migrate off of + // a strict dependency on state always being available. fn default_caller(&self, block_id: BlockId) -> Box; /// Checks if a given address is a validator, @@ -63,12 +65,31 @@ pub trait ValidatorSet: Send + Sync { let default = self.default_caller(BlockId::Hash(*parent)); self.get_with_caller(parent, nonce, &*default) } + /// Returns the current number of validators. fn count(&self, parent: &H256) -> usize { let default = self.default_caller(BlockId::Hash(*parent)); self.count_with_caller(parent, &*default) } + /// Whether a validation proof is required at the given block. + /// Usually indicates that the validator set changed at the given block. + /// + /// Should not inspect state! This is used in situations where + /// state is not generally available. + /// + /// Return `Ok` with a flag indicating whether it changed at the given header, + /// or `Unsure` indicating a need for more information. + /// + /// This may or may not be called in a loop. + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> super::RequiresProof; + + /// Generate a validation proof at the given header. + /// Must interact with state only through the given caller! + /// Otherwise, generated proofs may be wrong. + fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String>; + /// Checks if a given address is a validator, with the given function /// for executing synchronous calls to contracts. fn contains_with_caller(&self, parent_block_hash: &H256, address: &Address, caller: &Call) -> bool; diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index f098ca343..78270c4af 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -18,10 +18,10 @@ use std::collections::BTreeMap; use std::sync::Weak; -use engines::Call; +use engines::{Call, RequiresProof}; use util::{H256, Address, RwLock}; use ids::BlockId; -use header::BlockNumber; +use header::{BlockNumber, Header}; use client::{Client, BlockChainClient}; use super::ValidatorSet; @@ -42,21 +42,9 @@ impl Multi { } fn correct_set(&self, id: BlockId) -> Option<&ValidatorSet> { - match self - .block_number - .read()(id) - .map(|parent_block| self - .sets - .iter() - .rev() - .find(|&(block, _)| *block <= parent_block + 1) - .expect("constructor validation ensures that there is at least one validator set for block 0; - block 0 is less than any uint; - qed") - ) { - Ok((block, set)) => { - trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); - Some(&**set) + match self.block_number.read()(id).map(|parent_block| self.correct_set_by_number(parent_block)) { + Ok(set) => { + Some(set) }, Err(e) => { debug!(target: "engine", "ValidatorSet could not be recovered: {}", e); @@ -64,6 +52,18 @@ impl Multi { }, } } + + fn correct_set_by_number(&self, parent_block: BlockNumber) -> &ValidatorSet { + let (block, set) = self.sets.iter() + .rev() + .find(|&(block, _)| *block <= parent_block + 1) + .expect("constructor validation ensures that there is at least one validator set for block 0; + block 0 is less than any uint; + qed"); + + trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); + &**set + } } impl ValidatorSet for Multi { @@ -72,6 +72,16 @@ impl ValidatorSet for Multi { .unwrap_or(Box::new(|_, _| Err("No validator set for given ID.".into()))) } + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> RequiresProof + { + self.correct_set_by_number(header.number()).proof_required(header, block, receipts) + } + + fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { + self.correct_set_by_number(header.number()).generate_proof(header, caller) + } + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { self.correct_set(BlockId::Hash(*bh)) .map_or(false, |set| set.contains_with_caller(bh, address, caller)) diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 8a0599dde..61e16e198 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -26,6 +26,7 @@ use util::cache::MemoryLruCache; use engines::Call; use types::ids::BlockId; use client::{Client, BlockChainClient}; +use header::Header; use super::ValidatorSet; use super::simple_list::SimpleList; @@ -74,6 +75,19 @@ impl ValidatorSet for ValidatorSafeContract { .and_then(|c| c.call_contract(id, addr, data))) } + fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> ::engines::RequiresProof + { + // TODO: check blooms first and then logs for the + // ValidatorsChanged([parent_hash, nonce], new_validators) log event. + ::engines::RequiresProof::No + } + + fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { + self.get_list(caller).map(|list| ::rlp::encode_list(&list.into_inner()).to_vec()) + .ok_or_else(|| "Caller insufficient to get validator list.".into()) + } + fn contains_with_caller(&self, block_hash: &H256, address: &Address, caller: &Call) -> bool { let mut guard = self.validators.write(); let maybe_existing = guard diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index ac82cf76f..e513212a6 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -19,6 +19,7 @@ use util::{H256, Address, HeapSizeOf}; use engines::Call; +use header::Header; use super::ValidatorSet; #[derive(Debug, PartialEq, Eq, Default)] @@ -27,11 +28,17 @@ pub struct SimpleList { } impl SimpleList { + /// Create a new `SimpleList`. pub fn new(validators: Vec
) -> Self { SimpleList { validators: validators, } } + + /// Convert into inner representation. + pub fn into_inner(self) -> Vec
{ + self.validators + } } impl HeapSizeOf for SimpleList { @@ -45,6 +52,16 @@ impl ValidatorSet for SimpleList { Box::new(|_, _| Err("Simple list doesn't require calls.".into())) } + fn proof_required(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[::receipt::Receipt]>) + -> ::engines::RequiresProof + { + ::engines::RequiresProof::No + } + + fn generate_proof(&self, _header: &Header, _caller: &Call) -> Result, String> { + Ok(Vec::new()) + } + fn contains_with_caller(&self, _bh: &H256, address: &Address, _: &Call) -> bool { self.validators.contains(address) } From b4f3e30cd6827d404182aad45dc748712fbef150 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 12 Apr 2017 22:10:18 +0200 Subject: [PATCH 08/22] detect changes in safe_contract --- ethcore/native_contracts/build.rs | 4 +- ethcore/native_contracts/generator/src/lib.rs | 7 +- .../engines/validator_set/safe_contract.rs | 132 ++++++++++++++++-- 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/ethcore/native_contracts/build.rs b/ethcore/native_contracts/build.rs index 0a080e801..7da55a977 100644 --- a/ethcore/native_contracts/build.rs +++ b/ethcore/native_contracts/build.rs @@ -27,7 +27,9 @@ const SERVICE_TRANSACTION_ABI: &'static str = r#"[{"constant":false,"inputs":[{" const SECRETSTORE_ACL_STORAGE_ABI: &'static str = r#"[{"constant":true,"inputs":[{"name":"user","type":"address"},{"name":"document","type":"bytes32"}],"name":"checkPermissions","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]"#; -const VALIDATOR_SET_ABI: &'static str = r#"[{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"}],"payable":false,"type":"function"}]"#; +// be very careful changing these: ensure `ethcore/engines` validator sets have corresponding +// changes. +const VALIDATOR_SET_ABI: &'static str = r#"[{"constant":true,"inputs":[],"name":"transitionNonce","outputs":[{"name":"nonce","type":"uint256"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"validators","type":"address[]"}],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_parent_hash","type":"bytes32"},{"indexed":true,"name":"_nonce","type":"uint256"},{"indexed":false,"name":"_new_set","type":"address[]"}],"name":"ValidatorsChanged","type":"event"}]"#; const VALIDATOR_REPORT_ABI: &'static str = r#"[{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportMalicious","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"validator","type":"address"}],"name":"reportBenign","outputs":[],"payable":false,"type":"function"}]"#; diff --git a/ethcore/native_contracts/generator/src/lib.rs b/ethcore/native_contracts/generator/src/lib.rs index 28d658cb3..76985e682 100644 --- a/ethcore/native_contracts/generator/src/lib.rs +++ b/ethcore/native_contracts/generator/src/lib.rs @@ -47,7 +47,7 @@ pub fn generate_module(struct_name: &str, abi: &str) -> Result { Ok(format!(r##" use byteorder::{{BigEndian, ByteOrder}}; use futures::{{future, Future, IntoFuture, BoxFuture}}; -use ethabi::{{Contract, Interface, Token}}; +use ethabi::{{Contract, Interface, Token, Event}}; use util::{{self, Uint}}; pub struct {name} {{ @@ -70,6 +70,11 @@ impl {name} {{ }} }} + /// Access the underlying `ethabi` contract. + pub fn contract(this: &Self) -> &Contract {{ + &this.contract + }} + {functions} }} "##, diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 61e16e198..de7ecb3a2 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -23,16 +23,25 @@ use native_contracts::ValidatorSet as Provider; use util::*; use util::cache::MemoryLruCache; -use engines::Call; -use types::ids::BlockId; +use basic_types::LogBloom; use client::{Client, BlockChainClient}; +use engines::Call; use header::Header; +use ids::BlockId; +use log_entry::LogEntry; use super::ValidatorSet; use super::simple_list::SimpleList; const MEMOIZE_CAPACITY: usize = 500; +// TODO: ethabi should be able to generate this. +const EVENT_NAME: &'static [u8] = &*b"ValidatorsChanged(bytes32,uint256,address[])"; + +lazy_static! { + static ref EVENT_NAME_HASH: H256 = EVENT_NAME.sha3(); +} + /// The validator contract should have the following interface: pub struct ValidatorSafeContract { pub address: Address, @@ -41,6 +50,14 @@ pub struct ValidatorSafeContract { client: RwLock>>, // TODO [keorn]: remove } +fn encode_proof(nonce: U256, validators: &[Address]) -> Bytes { + use rlp::RlpStream; + + let mut stream = RlpStream::new_list(2); + stream.append(&nonce).append_list(validators); + stream.drain().to_vec() +} + impl ValidatorSafeContract { pub fn new(contract_address: Address) -> Self { ValidatorSafeContract { @@ -64,6 +81,40 @@ impl ValidatorSafeContract { }, } } + + /// Queries for the current validator set transition nonce. + fn get_nonce(&self, caller: &Call) -> Option<::util::U256> { + match self.provider.transition_nonce(caller).wait() { + Ok(nonce) => Some(nonce), + Err(s) => { + debug!(target: "engine", "Unable to fetch transition nonce: {}", s); + None + } + } + } + + // Whether the header matches the expected bloom. + // + // The expected log should have 3 topics: + // 1. ETHABI-encoded log name. + // 2. the block's parent hash. + // 3. the "nonce": n for the nth transition in history. + // + // We can only search for the first 2, since we don't have the third + // just yet. + // + // The parent hash is included to prevent + // malicious actors from brute forcing other logs that would + // produce the same bloom. + // + // The log data is an array of all new validator addresses. + fn expected_bloom(&self, header: &Header) -> LogBloom { + LogEntry { + address: self.address, + topics: vec![*EVENT_NAME_HASH, *header.parent_hash()], + data: Vec::new(), // irrelevant for bloom. + }.bloom() + } } impl ValidatorSet for ValidatorSafeContract { @@ -75,17 +126,80 @@ impl ValidatorSet for ValidatorSafeContract { .and_then(|c| c.call_contract(id, addr, data))) } - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + fn proof_required(&self, header: &Header, _block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) -> ::engines::RequiresProof { - // TODO: check blooms first and then logs for the - // ValidatorsChanged([parent_hash, nonce], new_validators) log event. - ::engines::RequiresProof::No + let bloom = self.expected_bloom(header); + let header_bloom = header.log_bloom(); + + if &bloom & header_bloom != bloom { return ::engines::RequiresProof::No } + + match receipts { + None => ::engines::RequiresProof::Unsure(::engines::Unsure::NeedsReceipts), + Some(receipts) => { + let check_log = |log: &LogEntry| { + log.address == self.address && + log.topics.len() == 3 && + log.topics[0] == *EVENT_NAME_HASH && + log.topics[1] == *header.parent_hash() + // don't have anything to compare nonce to yet. + }; + + let event = Provider::contract(&self.provider) + .event("ValidatorsChanged".into()) + .expect("Contract known ahead of time to have `ValidatorsChanged` event; qed"); + + let mut decoded_events = receipts.iter() + .filter(|r| &bloom & &r.log_bloom == bloom) + .flat_map(|r| r.logs.iter()) + .filter(move |l| check_log(l)) + .filter_map(|log| { + let topics = log.topics.iter().map(|x| x.0.clone()).collect(); + match event.decode_log(topics, log.data.clone()) { + Ok(decoded) => Some(decoded), + Err(_) => None, + } + }); + + + // TODO: are multiple transitions per block possible? + match decoded_events.next() { + None => ::engines::RequiresProof::No, + Some(matched_event) => { + // decode log manually until the native contract generator is + // good enough to do it for us. + let &(_, _, ref nonce_token) = &matched_event.params[2]; + let &(_, _, ref validators_token) = &matched_event.params[3]; + + let nonce: Option = nonce_token.clone().to_uint() + .map(H256).map(Into::into); + let validators = validators_token.clone().to_array() + .and_then(|a| a.into_iter() + .map(|x| x.to_address().map(H160)) + .collect::>>() + ); + + match (nonce, validators) { + (Some(nonce), Some(validators)) => + ::engines::RequiresProof::Yes(Some(encode_proof(nonce, &validators))), + _ => { + debug!(target: "engine", "Successfully decoded log turned out to be bad."); + ::engines::RequiresProof::No + } + } + } + } + } + } } - fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { - self.get_list(caller).map(|list| ::rlp::encode_list(&list.into_inner()).to_vec()) - .ok_or_else(|| "Caller insufficient to get validator list.".into()) + // the proof we generate is an RLP list containing two parts. + // (nonce, validators) + fn generate_proof(&self, _header: &Header, caller: &Call) -> Result, String> { + match (self.get_nonce(caller), self.get_list(caller)) { + (Some(nonce), Some(list)) => Ok(encode_proof(nonce, &list.into_inner())), + _ => Err("Caller insufficient to generate validator proof.".into()), + } } fn contains_with_caller(&self, block_hash: &H256, address: &Address, caller: &Call) -> bool { From 715d5daafe1f1c32878aea26ba32ad038bb4f9ca Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 13 Apr 2017 20:24:21 +0200 Subject: [PATCH 09/22] ChainVerifier for memoizing validator sets --- ethcore/res/ethereum/tests | 2 +- ethcore/src/client/client.rs | 22 ++ ethcore/src/engines/authority_round.rs | 197 ++++++++++++------ ethcore/src/engines/basic_authority.rs | 43 +++- ethcore/src/engines/chain_verifier.rs | 42 ++++ ethcore/src/engines/mod.rs | 45 ++-- ethcore/src/engines/validator_set/contract.rs | 6 +- ethcore/src/engines/validator_set/mod.rs | 5 +- ethcore/src/engines/validator_set/multi.rs | 4 + .../engines/validator_set/safe_contract.rs | 9 + .../src/engines/validator_set/simple_list.rs | 7 +- ethcore/src/ethereum/ethash.rs | 19 +- 12 files changed, 311 insertions(+), 90 deletions(-) create mode 100644 ethcore/src/engines/chain_verifier.rs diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index 9028c4801..d52059307 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit 9028c4801fd39fbb71a9796979182549a24e81c8 +Subproject commit d520593078fa0849dcd1f907e44ed0a616892e33 diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 4bd29d100..13cb95afe 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -569,11 +569,33 @@ impl Client { //let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new)); let mut batch = DBTransaction::new(); + + // generate validation proof if the engine requires them. + // TODO: make conditional? + let generate_proof = { + use engines::RequiresProof; + match self.engine.proof_required(block.header(), Some(block_data), Some(&receipts)) { + RequiresProof::Yes(_) => true, + RequiresProof::No => false, + RequiresProof::Unsure(_) => { + warn!(target: "client", "Detected invalid engine implementation."); + warn!(target: "client", "Engine claims to require more block data, but everything provided."); + false + } + } + }; + // CHECK! I *think* this is fine, even if the state_root is equal to another // already-imported block of the same number. // TODO: Prove it with a test. let mut state = block.drain(); + if generate_proof { + debug!(target: "client", "Generating validation proof for block {}", hash); + + // TODO + } + state.journal_under(&mut batch, number, hash).expect("DB commit failed"); let route = chain.insert_block(&mut batch, block_data, receipts); self.tracedb.read().import(&mut batch, TraceImportRequest { diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index e57fb27bc..8e8450768 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -37,7 +37,7 @@ use transaction::UnverifiedTransaction; use client::{Client, EngineClient}; use state::CleanupMode; use super::signer::EngineSigner; -use super::validator_set::{ValidatorSet, new_validator_set}; +use super::validator_set::{ValidatorSet, SimpleList, new_validator_set}; /// `AuthorityRound` params. #[derive(Debug, PartialEq)] @@ -75,27 +75,76 @@ impl From for AuthorityRoundParams { } } -/// Engine using `AuthorityRound` proof-of-work consensus algorithm, suitable for Ethereum -/// mainnet chains in the Olympic, Frontier and Homestead eras. +// Helper for managing the step. +#[derive(Debug)] +struct Step { + calibrate: bool, // whether calibration is enabled. + inner: AtomicUsize, + duration: Duration, +} + +impl Step { + fn load(&self) -> usize { self.inner.load(AtomicOrdering::SeqCst) } + fn duration_remaining(&self) -> Duration { + let now = unix_now(); + let step_end = self.duration * (self.load() as u32 + 1); + if step_end > now { + step_end - now + } else { + Duration::from_secs(0) + } + } + fn increment(&self) { + self.inner.fetch_add(1, AtomicOrdering::SeqCst); + } + fn calibrate(&self) { + if self.calibrate { + let new_step = unix_now().as_secs() / self.duration.as_secs(); + self.inner.store(new_step as usize, AtomicOrdering::SeqCst); + } + } + fn is_future(&self, given: usize) -> bool { + if given > self.load() + 1 { + // Make absolutely sure that the given step is correct. + self.calibrate(); + given > self.load() + 1 + } else { + false + } + } +} + +/// Engine using `AuthorityRound` proof-of-authority BFT consensus. pub struct AuthorityRound { params: CommonParams, gas_limit_bound_divisor: U256, block_reward: U256, registrar: Address, - step_duration: Duration, builtins: BTreeMap, transition_service: IoService<()>, - step: AtomicUsize, + step: Arc, proposed: AtomicBool, client: RwLock>>, signer: EngineSigner, validators: Box, - /// Is this Engine just for testing (prevents step calibration). - calibrate_step: bool, validate_score_transition: u64, eip155_transition: u64, } +// header-chain validator. +struct ChainVerifier { + step: Arc, + subchain_validators: SimpleList, +} + +impl super::ChainVerifier for ChainVerifier { + fn verify_light(&self, header: &Header) -> Result<(), Error> { + // always check the seal since it's fast. + // nothing heavier to do. + verify_external(header, &self.subchain_validators, &*self.step) + } +} + fn header_step(header: &Header) -> Result { UntrustedRlp::new(&header.seal().get(0).expect("was either checked with verify_block_basic or is genesis; has 2 fields; qed (Make sure the spec file has a correct genesis seal)")).as_val() } @@ -104,6 +153,26 @@ fn header_signature(header: &Header) -> Result { UntrustedRlp::new(&header.seal().get(1).expect("was checked with verify_block_basic; has 2 fields; qed")).as_val::().map(Into::into) } +fn verify_external(header: &Header, validators: &ValidatorSet, step: &Step) -> Result<(), Error> { + let header_step = header_step(header)?; + + // Give one step slack if step is lagging, double vote is still not possible. + if step.is_future(header_step) { + trace!(target: "engine", "verify_block_unordered: block from the future"); + validators.report_benign(header.author()); + Err(BlockError::InvalidSeal)? + } else { + let proposer_signature = header_signature(header)?; + let correct_proposer = validators.get(header.parent_hash(), header_step); + if !verify_address(&correct_proposer, &proposer_signature, &header.bare_hash())? { + trace!(target: "engine", "verify_block_unordered: bad proposer for step: {}", header_step); + Err(EngineError::NotProposer(Mismatch { expected: correct_proposer, found: header.author().clone() }))? + } else { + Ok(()) + } + } +} + trait AsMillis { fn as_millis(&self) -> u64; } @@ -125,15 +194,17 @@ impl AuthorityRound { gas_limit_bound_divisor: our_params.gas_limit_bound_divisor, block_reward: our_params.block_reward, registrar: our_params.registrar, - step_duration: our_params.step_duration, builtins: builtins, transition_service: IoService::<()>::start()?, - step: AtomicUsize::new(initial_step), + step: Arc::new(Step { + inner: AtomicUsize::new(initial_step), + calibrate: our_params.start_step.is_none(), + duration: our_params.step_duration, + }), proposed: AtomicBool::new(false), client: RwLock::new(None), signer: Default::default(), validators: new_validator_set(our_params.validators), - calibrate_step: our_params.start_step.is_none(), validate_score_transition: our_params.validate_score_transition, eip155_transition: our_params.eip155_transition, }); @@ -145,22 +216,6 @@ impl AuthorityRound { Ok(engine) } - fn calibrate_step(&self) { - if self.calibrate_step { - self.step.store((unix_now().as_secs() / self.step_duration.as_secs()) as usize, AtomicOrdering::SeqCst); - } - } - - fn remaining_step_duration(&self) -> Duration { - let now = unix_now(); - let step_end = self.step_duration * (self.step.load(AtomicOrdering::SeqCst) as u32 + 1); - if step_end > now { - step_end - now - } else { - Duration::from_secs(0) - } - } - fn step_proposer(&self, bh: &H256, step: usize) -> Address { self.validators.get(bh, step) } @@ -168,16 +223,6 @@ impl AuthorityRound { fn is_step_proposer(&self, bh: &H256, step: usize, address: &Address) -> bool { self.step_proposer(bh, step) == *address } - - fn is_future_step(&self, step: usize) -> bool { - if step > self.step.load(AtomicOrdering::SeqCst) + 1 { - // Make absolutely sure that the step is correct. - self.calibrate_step(); - step > self.step.load(AtomicOrdering::SeqCst) + 1 - } else { - false - } - } } fn unix_now() -> Duration { @@ -193,7 +238,8 @@ const ENGINE_TIMEOUT_TOKEN: TimerToken = 23; impl IoHandler<()> for TransitionHandler { fn initialize(&self, io: &IoContext<()>) { if let Some(engine) = self.engine.upgrade() { - io.register_timer_once(ENGINE_TIMEOUT_TOKEN, engine.remaining_step_duration().as_millis()) + let remaining = engine.step.duration_remaining(); + io.register_timer_once(ENGINE_TIMEOUT_TOKEN, remaining.as_millis()) .unwrap_or_else(|e| warn!(target: "engine", "Failed to start consensus step timer: {}.", e)) } } @@ -202,7 +248,8 @@ impl IoHandler<()> for TransitionHandler { if timer == ENGINE_TIMEOUT_TOKEN { if let Some(engine) = self.engine.upgrade() { engine.step(); - io.register_timer_once(ENGINE_TIMEOUT_TOKEN, engine.remaining_step_duration().as_millis()) + let remaining = engine.step.duration_remaining(); + io.register_timer_once(ENGINE_TIMEOUT_TOKEN, remaining.as_millis()) .unwrap_or_else(|e| warn!(target: "engine", "Failed to restart consensus step timer: {}.", e)) } } @@ -224,7 +271,7 @@ impl Engine for AuthorityRound { fn builtins(&self) -> &BTreeMap { &self.builtins } fn step(&self) { - self.step.fetch_add(1, AtomicOrdering::SeqCst); + self.step.increment(); self.proposed.store(false, AtomicOrdering::SeqCst); if let Some(ref weak) = *self.client.read() { if let Some(c) = weak.upgrade() { @@ -247,7 +294,7 @@ impl Engine for AuthorityRound { fn populate_from_parent(&self, header: &mut Header, parent: &Header, gas_floor_target: U256, _gas_ceil_target: U256) { // Chain scoring: total weight is sqrt(U256::max_value())*height - step - let new_difficulty = U256::from(U128::max_value()) + header_step(parent).expect("Header has been verified; qed").into() - self.step.load(AtomicOrdering::SeqCst).into(); + let new_difficulty = U256::from(U128::max_value()) + header_step(parent).expect("Header has been verified; qed").into() - self.step.load().into(); header.set_difficulty(new_difficulty); header.set_gas_limit({ let gas_limit = parent.gas_limit().clone(); @@ -271,7 +318,7 @@ impl Engine for AuthorityRound { fn generate_seal(&self, block: &ExecutedBlock) -> Seal { if self.proposed.load(AtomicOrdering::SeqCst) { return Seal::None; } let header = block.header(); - let step = self.step.load(AtomicOrdering::SeqCst); + let step = self.step.load(); if self.is_step_proposer(header.parent_hash(), step, header.author()) { if let Ok(signature) = self.signer.sign(header.bare_hash()) { trace!(target: "engine", "generate_seal: Issuing a block for step {}.", step); @@ -319,32 +366,19 @@ impl Engine for AuthorityRound { Ok(()) } - /// Do the validator and gas limit validation. + /// Do the step and gas limit validation. fn verify_block_family(&self, header: &Header, parent: &Header, _block: Option<&[u8]>) -> Result<(), Error> { let step = header_step(header)?; - // Give one step slack if step is lagging, double vote is still not possible. - if self.is_future_step(step) { - trace!(target: "engine", "verify_block_unordered: block from the future"); - self.validators.report_benign(header.author()); - Err(BlockError::InvalidSeal)? - } else { - let proposer_signature = header_signature(header)?; - let correct_proposer = self.step_proposer(header.parent_hash(), step); - if !verify_address(&correct_proposer, &proposer_signature, &header.bare_hash())? { - trace!(target: "engine", "verify_block_unordered: bad proposer for step: {}", step); - Err(EngineError::NotProposer(Mismatch { expected: correct_proposer, found: header.author().clone() }))? - } - } // Do not calculate difficulty for genesis blocks. if header.number() == 0 { return Err(From::from(BlockError::RidiculousNumber(OutOfBounds { min: Some(1), max: None, found: header.number() }))); } - // Check if parent is from a previous step. + // Ensure header is from the step after parent. let parent_step = header_step(parent)?; - if step == parent_step { - trace!(target: "engine", "Multiple blocks proposed for step {}.", step); + if step <= parent_step { + trace!(target: "engine", "Multiple blocks proposed for step {}.", parent_step); self.validators.report_malicious(header.author()); Err(EngineError::DoubleVote(header.author().clone()))?; } @@ -358,6 +392,11 @@ impl Engine for AuthorityRound { Ok(()) } + // Check the validators. + fn verify_block_external(&self, header: &Header, _block: Option<&[u8]>) -> Result<(), Error> { + verify_external(header, &*self.validators, &*self.step) + } + // the proofs we need just allow us to get the full validator set. fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { self.validators.generate_proof(header, caller) @@ -370,6 +409,16 @@ impl Engine for AuthorityRound { self.validators.proof_required(header, block, receipts) } + fn chain_verifier(&self, header: &Header, proof: Bytes) -> Result, Error> { + // extract a simple list from the proof. + let simple_list = self.validators.chain_verifier(header, proof)?; + + Ok(Box::new(ChainVerifier { + step: self.step.clone(), + subchain_validators: simple_list, + })) + } + fn verify_transaction_basic(&self, t: &UnverifiedTransaction, header: &Header) -> result::Result<(), Error> { t.check_low_s()?; @@ -453,7 +502,7 @@ mod tests { let mut header: Header = Header::default(); header.set_seal(vec![encode(&H520::default()).to_vec()]); - let verify_result = engine.verify_block_family(&header, &Default::default(), None); + let verify_result = engine.verify_block_external(&header, None); assert!(verify_result.is_err()); } @@ -507,9 +556,11 @@ mod tests { // Two validators. // Spec starts with step 2. header.set_seal(vec![encode(&2usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); - assert!(engine.verify_block_family(&header, &parent_header, None).is_err()); + assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); + assert!(engine.verify_block_external(&header, None).is_err()); header.set_seal(vec![encode(&1usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); + assert!(engine.verify_block_external(&header, None).is_ok()); } #[test] @@ -532,7 +583,33 @@ mod tests { // Spec starts with step 2. header.set_seal(vec![encode(&1usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); + assert!(engine.verify_block_external(&header, None).is_ok()); header.set_seal(vec![encode(&5usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); + assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); + assert!(engine.verify_block_external(&header, None).is_err()); + } + + #[test] + fn rejects_step_backwards() { + let tap = AccountProvider::transient_provider(); + let addr = tap.insert_account(Secret::from_slice(&"0".sha3()).unwrap(), "0").unwrap(); + + let mut parent_header: Header = Header::default(); + parent_header.set_seal(vec![encode(&4usize).to_vec()]); + parent_header.set_gas_limit(U256::from_str("222222").unwrap()); + let mut header: Header = Header::default(); + header.set_number(1); + header.set_gas_limit(U256::from_str("222222").unwrap()); + header.set_author(addr); + + let engine = Spec::new_test_round().engine; + + let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); + // Two validators. + // Spec starts with step 2. + header.set_seal(vec![encode(&5usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); + assert!(engine.verify_block_family(&header, &parent_header, None).is_ok()); + header.set_seal(vec![encode(&3usize).to_vec(), encode(&(&*signature as &[u8])).to_vec()]); assert!(engine.verify_block_family(&header, &parent_header, None).is_err()); } } diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index a785b360a..25cdd765f 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -31,7 +31,7 @@ use ethjson; use header::Header; use client::Client; use super::signer::EngineSigner; -use super::validator_set::{ValidatorSet, new_validator_set}; +use super::validator_set::{ValidatorSet, SimpleList, new_validator_set}; /// `BasicAuthority` params. #[derive(Debug, PartialEq)] @@ -51,6 +51,27 @@ impl From for BasicAuthorityParams { } } +struct ChainVerifier(SimpleList); + +impl super::ChainVerifier for ChainVerifier { + fn verify_light(&self, header: &Header) -> Result<(), Error> { + verify_external(header, &self.0) + } +} + +fn verify_external(header: &Header, validators: &ValidatorSet) -> Result<(), Error> { + use rlp::UntrustedRlp; + + // Check if the signature belongs to a validator, can depend on parent state. + let sig = UntrustedRlp::new(&header.seal()[0]).as_val::()?; + let signer = public_to_address(&recover(&sig.into(), &header.bare_hash())?); + + match validators.contains(header.parent_hash(), &signer) { + false => Err(BlockError::InvalidSeal.into()), + true => Ok(()) + } +} + /// Engine using `BasicAuthority`, trivial proof-of-authority consensus. pub struct BasicAuthority { params: CommonParams, @@ -137,15 +158,6 @@ impl Engine for BasicAuthority { } fn verify_block_family(&self, header: &Header, parent: &Header, _block: Option<&[u8]>) -> result::Result<(), Error> { - use rlp::UntrustedRlp; - - // Check if the signature belongs to a validator, can depend on parent state. - let sig = UntrustedRlp::new(&header.seal()[0]).as_val::()?; - let signer = public_to_address(&recover(&sig.into(), &header.bare_hash())?); - if !self.validators.contains(header.parent_hash(), &signer) { - return Err(BlockError::InvalidSeal)?; - } - // Do not calculate difficulty for genesis blocks. if header.number() == 0 { return Err(From::from(BlockError::RidiculousNumber(OutOfBounds { min: Some(1), max: None, found: header.number() }))); @@ -164,6 +176,10 @@ impl Engine for BasicAuthority { Ok(()) } + fn verify_block_external(&self, header: &Header, _block: Option<&[u8]>) -> Result<(), Error> { + verify_external(header, &*self.validators) + } + // the proofs we need just allow us to get the full validator set. fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { self.validators.generate_proof(header, caller) @@ -176,6 +192,13 @@ impl Engine for BasicAuthority { self.validators.proof_required(header, block, receipts) } + fn chain_verifier(&self, header: &Header, proof: Bytes) -> Result, Error> { + // extract a simple list from the proof. + let simple_list = self.validators.chain_verifier(header, proof)?; + + Ok(Box::new(ChainVerifier(simple_list))) + } + fn register_client(&self, client: Weak) { self.validators.register_contract(client); } diff --git a/ethcore/src/engines/chain_verifier.rs b/ethcore/src/engines/chain_verifier.rs new file mode 100644 index 000000000..399832fe9 --- /dev/null +++ b/ethcore/src/engines/chain_verifier.rs @@ -0,0 +1,42 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +// Chain verifiers. + +use error::Error; +use header::Header; + +/// Sequential chain verifier. +/// +/// See docs on `Engine` relating to proving functions for more details. +/// Headers here will have already passed "basic" verification. +pub trait ChainVerifier { + /// Lightly verify the next block header. + /// This may not be a header that requires a proof. + fn verify_light(&self, header: &Header) -> Result<(), Error>; + + /// Perform potentially heavier checks on the next block header. + fn verify_heavy(&self, header: &Header) -> Result<(), Error> { + self.verify_light(header) + } +} + +/// No-op chain verifier. +pub struct NoOp; + +impl ChainVerifier for NoOp { + fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } +} diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index f1e0a4d43..7d4906dd2 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -16,20 +16,22 @@ //! Consensus engine specification and basic implementations. -mod transition; -mod vote_collector; -mod null_engine; -mod instant_seal; -mod basic_authority; mod authority_round; -mod tendermint; -mod validator_set; +mod basic_authority; +mod chain_verifier; +mod instant_seal; +mod null_engine; mod signer; +mod tendermint; +mod transition; +mod validator_set; +mod vote_collector; -pub use self::null_engine::NullEngine; -pub use self::instant_seal::InstantSeal; -pub use self::basic_authority::BasicAuthority; pub use self::authority_round::AuthorityRound; +pub use self::basic_authority::BasicAuthority; +pub use self::chain_verifier::ChainVerifier; +pub use self::instant_seal::InstantSeal; +pub use self::null_engine::NullEngine; pub use self::tendermint::Tendermint; use std::sync::Weak; @@ -182,6 +184,9 @@ pub trait Engine : Sync + Send { /// may be provided for additional checks. Returns either a null `Ok` or a general error detailing the problem with import. fn verify_block_family(&self, _header: &Header, _parent: &Header, _block: Option<&[u8]>) -> Result<(), Error> { Ok(()) } + /// Phase 4 verification. Verify block header against potentially external data. + fn verify_block_external(&self, _header: &Header, _block: Option<&[u8]>) -> Result<(), Error> { Ok(()) } + /// Additional verification for transactions in blocks. // TODO: Add flags for which bits of the transaction to check. // TODO: consider including State in the params. @@ -214,13 +219,21 @@ pub trait Engine : Sync + Send { self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) } - /// Re-do all verification for a header with the given contract-calling interface + /// Generate validation proof. /// /// This will be used to generate proofs of validation as well as verify them. /// Must be called on blocks that have already passed basic verification. /// /// Return the "validation proof" generated. - fn prove_with_caller(&self, _header: &Header, _caller: &Call) -> Result { + /// This must be usable to generate a `ChainVerifier` for verifying all blocks + /// from the supplied header up to the next one where proof is required. + /// + /// For example, for PoA chains the proof will be a validator set, + /// and the corresponding `ChainVerifier` can be used to correctly validate + /// all blocks produced under that `ValidatorSet` + fn prove_with_caller(&self, _header: &Header, _caller: &Call) + -> Result, Error> + { Ok(Vec::new()) } @@ -231,6 +244,14 @@ pub trait Engine : Sync + Send { RequiresProof::No } + /// Create a subchain verifier from validation proof. + /// + /// The proof should be one generated by `prove_with_caller`. + /// See docs of `prove_with_caller` for description. + fn chain_verifier(&self, _header: &Header, _proof: Bytes) -> Result, Error> { + Ok(Box::new(self::chain_verifier::NoOp)) + } + /// Populate a header's fields based on its parent's header. /// Usually implements the chain scoring rule based on weight. /// The gas floor target must not be lower than the engine's minimum gas limit. diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index 5c4af4ba7..db6e8ed2c 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -76,6 +76,10 @@ impl ValidatorSet for ValidatorContract { self.validators.generate_proof(header, caller) } + fn chain_verifier(&self, header: &Header, proof: Vec) -> Result { + self.validators.chain_verifier(header, proof) + } + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { self.validators.contains_with_caller(bh, address, caller) } @@ -153,7 +157,7 @@ mod tests { header.set_parent_hash(client.chain_info().best_block_hash); // `reportBenign` when the designated proposer releases block from the future (bad clock). - assert!(client.engine().verify_block_family(&header, &header, None).is_err()); + assert!(client.engine().verify_block_external(&header, None).is_err()); // Seal a block. client.engine().step(); assert_eq!(client.chain_info().best_block_number, 1); diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index d5abf430d..48269e20f 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -28,7 +28,7 @@ use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; use header::Header; -use self::simple_list::SimpleList; +pub use self::simple_list::SimpleList; use self::contract::ValidatorContract; use self::safe_contract::ValidatorSafeContract; use self::multi::Multi; @@ -90,6 +90,9 @@ pub trait ValidatorSet: Send + Sync { /// Otherwise, generated proofs may be wrong. fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String>; + /// Create a fully self-contained validator set from the given proof. + fn chain_verifier(&self, header: &Header, proof: Vec) -> Result; + /// Checks if a given address is a validator, with the given function /// for executing synchronous calls to contracts. fn contains_with_caller(&self, parent_block_hash: &H256, address: &Address, caller: &Call) -> bool; diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index 78270c4af..b7a13c5e3 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -82,6 +82,10 @@ impl ValidatorSet for Multi { self.correct_set_by_number(header.number()).generate_proof(header, caller) } + fn chain_verifier(&self, header: &Header, proof: Vec) -> Result { + self.correct_set_by_number(header.number()).chain_verifier(header, proof) + } + fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { self.correct_set(BlockId::Hash(*bh)) .map_or(false, |set| set.contains_with_caller(bh, address, caller)) diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index de7ecb3a2..2af389484 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -202,6 +202,15 @@ impl ValidatorSet for ValidatorSafeContract { } } + fn chain_verifier(&self, _header: &Header, proof: Vec) -> Result { + use rlp::UntrustedRlp; + + let rlp = UntrustedRlp::new(&proof); + let validators: Vec
= rlp.list_at(1)?; + + Ok(SimpleList::new(validators)) + } + fn contains_with_caller(&self, block_hash: &H256, address: &Address, caller: &Call) -> bool { let mut guard = self.validators.write(); let maybe_existing = guard diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index e513212a6..339585d7d 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -22,7 +22,8 @@ use engines::Call; use header::Header; use super::ValidatorSet; -#[derive(Debug, PartialEq, Eq, Default)] +/// Validator set containing a known set of addresses. +#[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct SimpleList { validators: Vec
, } @@ -62,6 +63,10 @@ impl ValidatorSet for SimpleList { Ok(Vec::new()) } + fn chain_verifier(&self, _header: &Header, _: Vec) -> Result { + Ok(self.clone()) + } + fn contains_with_caller(&self, _bh: &H256, address: &Address, _: &Call) -> bool { self.validators.contains(address) } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 5bc31c5d9..585fd29aa 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -139,17 +139,24 @@ pub struct Ethash { impl Ethash { /// Create a new instance of Ethash engine - pub fn new(params: CommonParams, ethash_params: EthashParams, builtins: BTreeMap) -> Self { - Ethash { + pub fn new(params: CommonParams, ethash_params: EthashParams, builtins: BTreeMap) -> Arc { + Arc::new(Ethash { params: params, ethash_params: ethash_params, builtins: builtins, pow: EthashManager::new(), - } + }) } } -impl Engine for Ethash { +impl ::engines::ChainVerifier for Arc { + fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } + fn verify_heavy(&self, header: &Header) -> Result<(), Error> { + self.verify_block_unordered(header, None) + } +} + +impl Engine for Arc { fn name(&self) -> &str { "Ethash" } fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) } // Two fields - mix @@ -385,6 +392,10 @@ impl Engine for Ethash { Ok(()) } + + fn chain_verifier(&self, _header: &Header, _proof: Vec) -> Result, Error> { + Ok(Box::new(self.clone())) + } } // Try to round gas_limit a bit so that: From e1fa4ab8ec3c1e86a16539c6ac96c07582112ee6 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 13 Apr 2017 20:31:29 +0200 Subject: [PATCH 10/22] verify_external in verifier trait --- ethcore/src/client/client.rs | 6 ++++++ ethcore/src/verification/canon_verifier.rs | 4 ++++ ethcore/src/verification/noop_verifier.rs | 4 ++++ ethcore/src/verification/verifier.rs | 2 ++ 4 files changed, 16 insertions(+) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 13cb95afe..03353a2a4 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -380,6 +380,12 @@ impl Client { return Err(()); }; + let verify_external_result = self.verifier.verify_block_external(header, &block.bytes, engine); + if let Err(e) = verify_external_result { + warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + // Check if Parent is in chain let chain_has_parent = chain.block_header(header.parent_hash()); if let Some(parent) = chain_has_parent { diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs index 849f7caad..a5d8fe73f 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/src/verification/canon_verifier.rs @@ -34,4 +34,8 @@ impl Verifier for CanonVerifier { fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error> { verification::verify_block_final(expected, got) } + + fn verify_block_external(&self, header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> { + engine.verify_block_external(header, Some(bytes)) + } } diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index 2fcd877f5..8464ba1e2 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -34,4 +34,8 @@ impl Verifier for NoopVerifier { fn verify_block_final(&self, _expected: &Header, _got: &Header) -> Result<(), Error> { Ok(()) } + + fn verify_block_external(&self, _header: &Header, _bytes: &[u8], _engine: &Engine) -> Result<(), Error> { + Ok(()) + } } diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs index e5dabd392..55b711c1c 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/src/verification/verifier.rs @@ -27,4 +27,6 @@ pub trait Verifier: Send + Sync { fn verify_block_family(&self, header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>; /// Do a final verification check for an enacted header vs its expected counterpart. fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error>; + /// Verify a block, inspecing external state. + fn verify_block_external(&self, header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error>; } From f6f9816ef4a35d5e3af0c8f59dd1e4df80260b54 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 18 Apr 2017 14:19:10 +0200 Subject: [PATCH 11/22] epoch verifier rename --- ethcore/src/client/client.rs | 10 ++-- ethcore/src/engines/authority_round.rs | 23 +++++---- ethcore/src/engines/basic_authority.rs | 31 +++++++----- .../{chain_verifier.rs => epoch_verifier.rs} | 18 ++++--- ethcore/src/engines/mod.rs | 48 ++++++++++--------- ethcore/src/engines/validator_set/contract.rs | 14 +++--- ethcore/src/engines/validator_set/mod.rs | 26 ++++++---- ethcore/src/engines/validator_set/multi.rs | 32 +++++++------ .../engines/validator_set/safe_contract.rs | 27 ++++++----- .../src/engines/validator_set/simple_list.rs | 14 +++--- ethcore/src/ethereum/ethash.rs | 13 ++++- 11 files changed, 148 insertions(+), 108 deletions(-) rename ethcore/src/engines/{chain_verifier.rs => epoch_verifier.rs} (75%) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 03353a2a4..a91294e27 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -579,11 +579,11 @@ impl Client { // generate validation proof if the engine requires them. // TODO: make conditional? let generate_proof = { - use engines::RequiresProof; - match self.engine.proof_required(block.header(), Some(block_data), Some(&receipts)) { - RequiresProof::Yes(_) => true, - RequiresProof::No => false, - RequiresProof::Unsure(_) => { + use engines::EpochChange; + match self.engine.is_epoch_end(block.header(), Some(block_data), Some(&receipts)) { + EpochChange::Yes(_) => true, + EpochChange::No => false, + EpochChange::Unsure(_) => { warn!(target: "client", "Detected invalid engine implementation."); warn!(target: "client", "Engine claims to require more block data, but everything provided."); false diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index 8e8450768..1f3edf57c 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -132,12 +132,14 @@ pub struct AuthorityRound { } // header-chain validator. -struct ChainVerifier { +struct EpochVerifier { + epoch_number: U256, step: Arc, subchain_validators: SimpleList, } -impl super::ChainVerifier for ChainVerifier { +impl super::EpochVerifier for EpochVerifier { + fn epoch_number(&self) -> U256 { self.epoch_number.clone() } fn verify_light(&self, header: &Header) -> Result<(), Error> { // always check the seal since it's fast. // nothing heavier to do. @@ -398,22 +400,23 @@ impl Engine for AuthorityRound { } // the proofs we need just allow us to get the full validator set. - fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { - self.validators.generate_proof(header, caller) + fn epoch_proof(&self, header: &Header, caller: &Call) -> Result { + self.validators.epoch_proof(header, caller) .map_err(|e| EngineError::InsufficientProof(e).into()) } - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> super::RequiresProof + fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> super::EpochChange { - self.validators.proof_required(header, block, receipts) + self.validators.is_epoch_end(header, block, receipts) } - fn chain_verifier(&self, header: &Header, proof: Bytes) -> Result, Error> { + fn epoch_verifier(&self, header: &Header, proof: &[u8]) -> Result, Error> { // extract a simple list from the proof. - let simple_list = self.validators.chain_verifier(header, proof)?; + let (num, simple_list) = self.validators.epoch_set(header, proof)?; - Ok(Box::new(ChainVerifier { + Ok(Box::new(EpochVerifier { + epoch_number: num, step: self.step.clone(), subchain_validators: simple_list, })) diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index 25cdd765f..94c3fe578 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -23,7 +23,7 @@ use account_provider::AccountProvider; use block::*; use builtin::Builtin; use spec::CommonParams; -use engines::{Engine, EngineError, Seal, Call, RequiresProof}; +use engines::{Engine, EngineError, Seal, Call, EpochChange}; use env_info::EnvInfo; use error::{BlockError, Error}; use evm::Schedule; @@ -51,11 +51,15 @@ impl From for BasicAuthorityParams { } } -struct ChainVerifier(SimpleList); +struct EpochVerifier { + epoch_number: U256, + list: SimpleList, +} -impl super::ChainVerifier for ChainVerifier { +impl super::EpochVerifier for EpochVerifier { + fn epoch_number(&self) -> U256 { self.epoch_number.clone() } fn verify_light(&self, header: &Header) -> Result<(), Error> { - verify_external(header, &self.0) + verify_external(header, &self.list) } } @@ -181,22 +185,25 @@ impl Engine for BasicAuthority { } // the proofs we need just allow us to get the full validator set. - fn prove_with_caller(&self, header: &Header, caller: &Call) -> Result { - self.validators.generate_proof(header, caller) + fn epoch_proof(&self, header: &Header, caller: &Call) -> Result { + self.validators.epoch_proof(header, caller) .map_err(|e| EngineError::InsufficientProof(e).into()) } - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> RequiresProof + fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> EpochChange { - self.validators.proof_required(header, block, receipts) + self.validators.is_epoch_end(header, block, receipts) } - fn chain_verifier(&self, header: &Header, proof: Bytes) -> Result, Error> { + fn epoch_verifier(&self, header: &Header, proof: &[u8]) -> Result, Error> { // extract a simple list from the proof. - let simple_list = self.validators.chain_verifier(header, proof)?; + let (num, simple_list) = self.validators.epoch_set(header, proof)?; - Ok(Box::new(ChainVerifier(simple_list))) + Ok(Box::new(EpochVerifier { + epoch_number: num, + list: simple_list, + })) } fn register_client(&self, client: Weak) { diff --git a/ethcore/src/engines/chain_verifier.rs b/ethcore/src/engines/epoch_verifier.rs similarity index 75% rename from ethcore/src/engines/chain_verifier.rs rename to ethcore/src/engines/epoch_verifier.rs index 399832fe9..fd0847e76 100644 --- a/ethcore/src/engines/chain_verifier.rs +++ b/ethcore/src/engines/epoch_verifier.rs @@ -14,18 +14,21 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -// Chain verifiers. +// Epoch verifiers. use error::Error; use header::Header; +use util::U256; -/// Sequential chain verifier. +/// Verifier for all blocks within an epoch without accessing /// /// See docs on `Engine` relating to proving functions for more details. -/// Headers here will have already passed "basic" verification. -pub trait ChainVerifier { +pub trait EpochVerifier: Sync { + /// Get the epoch number. + fn epoch_number(&self) -> U256; + /// Lightly verify the next block header. - /// This may not be a header that requires a proof. + /// This may not be a header belonging to a different epoch. fn verify_light(&self, header: &Header) -> Result<(), Error>; /// Perform potentially heavier checks on the next block header. @@ -34,9 +37,10 @@ pub trait ChainVerifier { } } -/// No-op chain verifier. +/// Special "no-op" verifier for stateless, epoch-less engines. pub struct NoOp; -impl ChainVerifier for NoOp { +impl EpochVerifier for NoOp { + fn epoch_number(&self) -> U256 { 0.into() } fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index 7d4906dd2..b67afdf2f 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -18,7 +18,7 @@ mod authority_round; mod basic_authority; -mod chain_verifier; +mod epoch_verifier; mod instant_seal; mod null_engine; mod signer; @@ -29,7 +29,7 @@ mod vote_collector; pub use self::authority_round::AuthorityRound; pub use self::basic_authority::BasicAuthority; -pub use self::chain_verifier::ChainVerifier; +pub use self::epoch_verifier::EpochVerifier; pub use self::instant_seal::InstantSeal; pub use self::null_engine::NullEngine; pub use self::tendermint::Tendermint; @@ -98,18 +98,19 @@ pub enum Seal { /// Type alias for a function we can make calls through synchronously. pub type Call = Fn(Address, Bytes) -> Result; -/// Results of a query of whether a validation proof is necessary at a block. -pub enum RequiresProof { +/// Results of a query of whether an epoch change occurred at the given block. +#[derive(Debug, Clone, PartialEq)] +pub enum EpochChange { /// Cannot determine until more data is passed. Unsure(Unsure), - /// Validation proof not required. + /// No epoch change. No, /// Validation proof required, and the expected output - /// if it can be recovered. - Yes(Option), + Yes(Bytes), } -/// More data required to determine if a validation proof is required. +/// More data required to determine if an epoch change occurred at a given block. +#[derive(Debug, Clone, Copy, PartialEq)] pub enum Unsure { /// Needs the body. NeedsBody, @@ -219,37 +220,38 @@ pub trait Engine : Sync + Send { self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) } - /// Generate validation proof. + /// Generate epoch change proof. /// - /// This will be used to generate proofs of validation as well as verify them. + /// This will be used to generate proofs of epoch change as well as verify them. /// Must be called on blocks that have already passed basic verification. /// - /// Return the "validation proof" generated. - /// This must be usable to generate a `ChainVerifier` for verifying all blocks + /// Return the "epoch proof" generated. + /// This must be usable to generate a `EpochVerifier` for verifying all blocks /// from the supplied header up to the next one where proof is required. /// /// For example, for PoA chains the proof will be a validator set, - /// and the corresponding `ChainVerifier` can be used to correctly validate + /// and the corresponding `EpochVerifier` can be used to correctly validate /// all blocks produced under that `ValidatorSet` - fn prove_with_caller(&self, _header: &Header, _caller: &Call) + fn epoch_proof(&self, _header: &Header, _caller: &Call) -> Result, Error> { Ok(Vec::new()) } - /// Whether a proof is required for the given header. - fn proof_required(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>) - -> RequiresProof + /// Whether an epoch change occurred at the given header. + /// Should not interact with state. + fn is_epoch_end(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>) + -> EpochChange { - RequiresProof::No + EpochChange::No } - /// Create a subchain verifier from validation proof. + /// Create an epoch verifier from validation proof. /// - /// The proof should be one generated by `prove_with_caller`. - /// See docs of `prove_with_caller` for description. - fn chain_verifier(&self, _header: &Header, _proof: Bytes) -> Result, Error> { - Ok(Box::new(self::chain_verifier::NoOp)) + /// The proof should be one generated by `epoch_proof`. + /// See docs of `epoch_proof` for description. + fn epoch_verifier(&self, _header: &Header, _proof: &[u8]) -> Result, Error> { + Ok(Box::new(self::epoch_verifier::NoOp)) } /// Populate a header's fields based on its parent's header. diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index db6e8ed2c..0fdef84a3 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -66,18 +66,18 @@ impl ValidatorSet for ValidatorContract { self.validators.default_caller(id) } - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> ::engines::RequiresProof + fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> ::engines::EpochChange { - self.validators.proof_required(header, block, receipts) + self.validators.is_epoch_end(header, block, receipts) } - fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { - self.validators.generate_proof(header, caller) + fn epoch_proof(&self, header: &Header, caller: &Call) -> Result, String> { + self.validators.epoch_proof(header, caller) } - fn chain_verifier(&self, header: &Header, proof: Vec) -> Result { - self.validators.chain_verifier(header, proof) + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, super::SimpleList), ::error::Error> { + self.validators.epoch_set(header, proof) } fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index 48269e20f..e9dc333c5 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -23,7 +23,7 @@ mod multi; use std::sync::Weak; use ids::BlockId; -use util::{Address, H256}; +use util::{Address, H256, U256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; use header::Header; @@ -72,26 +72,32 @@ pub trait ValidatorSet: Send + Sync { self.count_with_caller(parent, &*default) } - /// Whether a validation proof is required at the given block. + /// Whether this block is the last one in its epoch. /// Usually indicates that the validator set changed at the given block. /// /// Should not inspect state! This is used in situations where /// state is not generally available. /// - /// Return `Ok` with a flag indicating whether it changed at the given header, + /// Return `Yes` or `No` indicating whether it changed at the given header, /// or `Unsure` indicating a need for more information. /// - /// This may or may not be called in a loop. - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> super::RequiresProof; + /// If block or receipts are provided, do not return `Unsure` indicating + /// need for them. + fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> super::EpochChange; - /// Generate a validation proof at the given header. + /// Generate epoch proof. /// Must interact with state only through the given caller! /// Otherwise, generated proofs may be wrong. - fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String>; + fn epoch_proof(&self, header: &Header, caller: &Call) -> Result, String>; - /// Create a fully self-contained validator set from the given proof. - fn chain_verifier(&self, header: &Header, proof: Vec) -> Result; + /// Recover the validator set for all + /// + /// May fail if the given header doesn't kick off an epoch or + /// the proof is invalid. + /// + /// Returns the epoch number and proof. + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, SimpleList), ::error::Error>; /// Checks if a given address is a validator, with the given function /// for executing synchronous calls to contracts. diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index b7a13c5e3..eecf88432 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -18,8 +18,8 @@ use std::collections::BTreeMap; use std::sync::Weak; -use engines::{Call, RequiresProof}; -use util::{H256, Address, RwLock}; +use engines::{Call, EpochChange}; +use util::{H256, Address, RwLock, U256}; use ids::BlockId; use header::{BlockNumber, Header}; use client::{Client, BlockChainClient}; @@ -43,9 +43,7 @@ impl Multi { fn correct_set(&self, id: BlockId) -> Option<&ValidatorSet> { match self.block_number.read()(id).map(|parent_block| self.correct_set_by_number(parent_block)) { - Ok(set) => { - Some(set) - }, + Ok((_, set)) => Some(set), Err(e) => { debug!(target: "engine", "ValidatorSet could not be recovered: {}", e); None @@ -53,7 +51,9 @@ impl Multi { } } - fn correct_set_by_number(&self, parent_block: BlockNumber) -> &ValidatorSet { + // get correct set by block number, along with block number at which + // this set was activated. + fn correct_set_by_number(&self, parent_block: BlockNumber) -> (BlockNumber, &ValidatorSet) { let (block, set) = self.sets.iter() .rev() .find(|&(block, _)| *block <= parent_block + 1) @@ -62,7 +62,7 @@ impl Multi { qed"); trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); - &**set + (*block, &**set) } } @@ -72,18 +72,22 @@ impl ValidatorSet for Multi { .unwrap_or(Box::new(|_, _| Err("No validator set for given ID.".into()))) } - fn proof_required(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> RequiresProof + fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> EpochChange { - self.correct_set_by_number(header.number()).proof_required(header, block, receipts) + self.correct_set_by_number(header.number()).1.is_epoch_end(header, block, receipts) } - fn generate_proof(&self, header: &Header, caller: &Call) -> Result, String> { - self.correct_set_by_number(header.number()).generate_proof(header, caller) + fn epoch_proof(&self, header: &Header, caller: &Call) -> Result, String> { + self.correct_set_by_number(header.number()).1.epoch_proof(header, caller) } - fn chain_verifier(&self, header: &Header, proof: Vec) -> Result { - self.correct_set_by_number(header.number()).chain_verifier(header, proof) + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, super::SimpleList), ::error::Error> { + // "multi" epoch is the inner set's epoch plus the transition block to that set. + // ensures epoch increases monotonically. + let (set_block, set) = self.correct_set_by_number(header.number()); + let (inner_epoch, list) = set.epoch_set(header, proof)?; + Ok((U256::from(set_block) + inner_epoch, list)) } fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 2af389484..3284ec5ab 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -126,16 +126,16 @@ impl ValidatorSet for ValidatorSafeContract { .and_then(|c| c.call_contract(id, addr, data))) } - fn proof_required(&self, header: &Header, _block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) - -> ::engines::RequiresProof + fn is_epoch_end(&self, header: &Header, _block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) + -> ::engines::EpochChange { let bloom = self.expected_bloom(header); let header_bloom = header.log_bloom(); - if &bloom & header_bloom != bloom { return ::engines::RequiresProof::No } + if &bloom & header_bloom != bloom { return ::engines::EpochChange::No } match receipts { - None => ::engines::RequiresProof::Unsure(::engines::Unsure::NeedsReceipts), + None => ::engines::EpochChange::Unsure(::engines::Unsure::NeedsReceipts), Some(receipts) => { let check_log = |log: &LogEntry| { log.address == self.address && @@ -149,7 +149,11 @@ impl ValidatorSet for ValidatorSafeContract { .event("ValidatorsChanged".into()) .expect("Contract known ahead of time to have `ValidatorsChanged` event; qed"); + // iterate in reverse because only the _last_ change in a given + // block actually has any effect. + // the contract should only increment the nonce once. let mut decoded_events = receipts.iter() + .rev() .filter(|r| &bloom & &r.log_bloom == bloom) .flat_map(|r| r.logs.iter()) .filter(move |l| check_log(l)) @@ -164,7 +168,7 @@ impl ValidatorSet for ValidatorSafeContract { // TODO: are multiple transitions per block possible? match decoded_events.next() { - None => ::engines::RequiresProof::No, + None => ::engines::EpochChange::No, Some(matched_event) => { // decode log manually until the native contract generator is // good enough to do it for us. @@ -181,10 +185,10 @@ impl ValidatorSet for ValidatorSafeContract { match (nonce, validators) { (Some(nonce), Some(validators)) => - ::engines::RequiresProof::Yes(Some(encode_proof(nonce, &validators))), + ::engines::EpochChange::Yes(encode_proof(nonce, &validators)), _ => { debug!(target: "engine", "Successfully decoded log turned out to be bad."); - ::engines::RequiresProof::No + ::engines::EpochChange::No } } } @@ -195,20 +199,21 @@ impl ValidatorSet for ValidatorSafeContract { // the proof we generate is an RLP list containing two parts. // (nonce, validators) - fn generate_proof(&self, _header: &Header, caller: &Call) -> Result, String> { + fn epoch_proof(&self, _header: &Header, caller: &Call) -> Result, String> { match (self.get_nonce(caller), self.get_list(caller)) { (Some(nonce), Some(list)) => Ok(encode_proof(nonce, &list.into_inner())), _ => Err("Caller insufficient to generate validator proof.".into()), } } - fn chain_verifier(&self, _header: &Header, proof: Vec) -> Result { + fn epoch_set(&self, _header: &Header, proof: &[u8]) -> Result<(U256, SimpleList), ::error::Error> { use rlp::UntrustedRlp; - let rlp = UntrustedRlp::new(&proof); + let rlp = UntrustedRlp::new(proof); + let nonce: U256 = rlp.val_at(0)?; let validators: Vec
= rlp.list_at(1)?; - Ok(SimpleList::new(validators)) + Ok((nonce, SimpleList::new(validators))) } fn contains_with_caller(&self, block_hash: &H256, address: &Address, caller: &Call) -> bool { diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index 339585d7d..59209fac0 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -16,7 +16,7 @@ /// Preconfigured validator list. -use util::{H256, Address, HeapSizeOf}; +use util::{H256, Address, HeapSizeOf, U256}; use engines::Call; use header::Header; @@ -53,18 +53,18 @@ impl ValidatorSet for SimpleList { Box::new(|_, _| Err("Simple list doesn't require calls.".into())) } - fn proof_required(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[::receipt::Receipt]>) - -> ::engines::RequiresProof + fn is_epoch_end(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[::receipt::Receipt]>) + -> ::engines::EpochChange { - ::engines::RequiresProof::No + ::engines::EpochChange::No } - fn generate_proof(&self, _header: &Header, _caller: &Call) -> Result, String> { + fn epoch_proof(&self, _header: &Header, _caller: &Call) -> Result, String> { Ok(Vec::new()) } - fn chain_verifier(&self, _header: &Header, _: Vec) -> Result { - Ok(self.clone()) + fn epoch_set(&self, _header: &Header, _: &[u8]) -> Result<(U256, SimpleList), ::error::Error> { + Ok((0.into(), self.clone())) } fn contains_with_caller(&self, _bh: &H256, address: &Address, _: &Call) -> bool { diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 585fd29aa..867d4f775 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -149,7 +149,16 @@ impl Ethash { } } -impl ::engines::ChainVerifier for Arc { +// TODO [rphmeier] +// +// for now, this is different than Ethash's own epochs, and signal +// "consensus epochs". +// in this sense, `Ethash` is epochless: the same `EpochVerifier` can be used +// for any block in the chain. +// in the future, we might move the Ethash epoch +// caching onto this mechanism as well. +impl ::engines::EpochVerifier for Arc { + fn epoch_number(&self) -> U256 { 0.into() } fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } fn verify_heavy(&self, header: &Header) -> Result<(), Error> { self.verify_block_unordered(header, None) @@ -393,7 +402,7 @@ impl Engine for Arc { Ok(()) } - fn chain_verifier(&self, _header: &Header, _proof: Vec) -> Result, Error> { + fn epoch_verifier(&self, _header: &Header, _proof: &[u8]) -> Result, Error> { Ok(Box::new(self.clone())) } } From 6da6c755a5c862e3110552c2c22206161ff271a0 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Tue, 18 Apr 2017 15:45:15 +0200 Subject: [PATCH 12/22] iterate DB by prefix --- util/src/kvdb.rs | 73 +++++++++++++++++++++++++++++---------- util/src/migration/mod.rs | 7 +++- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 5d4c02152..056b49c65 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -167,6 +167,10 @@ pub trait KeyValueDB: Sync + Send { /// Iterate over flushed data for a given column. fn iter<'a>(&'a self, col: Option) -> Box, Box<[u8]>)> + 'a>; + /// Iterate over flushed data for a given column, starting from a given prefix. + fn iter_from_prefix<'a>(&'a self, col: Option, prefix: &'a [u8]) + -> Box, Box<[u8]>)> + 'a>; + /// Attempt to replace this database with a new one located at the given path. fn restore(&self, new_db: &str) -> Result<(), UtilError>; } @@ -247,7 +251,21 @@ impl KeyValueDB for InMemory { .into_iter() .map(|(k, v)| (k.into_boxed_slice(), v.to_vec().into_boxed_slice())) ), - None => Box::new(None.into_iter()) + None => Box::new(None.into_iter()), + } + } + + fn iter_from_prefix<'a>(&'a self, col: Option, prefix: &'a [u8]) + -> Box, Box<[u8]>)> + 'a> + { + match self.columns.read().get(&col) { + Some(map) => Box::new( + map.clone() + .into_iter() + .skip_while(move |&(ref k, _)| !k.starts_with(prefix)) + .map(|(k, v)| (k.into_boxed_slice(), v.to_vec().into_boxed_slice())) + ), + None => Box::new(None.into_iter()), } } @@ -691,23 +709,17 @@ impl Database { /// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values. // TODO: support prefix seek for unflushed data pub fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option> { - match *self.db.read() { - Some(DBAndColumns { ref db, ref cfs }) => { - let mut iter = col.map_or_else(|| db.iterator_opt(IteratorMode::From(prefix, Direction::Forward), &self.read_opts), - |c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward), &self.read_opts) - .expect("iterator params are valid; qed")); - match iter.next() { - // TODO: use prefix_same_as_start read option (not availabele in C API currently) - Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, - _ => None - } - }, - None => None, - } + self.iter_from_prefix(col, prefix).and_then(|mut iter| { + match iter.next() { + // TODO: use prefix_same_as_start read option (not availabele in C API currently) + Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, + _ => None + } + }) } /// Get database iterator for flushed data. - pub fn iter(&self, col: Option) -> DatabaseIterator { + pub fn iter(&self, col: Option) -> Option { //TODO: iterate over overlay match *self.db.read() { Some(DBAndColumns { ref db, ref cfs }) => { @@ -717,12 +729,28 @@ impl Database { .expect("iterator params are valid; qed") ); - DatabaseIterator { + Some(DatabaseIterator { iter: iter, _marker: PhantomData, - } + }) }, - None => panic!("Not supported yet") //TODO: return an empty iterator or change return type + None => None, + } + } + + fn iter_from_prefix(&self, col: Option, prefix: &[u8]) -> Option { + match *self.db.read() { + Some(DBAndColumns { ref db, ref cfs }) => { + let iter = col.map_or_else(|| db.iterator_opt(IteratorMode::From(prefix, Direction::Forward), &self.read_opts), + |c| db.iterator_cf_opt(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward), &self.read_opts) + .expect("iterator params are valid; qed")); + + Some(DatabaseIterator { + iter: iter, + _marker: PhantomData, + }) + }, + None => None, } } @@ -836,7 +864,14 @@ impl KeyValueDB for Database { fn iter<'a>(&'a self, col: Option) -> Box, Box<[u8]>)> + 'a> { let unboxed = Database::iter(self, col); - Box::new(unboxed) + Box::new(unboxed.into_iter().flat_map(|inner| inner)) + } + + fn iter_from_prefix<'a>(&'a self, col: Option, prefix: &'a [u8]) + -> Box, Box<[u8]>)> + 'a> + { + let unboxed = Database::iter_from_prefix(self, col, prefix); + Box::new(unboxed.into_iter().flat_map(|inner| inner)) } fn restore(&self, new_db: &str) -> Result<(), UtilError> { diff --git a/util/src/migration/mod.rs b/util/src/migration/mod.rs index 6ec465517..93a98f1dc 100644 --- a/util/src/migration/mod.rs +++ b/util/src/migration/mod.rs @@ -157,7 +157,12 @@ impl Migration for T { fn migrate(&mut self, source: Arc, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { let mut batch = Batch::new(config, col); - for (key, value) in source.iter(col) { + let iter = match source.iter(col) { + Some(iter) => iter, + None => return Ok(()), + }; + + for (key, value) in iter { if let Some((key, value)) = self.simple_migrate(key.to_vec(), value.to_vec()) { batch.insert(key, value, dest)?; } From a278dd5a0a45510c3f593e3c8a1f8e36f58adde1 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 19 Apr 2017 14:58:19 +0200 Subject: [PATCH 13/22] store epoch transition proofs in DB --- Cargo.lock | 10 +- ethcore/light/src/provider.rs | 2 +- ethcore/src/blockchain/blockchain.rs | 18 ++++ ethcore/src/blockchain/extras.rs | 92 +++++++++++++++++- ethcore/src/blockchain/mod.rs | 1 + ethcore/src/client/client.rs | 96 ++++++++++++++----- ethcore/src/client/test_client.rs | 2 +- ethcore/src/client/traits.rs | 4 +- ethcore/src/engines/authority_round.rs | 4 +- ethcore/src/engines/basic_authority.rs | 4 +- ethcore/src/engines/epoch_verifier.rs | 5 +- ethcore/src/engines/mod.rs | 6 +- ethcore/src/engines/validator_set/contract.rs | 2 +- ethcore/src/engines/validator_set/mod.rs | 4 +- ethcore/src/engines/validator_set/multi.rs | 13 ++- .../engines/validator_set/safe_contract.rs | 13 +-- .../src/engines/validator_set/simple_list.rs | 6 +- ethcore/src/ethereum/ethash.rs | 2 +- ethcore/src/migrations/state/v7.rs | 2 +- ethcore/src/migrations/v10.rs | 2 +- ethcore/src/migrations/v9.rs | 2 +- ethcore/src/tests/client.rs | 2 +- util/Cargo.toml | 2 +- util/rlp/Cargo.toml | 2 +- util/src/journaldb/mod.rs | 4 +- util/src/kvdb.rs | 3 + 26 files changed, 234 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 830e1e75a..20887561b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,8 +339,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "elastic-array" -version = "0.6.0" -source = "git+https://github.com/paritytech/elastic-array#346f1ba5982576dab9d0b8fa178b50e1db0a21cd" +version = "0.7.0" +source = "git+https://github.com/paritytech/elastic-array?branch=0.7.0#970a11eca8a6b3591b476155c5896b4757b5c7b8" dependencies = [ "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -713,7 +713,7 @@ version = "1.7.0" dependencies = [ "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "clippy 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.6.0 (git+https://github.com/paritytech/elastic-array)", + "elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)", "env_logger 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "eth-secp256k1 0.5.6 (git+https://github.com/paritytech/rust-secp256k1)", "ethcore-bigint 0.1.2", @@ -2018,7 +2018,7 @@ name = "rlp" version = "0.1.0" dependencies = [ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.6.0 (git+https://github.com/paritytech/elastic-array)", + "elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)", "ethcore-bigint 0.1.2", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2790,7 +2790,7 @@ dependencies = [ "checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8" "checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f" "checksum either 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2b503c86dad62aaf414ecf2b8c527439abedb3f8d812537f0b12bfd6f32a91" -"checksum elastic-array 0.6.0 (git+https://github.com/paritytech/elastic-array)" = "" +"checksum elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)" = "" "checksum env_logger 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e3856f1697098606fc6cb97a93de88ca3f3bc35bb878c725920e6e82ecf05e83" "checksum eth-secp256k1 0.5.6 (git+https://github.com/paritytech/rust-secp256k1)" = "" "checksum ethabi 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "63df67d0af5e3cb906b667ca1a6e00baffbed87d0d8f5f78468a1f5eb3a66345" diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index e74ab1c70..3632783ca 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -258,7 +258,7 @@ impl Provider for T { }.fake_sign(req.from); self.prove_transaction(transaction, id) - .map(|proof| ::request::ExecutionResponse { items: proof }) + .map(|(_, proof)| ::request::ExecutionResponse { items: proof }) } fn ready_transactions(&self) -> Vec { diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 8986dc6b8..00420e58f 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -797,6 +797,24 @@ impl BlockChain { } } + /// Insert an epoch transition. Provide an epoch number being transitioned to + /// and epoch transition object. + /// + /// The block the transition occurred at should have already been inserted into the chain. + pub fn insert_epoch_transition(&self, batch: &mut DBTransaction, epoch_num: u64, transition: EpochTransition) { + let mut transitions = match self.db.read(db::COL_EXTRA, &epoch_num) { + Some(existing) => existing, + None => EpochTransitions { + number: epoch_num, + candidates: Vec::with_capacity(1), + } + }; + + transitions.candidates.push(transition); + + batch.write(db::COL_EXTRA, &epoch_num, &transitions); + } + /// Add a child to a given block. Assumes that the block hash is in /// the chain and the child's parent is this block. /// diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 0c8616a45..b9fd5d3bd 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -18,6 +18,7 @@ use bloomchain; use util::*; +use util::kvdb::PREFIX_LEN as DB_PREFIX_LEN; use rlp::*; use header::BlockNumber; use receipt::Receipt; @@ -37,6 +38,8 @@ pub enum ExtrasIndex { BlocksBlooms = 3, /// Block receipts index BlockReceipts = 4, + /// Epoch transition data index. + EpochTransitions = 5, } fn with_index(hash: &H256, i: ExtrasIndex) -> H264 { @@ -134,6 +137,39 @@ impl Key for H256 { } } +/// length of epoch keys. +pub const EPOCH_KEY_LEN: usize = DB_PREFIX_LEN + 16; + +/// epoch key prefix. +/// used to iterate over all epoch transitions in order from genesis. +pub fn epoch_key_prefix() -> [u8; DB_PREFIX_LEN] { + let mut arr = [0u8; DB_PREFIX_LEN]; + arr[0] = ExtrasIndex::EpochTransitions as u8; + + arr +} + +pub struct EpochTransitionsKey([u8; EPOCH_KEY_LEN]); +impl Deref for EpochTransitionsKey { + type Target = [u8]; + + fn deref(&self) -> &[u8] { &self.0[..] } +} + +impl Key for u64 { + type Target = EpochTransitionsKey; + + fn key(&self) -> Self::Target { + let mut arr = [0u8; EPOCH_KEY_LEN]; + arr[..DB_PREFIX_LEN].copy_from_slice(&epoch_key_prefix()[..]); + + write!(&mut arr[DB_PREFIX_LEN..], "{:016x}", self) + .expect("format arg is valid; no more than 16 chars will be written; qed"); + + EpochTransitionsKey(arr) + } +} + /// Familial details concerning a block #[derive(Debug, Clone)] pub struct BlockDetails { @@ -144,7 +180,7 @@ pub struct BlockDetails { /// Parent block hash pub parent: H256, /// List of children block hashes - pub children: Vec + pub children: Vec, } impl HeapSizeOf for BlockDetails { @@ -241,6 +277,60 @@ impl HeapSizeOf for BlockReceipts { } } +/// Candidate transitions to an epoch with specific number. +#[derive(Clone)] +pub struct EpochTransitions { + pub number: u64, + pub candidates: Vec, +} + +impl Encodable for EpochTransitions { + fn rlp_append(&self, s: &mut RlpStream) { + s.begin_list(2).append(&self.number).append_list(&self.candidates); + } +} + +impl Decodable for EpochTransitions { + fn decode(rlp: &UntrustedRlp) -> Result { + Ok(EpochTransitions { + number: rlp.val_at(0)?, + candidates: rlp.list_at(1)?, + }) + } +} + +#[derive(Debug, Clone)] +pub struct EpochTransition { + pub block_hash: H256, // block hash at which the transition occurred. + pub proof: Vec, // "transition/epoch" proof from the engine. + pub state_proof: Vec, // state items necessary to regenerate proof. +} + +impl Encodable for EpochTransition { + fn rlp_append(&self, s: &mut RlpStream) { + s.begin_list(3) + .append(&self.block_hash) + .append(&self.proof) + .begin_list(self.state_proof.len()); + + for item in &self.state_proof { + s.append(&&**item); + } + } +} + +impl Decodable for EpochTransition { + fn decode(rlp: &UntrustedRlp) -> Result { + Ok(EpochTransition { + block_hash: rlp.val_at(0)?, + proof: rlp.val_at(1)?, + state_proof: rlp.at(2)?.iter().map(|x| { + Ok(DBValue::from_slice(x.data()?)) + }).collect::, _>>()?, + }) + } +} + #[cfg(test)] mod tests { use rlp::*; diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs index 55a67e2b6..2d6d6ddb3 100644 --- a/ethcore/src/blockchain/mod.rs +++ b/ethcore/src/blockchain/mod.rs @@ -31,5 +31,6 @@ pub mod generator; pub use self::blockchain::{BlockProvider, BlockChain}; pub use self::cache::CacheSize; pub use self::config::Config; +pub use self::extras::EpochTransition; pub use types::tree_route::TreeRoute; pub use self::import_route::ImportRoute; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index a91294e27..ab9a07691 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -32,13 +32,13 @@ use util::kvdb::*; // other use basic_types::Seal; use block::*; -use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute}; +use blockchain::{BlockChain, BlockProvider, EpochTransition, TreeRoute, ImportRoute}; use blockchain::extras::TransactionAddress; use client::Error as ClientError; use client::{ BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient, MiningBlockChainClient, EngineClient, TraceFilter, CallAnalytics, BlockImportError, Mode, - ChainNotify, PruningInfo, + ChainNotify, PruningInfo, ProvingBlockChainClient, }; use encoded; use engines::Engine; @@ -578,15 +578,15 @@ impl Client { // generate validation proof if the engine requires them. // TODO: make conditional? - let generate_proof = { + let entering_new_epoch = { use engines::EpochChange; match self.engine.is_epoch_end(block.header(), Some(block_data), Some(&receipts)) { - EpochChange::Yes(_) => true, - EpochChange::No => false, + EpochChange::Yes(e, p) => Some((block.header().clone(), e, p)), + EpochChange::No => None, EpochChange::Unsure(_) => { warn!(target: "client", "Detected invalid engine implementation."); warn!(target: "client", "Engine claims to require more block data, but everything provided."); - false + None } } }; @@ -596,14 +596,53 @@ impl Client { // TODO: Prove it with a test. let mut state = block.drain(); - if generate_proof { - debug!(target: "client", "Generating validation proof for block {}", hash); - - // TODO - } - state.journal_under(&mut batch, number, hash).expect("DB commit failed"); let route = chain.insert_block(&mut batch, block_data, receipts); + + if let Some((header, epoch, expected)) = entering_new_epoch { + use std::cell::RefCell; + use std::collections::BTreeSet; + + debug!(target: "client", "Generating validation proof for block {}", hash); + + // proof is two-part. state items read in lexicographical order, + // and the secondary "proof" part. + let read_values = RefCell::new(BTreeSet::new()); + let block_id = BlockId::Hash(hash.clone()); + let proof = { + let call = |a, d| { + let tx = self.contract_call_tx(block_id, a, d); + let (result, items) = self.prove_transaction(tx, block_id) + .ok_or_else(|| format!("Unable to make call to generate epoch proof."))?; + + read_values.borrow_mut().extend(items); + Ok(result) + }; + + self.engine.epoch_proof(&header, &call) + }; + + match proof { + Ok(proof) => { + if proof != expected { + warn!(target: "client", "Extracted epoch change proof different than expected."); + warn!(target: "client", "Using a custom engine implementation?"); + } + + // insert into database, using the generated proof. + chain.insert_epoch_transition(&mut batch, epoch, EpochTransition { + block_hash: hash.clone(), + proof: proof, + state_proof: read_values.into_inner().into_iter().collect(), + }); + } + Err(e) => { + warn!(target: "client", "Error generating epoch change proof for block {}: {}", hash, e); + warn!(target: "client", "Snapshots generated by this node will be incomplete."); + } + } + } + self.tracedb.read().import(&mut batch, TraceImportRequest { traces: traces.into(), block_hash: hash.clone(), @@ -893,6 +932,20 @@ impl Client { } } } + + // transaction for calling contracts from services like engine. + // from the null sender, with 50M gas. + fn contract_call_tx(&self, block_id: BlockId, address: Address, data: Bytes) -> SignedTransaction { + let from = Address::default(); + Transaction { + nonce: self.nonce(&from, block_id).unwrap_or_else(|| self.engine.account_start_nonce()), + action: Action::Call(address), + gas: U256::from(50_000_000), + gas_price: U256::default(), + value: U256::default(), + data: data, + }.fake_sign(from) + } } impl snapshot::DatabaseRestore for Client { @@ -1483,15 +1536,7 @@ impl BlockChainClient for Client { } fn call_contract(&self, block_id: BlockId, address: Address, data: Bytes) -> Result { - let from = Address::default(); - let transaction = Transaction { - nonce: self.latest_nonce(&from), - action: Action::Call(address), - gas: U256::from(50_000_000), - gas_price: U256::default(), - value: U256::default(), - data: data, - }.fake_sign(from); + let transaction = self.contract_call_tx(block_id, address, data); self.call(&transaction, block_id, Default::default()) .map_err(|e| format!("{:?}", e)) @@ -1643,7 +1688,7 @@ impl MayPanic for Client { } } -impl ::client::ProvingBlockChainClient for Client { +impl ProvingBlockChainClient for Client { fn prove_storage(&self, key1: H256, key2: H256, id: BlockId) -> Option<(Vec, H256)> { self.state_at(id) .and_then(move |state| state.prove_storage(key1, key2).ok()) @@ -1654,7 +1699,7 @@ impl ::client::ProvingBlockChainClient for Client { .and_then(move |state| state.prove_account(key1).ok()) } - fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option> { + fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option<(Bytes, Vec)> { let (state, mut env_info) = match (self.state_at(id), self.env_info(id)) { (Some(s), Some(e)) => (s, e), _ => return None, @@ -1669,8 +1714,9 @@ impl ::client::ProvingBlockChainClient for Client { let res = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm).transact(&transaction, options); match res { - Err(ExecutionError::Internal(_)) => return None, - _ => return Some(state.drop().1.extract_proof()), + Err(ExecutionError::Internal(_)) => None, + Err(_) => Some((Vec::new(), state.drop().1.extract_proof())), + Ok(res) => Some((res.output, state.drop().1.extract_proof())), } } } diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 16f38203f..43d75cecd 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -767,7 +767,7 @@ impl ProvingBlockChainClient for TestBlockChainClient { None } - fn prove_transaction(&self, _: SignedTransaction, _: BlockId) -> Option> { + fn prove_transaction(&self, _: SignedTransaction, _: BlockId) -> Option<(Bytes, Vec)> { None } } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index a612d8a77..e17c324aa 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -324,5 +324,7 @@ pub trait ProvingBlockChainClient: BlockChainClient { fn prove_account(&self, key1: H256, id: BlockId) -> Option<(Vec, BasicAccount)>; /// Prove execution of a transaction at the given block. - fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option>; + /// Returns the output of the call and a vector of database items necessary + /// to reproduce it. + fn prove_transaction(&self, transaction: SignedTransaction, id: BlockId) -> Option<(Bytes, Vec)>; } diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index 1f3edf57c..55640478e 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -133,13 +133,13 @@ pub struct AuthorityRound { // header-chain validator. struct EpochVerifier { - epoch_number: U256, + epoch_number: u64, step: Arc, subchain_validators: SimpleList, } impl super::EpochVerifier for EpochVerifier { - fn epoch_number(&self) -> U256 { self.epoch_number.clone() } + fn epoch_number(&self) -> u64 { self.epoch_number.clone() } fn verify_light(&self, header: &Header) -> Result<(), Error> { // always check the seal since it's fast. // nothing heavier to do. diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index 94c3fe578..35db2a486 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -52,12 +52,12 @@ impl From for BasicAuthorityParams { } struct EpochVerifier { - epoch_number: U256, + epoch_number: u64, list: SimpleList, } impl super::EpochVerifier for EpochVerifier { - fn epoch_number(&self) -> U256 { self.epoch_number.clone() } + fn epoch_number(&self) -> u64 { self.epoch_number.clone() } fn verify_light(&self, header: &Header) -> Result<(), Error> { verify_external(header, &self.list) } diff --git a/ethcore/src/engines/epoch_verifier.rs b/ethcore/src/engines/epoch_verifier.rs index fd0847e76..62740fbd4 100644 --- a/ethcore/src/engines/epoch_verifier.rs +++ b/ethcore/src/engines/epoch_verifier.rs @@ -18,14 +18,13 @@ use error::Error; use header::Header; -use util::U256; /// Verifier for all blocks within an epoch without accessing /// /// See docs on `Engine` relating to proving functions for more details. pub trait EpochVerifier: Sync { /// Get the epoch number. - fn epoch_number(&self) -> U256; + fn epoch_number(&self) -> u64; /// Lightly verify the next block header. /// This may not be a header belonging to a different epoch. @@ -41,6 +40,6 @@ pub trait EpochVerifier: Sync { pub struct NoOp; impl EpochVerifier for NoOp { - fn epoch_number(&self) -> U256 { 0.into() } + fn epoch_number(&self) -> u64 { 0 } fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index b67afdf2f..badac3cf2 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -96,7 +96,7 @@ pub enum Seal { } /// Type alias for a function we can make calls through synchronously. -pub type Call = Fn(Address, Bytes) -> Result; +pub type Call<'a> = Fn(Address, Bytes) -> Result + 'a; /// Results of a query of whether an epoch change occurred at the given block. #[derive(Debug, Clone, PartialEq)] @@ -105,8 +105,8 @@ pub enum EpochChange { Unsure(Unsure), /// No epoch change. No, - /// Validation proof required, and the expected output - Yes(Bytes), + /// Validation proof required, and the new epoch number and expected proof. + Yes(u64, Bytes), } /// More data required to determine if an epoch change occurred at a given block. diff --git a/ethcore/src/engines/validator_set/contract.rs b/ethcore/src/engines/validator_set/contract.rs index 0fdef84a3..22506c348 100644 --- a/ethcore/src/engines/validator_set/contract.rs +++ b/ethcore/src/engines/validator_set/contract.rs @@ -76,7 +76,7 @@ impl ValidatorSet for ValidatorContract { self.validators.epoch_proof(header, caller) } - fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, super::SimpleList), ::error::Error> { + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(u64, super::SimpleList), ::error::Error> { self.validators.epoch_set(header, proof) } diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index e9dc333c5..a7e0f196f 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -23,7 +23,7 @@ mod multi; use std::sync::Weak; use ids::BlockId; -use util::{Address, H256, U256}; +use util::{Address, H256}; use ethjson::spec::ValidatorSet as ValidatorSpec; use client::Client; use header::Header; @@ -97,7 +97,7 @@ pub trait ValidatorSet: Send + Sync { /// the proof is invalid. /// /// Returns the epoch number and proof. - fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, SimpleList), ::error::Error>; + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(u64, SimpleList), ::error::Error>; /// Checks if a given address is a validator, with the given function /// for executing synchronous calls to contracts. diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs index eecf88432..abb034698 100644 --- a/ethcore/src/engines/validator_set/multi.rs +++ b/ethcore/src/engines/validator_set/multi.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::sync::Weak; use engines::{Call, EpochChange}; -use util::{H256, Address, RwLock, U256}; +use util::{H256, Address, RwLock}; use ids::BlockId; use header::{BlockNumber, Header}; use client::{Client, BlockChainClient}; @@ -75,19 +75,24 @@ impl ValidatorSet for Multi { fn is_epoch_end(&self, header: &Header, block: Option<&[u8]>, receipts: Option<&[::receipt::Receipt]>) -> EpochChange { - self.correct_set_by_number(header.number()).1.is_epoch_end(header, block, receipts) + let (set_block, set) = self.correct_set_by_number(header.number()); + + match set.is_epoch_end(header, block, receipts) { + EpochChange::Yes(num, proof) => EpochChange::Yes(set_block + num, proof), + other => other, + } } fn epoch_proof(&self, header: &Header, caller: &Call) -> Result, String> { self.correct_set_by_number(header.number()).1.epoch_proof(header, caller) } - fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(U256, super::SimpleList), ::error::Error> { + fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(u64, super::SimpleList), ::error::Error> { // "multi" epoch is the inner set's epoch plus the transition block to that set. // ensures epoch increases monotonically. let (set_block, set) = self.correct_set_by_number(header.number()); let (inner_epoch, list) = set.epoch_set(header, proof)?; - Ok((U256::from(set_block) + inner_epoch, list)) + Ok((set_block + inner_epoch, list)) } fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool { diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 3284ec5ab..cbfaa9883 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -165,8 +165,6 @@ impl ValidatorSet for ValidatorSafeContract { } }); - - // TODO: are multiple transitions per block possible? match decoded_events.next() { None => ::engines::EpochChange::No, Some(matched_event) => { @@ -184,8 +182,11 @@ impl ValidatorSet for ValidatorSafeContract { ); match (nonce, validators) { - (Some(nonce), Some(validators)) => - ::engines::EpochChange::Yes(encode_proof(nonce, &validators)), + (Some(nonce), Some(validators)) => { + let proof = encode_proof(nonce, &validators); + let new_epoch = nonce.low_u64(); + ::engines::EpochChange::Yes(new_epoch, proof) + } _ => { debug!(target: "engine", "Successfully decoded log turned out to be bad."); ::engines::EpochChange::No @@ -206,11 +207,11 @@ impl ValidatorSet for ValidatorSafeContract { } } - fn epoch_set(&self, _header: &Header, proof: &[u8]) -> Result<(U256, SimpleList), ::error::Error> { + fn epoch_set(&self, _header: &Header, proof: &[u8]) -> Result<(u64, SimpleList), ::error::Error> { use rlp::UntrustedRlp; let rlp = UntrustedRlp::new(proof); - let nonce: U256 = rlp.val_at(0)?; + let nonce: u64 = rlp.val_at(0)?; let validators: Vec
= rlp.list_at(1)?; Ok((nonce, SimpleList::new(validators))) diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index 59209fac0..15cf141f6 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -16,7 +16,7 @@ /// Preconfigured validator list. -use util::{H256, Address, HeapSizeOf, U256}; +use util::{H256, Address, HeapSizeOf}; use engines::Call; use header::Header; @@ -63,8 +63,8 @@ impl ValidatorSet for SimpleList { Ok(Vec::new()) } - fn epoch_set(&self, _header: &Header, _: &[u8]) -> Result<(U256, SimpleList), ::error::Error> { - Ok((0.into(), self.clone())) + fn epoch_set(&self, _header: &Header, _: &[u8]) -> Result<(u64, SimpleList), ::error::Error> { + Ok((0, self.clone())) } fn contains_with_caller(&self, _bh: &H256, address: &Address, _: &Call) -> bool { diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 867d4f775..548d0dbaa 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -158,7 +158,7 @@ impl Ethash { // in the future, we might move the Ethash epoch // caching onto this mechanism as well. impl ::engines::EpochVerifier for Arc { - fn epoch_number(&self) -> U256 { 0.into() } + fn epoch_number(&self) -> u64 { 0 } fn verify_light(&self, _header: &Header) -> Result<(), Error> { Ok(()) } fn verify_heavy(&self, header: &Header) -> Result<(), Error> { self.verify_block_unordered(header, None) diff --git a/ethcore/src/migrations/state/v7.rs b/ethcore/src/migrations/state/v7.rs index e9da80c31..174945c9b 100644 --- a/ethcore/src/migrations/state/v7.rs +++ b/ethcore/src/migrations/state/v7.rs @@ -238,7 +238,7 @@ impl Migration for OverlayRecentV7 { } let mut count = 0; - for (key, value) in source.iter(None) { + for (key, value) in source.iter(None).into_iter().flat_map(|inner| inner) { count += 1; if count == 100_000 { count = 0; diff --git a/ethcore/src/migrations/v10.rs b/ethcore/src/migrations/v10.rs index 45fc457bd..15abf4790 100644 --- a/ethcore/src/migrations/v10.rs +++ b/ethcore/src/migrations/v10.rs @@ -102,7 +102,7 @@ impl Migration for ToV10 { fn migrate(&mut self, source: Arc, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { let mut batch = Batch::new(config, col); - for (key, value) in source.iter(col) { + for (key, value) in source.iter(col).into_iter().flat_map(|inner| inner) { self.progress.tick(); batch.insert(key.to_vec(), value.to_vec(), dest)?; } diff --git a/ethcore/src/migrations/v9.rs b/ethcore/src/migrations/v9.rs index 9a6eb5088..4ca6e3a82 100644 --- a/ethcore/src/migrations/v9.rs +++ b/ethcore/src/migrations/v9.rs @@ -59,7 +59,7 @@ impl Migration for ToV9 { fn migrate(&mut self, source: Arc, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { let mut batch = Batch::new(config, self.column); - for (key, value) in source.iter(col) { + for (key, value) in source.iter(col).into_iter().flat_map(|inner| inner) { self.progress.tick(); match self.extract { Extract::Header => { diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index e61edd478..7d630fbee 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -367,7 +367,7 @@ fn transaction_proof() { data: Vec::new(), }.fake_sign(address); - let proof = client.prove_transaction(transaction.clone(), BlockId::Latest).unwrap(); + let proof = client.prove_transaction(transaction.clone(), BlockId::Latest).unwrap().1; let backend = state::backend::ProofCheck::new(&proof); let mut factories = ::factory::Factories::default(); diff --git a/util/Cargo.toml b/util/Cargo.toml index d4e2dcfb5..8b902467f 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -16,7 +16,7 @@ time = "0.1.34" rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" } eth-secp256k1 = { git = "https://github.com/paritytech/rust-secp256k1" } rust-crypto = "0.2.34" -elastic-array = { git = "https://github.com/paritytech/elastic-array" } +elastic-array = "0.7.0" rlp = { path = "rlp" } heapsize = { version = "0.3", features = ["unstable"] } itertools = "0.5" diff --git a/util/rlp/Cargo.toml b/util/rlp/Cargo.toml index 17aed5dad..643e9dc99 100644 --- a/util/rlp/Cargo.toml +++ b/util/rlp/Cargo.toml @@ -6,7 +6,7 @@ version = "0.1.0" authors = ["Parity Technologies "] [dependencies] -elastic-array = { git = "https://github.com/paritytech/elastic-array" } +elastic-array = "0.7.0" ethcore-bigint = { path = "../bigint" } lazy_static = "0.2" rustc-serialize = "0.3" diff --git a/util/src/journaldb/mod.rs b/util/src/journaldb/mod.rs index e949b269f..84a71339a 100644 --- a/util/src/journaldb/mod.rs +++ b/util/src/journaldb/mod.rs @@ -125,8 +125,8 @@ pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option Date: Wed, 19 Apr 2017 15:35:12 +0200 Subject: [PATCH 14/22] ensure genesis validator set in DB --- Cargo.lock | 8 +-- ethcore/src/blockchain/blockchain.rs | 8 ++- ethcore/src/client/client.rs | 100 +++++++++++++++------------ 3 files changed, 65 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20887561b..3298887d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -340,7 +340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "elastic-array" version = "0.7.0" -source = "git+https://github.com/paritytech/elastic-array?branch=0.7.0#970a11eca8a6b3591b476155c5896b4757b5c7b8" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -713,7 +713,7 @@ version = "1.7.0" dependencies = [ "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "clippy 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)", + "elastic-array 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "eth-secp256k1 0.5.6 (git+https://github.com/paritytech/rust-secp256k1)", "ethcore-bigint 0.1.2", @@ -2018,7 +2018,7 @@ name = "rlp" version = "0.1.0" dependencies = [ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)", + "elastic-array 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bigint 0.1.2", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2790,7 +2790,7 @@ dependencies = [ "checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8" "checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f" "checksum either 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2b503c86dad62aaf414ecf2b8c527439abedb3f8d812537f0b12bfd6f32a91" -"checksum elastic-array 0.7.0 (git+https://github.com/paritytech/elastic-array?branch=0.7.0)" = "" +"checksum elastic-array 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71a64decd4b8cd06654a4e643c45cb558ad554abbffd82a7e16e34f45f51b605" "checksum env_logger 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e3856f1697098606fc6cb97a93de88ca3f3bc35bb878c725920e6e82ecf05e83" "checksum eth-secp256k1 0.5.6 (git+https://github.com/paritytech/rust-secp256k1)" = "" "checksum ethabi 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "63df67d0af5e3cb906b667ca1a6e00baffbed87d0d8f5f78468a1f5eb3a66345" diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 00420e58f..79169fd82 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -810,9 +810,11 @@ impl BlockChain { } }; - transitions.candidates.push(transition); - - batch.write(db::COL_EXTRA, &epoch_num, &transitions); + // ensure we don't write any duplicates. + if transitions.candidates.iter().find(|c| c.block_hash == transition.block_hash).is_none() { + transitions.candidates.push(transition); + batch.write(db::COL_EXTRA, &epoch_num, &transitions); + } } /// Add a child to a given block. Assumes that the block hash is in diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index ab9a07691..26d823dce 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -49,7 +49,7 @@ use evm::{Factory as EvmFactory, Schedule}; use executive::{Executive, Executed, TransactOptions, contract_address}; use factory::Factories; use futures::{future, Future}; -use header::BlockNumber; +use header::{BlockNumber, Header}; use io::*; use log_entry::LocalizedLogEntry; use miner::{Miner, MinerService, TransactionImportResult}; @@ -247,17 +247,29 @@ impl Client { exit_handler: Mutex::new(None), }); + // prune old states. { let state_db = client.state_db.lock().boxed_clone(); let chain = client.chain.read(); client.prune_ancient(state_db, &chain)?; } + // ensure genesis epoch proof in the DB. + { + let mut batch = DBTransaction::new(); + let chain = client.chain.read(); + client.generate_epoch_proof(&mut batch, &spec.genesis_header(), 0, &*chain); + client.db.read().write_buffered(batch); + } + if let Some(reg_addr) = client.additional_params().get("registrar").and_then(|s| Address::from_str(s).ok()) { trace!(target: "client", "Found registrar at {}", reg_addr); let registrar = Registry::new(reg_addr); *client.registrar.lock() = Some(registrar); } + + // ensure buffered changes are flushed. + client.db.read().flush().map_err(ClientError::Database)?; Ok(client) } @@ -581,7 +593,7 @@ impl Client { let entering_new_epoch = { use engines::EpochChange; match self.engine.is_epoch_end(block.header(), Some(block_data), Some(&receipts)) { - EpochChange::Yes(e, p) => Some((block.header().clone(), e, p)), + EpochChange::Yes(e, _) => Some((block.header().clone(), e)), EpochChange::No => None, EpochChange::Unsure(_) => { warn!(target: "client", "Detected invalid engine implementation."); @@ -599,48 +611,8 @@ impl Client { state.journal_under(&mut batch, number, hash).expect("DB commit failed"); let route = chain.insert_block(&mut batch, block_data, receipts); - if let Some((header, epoch, expected)) = entering_new_epoch { - use std::cell::RefCell; - use std::collections::BTreeSet; - - debug!(target: "client", "Generating validation proof for block {}", hash); - - // proof is two-part. state items read in lexicographical order, - // and the secondary "proof" part. - let read_values = RefCell::new(BTreeSet::new()); - let block_id = BlockId::Hash(hash.clone()); - let proof = { - let call = |a, d| { - let tx = self.contract_call_tx(block_id, a, d); - let (result, items) = self.prove_transaction(tx, block_id) - .ok_or_else(|| format!("Unable to make call to generate epoch proof."))?; - - read_values.borrow_mut().extend(items); - Ok(result) - }; - - self.engine.epoch_proof(&header, &call) - }; - - match proof { - Ok(proof) => { - if proof != expected { - warn!(target: "client", "Extracted epoch change proof different than expected."); - warn!(target: "client", "Using a custom engine implementation?"); - } - - // insert into database, using the generated proof. - chain.insert_epoch_transition(&mut batch, epoch, EpochTransition { - block_hash: hash.clone(), - proof: proof, - state_proof: read_values.into_inner().into_iter().collect(), - }); - } - Err(e) => { - warn!(target: "client", "Error generating epoch change proof for block {}: {}", hash, e); - warn!(target: "client", "Snapshots generated by this node will be incomplete."); - } - } + if let Some((header, epoch)) = entering_new_epoch { + self.generate_epoch_proof(&mut batch, &header, epoch, &chain); } self.tracedb.read().import(&mut batch, TraceImportRequest { @@ -665,6 +637,46 @@ impl Client { route } + // generate an epoch transition proof at the given block, and write it into the given blockchain. + fn generate_epoch_proof(&self, batch: &mut DBTransaction, header: &Header, epoch_number: u64, chain: &BlockChain) { + use std::cell::RefCell; + use std::collections::BTreeSet; + + let hash = header.hash(); + debug!(target: "client", "Generating validation proof for block {}", hash); + + // proof is two-part. state items read in lexicographical order, + // and the secondary "proof" part. + let read_values = RefCell::new(BTreeSet::new()); + let block_id = BlockId::Hash(hash.clone()); + let proof = { + let call = |a, d| { + let tx = self.contract_call_tx(block_id, a, d); + let (result, items) = self.prove_transaction(tx, block_id) + .ok_or_else(|| format!("Unable to make call to generate epoch proof."))?; + + read_values.borrow_mut().extend(items); + Ok(result) + }; + + self.engine.epoch_proof(&header, &call) + }; + + // insert into database, using the generated proof. + match proof { + Ok(proof) => + chain.insert_epoch_transition(batch, epoch_number, EpochTransition { + block_hash: hash.clone(), + proof: proof, + state_proof: read_values.into_inner().into_iter().collect(), + }), + Err(e) => { + warn!(target: "client", "Error generating epoch change proof for block {}: {}", hash, e); + warn!(target: "client", "Snapshots generated by this node will be incomplete."); + } + } + } + // prune ancient states until below the memory limit or only the minimum amount remain. fn prune_ancient(&self, mut state_db: StateDB, chain: &BlockChain) -> Result<(), ClientError> { let number = match state_db.journal_db().latest_era() { From 4d3f137e1e12d3788040c51260d2dcf03d6715b3 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 19 Apr 2017 16:27:45 +0200 Subject: [PATCH 15/22] iterate over all epochs --- ethcore/src/blockchain/blockchain.rs | 102 +++++++++++++++++++++++++++ ethcore/src/blockchain/extras.rs | 20 +++--- ethcore/src/client/client.rs | 1 + 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 79169fd82..0824bd351 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -419,6 +419,45 @@ impl<'a> Iterator for AncestryIter<'a> { } } +/// An iterator which walks all epoch transitions. +/// Returns epoch transitions. +pub struct EpochTransitionIter<'a> { + chain: &'a BlockChain, + prefix_iter: Box, Box<[u8]>)> + 'a>, +} + +impl<'a> Iterator for EpochTransitionIter<'a> { + type Item = (u64, EpochTransition); + + fn next(&mut self) -> Option { + loop { + match self.prefix_iter.next() { + Some((key, val)) => { + // iterator may continue beyond values beginning with this + // prefix. + if !key.starts_with(&EPOCH_KEY_PREFIX[..]) { return None } + + let transitions: EpochTransitions = ::rlp::decode(&val[..]); + + // if there are multiple candidates, at most one will be on the + // canon chain. + for transition in transitions.candidates.into_iter() { + let is_in_canon_chain = self.chain.block_hash(transition.block_number) + .map_or(false, |hash| hash == transition.block_hash); + + if is_in_canon_chain { + return Some((transitions.number, transition)) + } + } + + // some epochs never occurred on the main chain. + } + None => return None, + } + } + } +} + impl BlockChain { /// Create new instance of blockchain from given Genesis. pub fn new(config: Config, genesis: &[u8], db: Arc) -> BlockChain { @@ -817,6 +856,15 @@ impl BlockChain { } } + /// Iterate over all epoch transitions. + pub fn epoch_transitions(&self) -> EpochTransitionIter { + let iter = self.db.iter_from_prefix(db::COL_EXTRA, &EPOCH_KEY_PREFIX[..]); + EpochTransitionIter { + chain: self, + prefix_iter: iter, + } + } + /// Add a child to a given block. Assumes that the block hash is in /// the chain and the child's parent is this block. /// @@ -2130,4 +2178,58 @@ mod tests { assert_eq!(bc.rewind(), Some(genesis_hash.clone())); assert_eq!(bc.rewind(), None); } + + #[test] + fn epoch_transitions_iter() { + use blockchain::extras::EpochTransition; + + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + + let db = new_db(); + { + let bc = new_chain(&genesis, db.clone()); + let uncle = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + + let mut batch = db.transaction(); + // create a longer fork + for i in 0..5 { + let canon_block = canon_chain.generate(&mut finalizer).unwrap(); + let hash = BlockView::new(&canon_block).header_view().sha3(); + + bc.insert_block(&mut batch, &canon_block, vec![]); + bc.insert_epoch_transition(&mut batch, i, EpochTransition { + block_hash: hash, + block_number: i + 1, + proof: vec![], + state_proof: vec![], + }); + bc.commit(); + } + + assert_eq!(bc.best_block_number(), 5); + + let hash = BlockView::new(&uncle).header_view().sha3(); + bc.insert_block(&mut batch, &uncle, vec![]); + bc.insert_epoch_transition(&mut batch, 999, EpochTransition { + block_hash: hash, + block_number: 1, + proof: vec![], + state_proof: vec![] + }); + + db.write(batch).unwrap(); + bc.commit(); + + // epoch 999 not in canonical chain. + assert_eq!(bc.epoch_transitions().map(|(i, _)| i).collect::>(), vec![0, 1, 2, 3, 4]); + } + + // re-loading the blockchain should load the correct best block. + let bc = new_chain(&genesis, db); + + assert_eq!(bc.best_block_number(), 5); + assert_eq!(bc.epoch_transitions().map(|(i, _)| i).collect::>(), vec![0, 1, 2, 3, 4]); + } } diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index b9fd5d3bd..7de49f3bf 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -142,12 +142,9 @@ pub const EPOCH_KEY_LEN: usize = DB_PREFIX_LEN + 16; /// epoch key prefix. /// used to iterate over all epoch transitions in order from genesis. -pub fn epoch_key_prefix() -> [u8; DB_PREFIX_LEN] { - let mut arr = [0u8; DB_PREFIX_LEN]; - arr[0] = ExtrasIndex::EpochTransitions as u8; - - arr -} +pub const EPOCH_KEY_PREFIX: &'static [u8; DB_PREFIX_LEN] = &[ + ExtrasIndex::EpochTransitions as u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; pub struct EpochTransitionsKey([u8; EPOCH_KEY_LEN]); impl Deref for EpochTransitionsKey { @@ -161,7 +158,7 @@ impl Key for u64 { fn key(&self) -> Self::Target { let mut arr = [0u8; EPOCH_KEY_LEN]; - arr[..DB_PREFIX_LEN].copy_from_slice(&epoch_key_prefix()[..]); + arr[..DB_PREFIX_LEN].copy_from_slice(&EPOCH_KEY_PREFIX[..]); write!(&mut arr[DB_PREFIX_LEN..], "{:016x}", self) .expect("format arg is valid; no more than 16 chars will be written; qed"); @@ -302,14 +299,16 @@ impl Decodable for EpochTransitions { #[derive(Debug, Clone)] pub struct EpochTransition { pub block_hash: H256, // block hash at which the transition occurred. + pub block_number: BlockNumber, // block number at which the tranition occurred. pub proof: Vec, // "transition/epoch" proof from the engine. pub state_proof: Vec, // state items necessary to regenerate proof. } impl Encodable for EpochTransition { fn rlp_append(&self, s: &mut RlpStream) { - s.begin_list(3) + s.begin_list(4) .append(&self.block_hash) + .append(&self.block_number) .append(&self.proof) .begin_list(self.state_proof.len()); @@ -323,8 +322,9 @@ impl Decodable for EpochTransition { fn decode(rlp: &UntrustedRlp) -> Result { Ok(EpochTransition { block_hash: rlp.val_at(0)?, - proof: rlp.val_at(1)?, - state_proof: rlp.at(2)?.iter().map(|x| { + block_number: rlp.val_at(1)?, + proof: rlp.val_at(2)?, + state_proof: rlp.at(3)?.iter().map(|x| { Ok(DBValue::from_slice(x.data()?)) }).collect::, _>>()?, }) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 26d823dce..88fd181bb 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -667,6 +667,7 @@ impl Client { Ok(proof) => chain.insert_epoch_transition(batch, epoch_number, EpochTransition { block_hash: hash.clone(), + block_number: header.number(), proof: proof, state_proof: read_values.into_inner().into_iter().collect(), }), From 2ec3397b7ddc1aa6690cc02dd2a7210204132283 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 19 Apr 2017 20:31:53 +0200 Subject: [PATCH 16/22] snapshot chunk and restore traits --- ethcore/src/client/client.rs | 2 +- ethcore/src/engines/mod.rs | 7 + ethcore/src/engines/null_engine.rs | 4 + ethcore/src/ethereum/ethash.rs | 4 + ethcore/src/snapshot/consensus/mod.rs | 349 ++++++++++++++++++ ethcore/src/snapshot/error.rs | 3 + ethcore/src/snapshot/mod.rs | 318 +++------------- ethcore/src/snapshot/service.rs | 36 +- .../src/snapshot/snapshot_service_trait.rs | 4 - ethcore/src/snapshot/tests/blocks.rs | 36 +- rpc/src/v1/tests/helpers/snapshot_service.rs | 3 +- sync/src/tests/snapshot.rs | 7 - 12 files changed, 446 insertions(+), 327 deletions(-) create mode 100644 ethcore/src/snapshot/consensus/mod.rs diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 88fd181bb..e63b97dfb 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -894,7 +894,7 @@ impl Client { }, }; - snapshot::take_snapshot(&self.chain.read(), start_hash, db.as_hashdb(), writer, p)?; + snapshot::take_snapshot(&*self.engine, &self.chain.read(), start_hash, db.as_hashdb(), writer, p)?; Ok(()) } diff --git a/ethcore/src/engines/mod.rs b/ethcore/src/engines/mod.rs index badac3cf2..ad573637c 100644 --- a/ethcore/src/engines/mod.rs +++ b/ethcore/src/engines/mod.rs @@ -45,6 +45,7 @@ use error::{Error, TransactionError}; use evm::Schedule; use header::Header; use spec::CommonParams; +use snapshot::SnapshotComponents; use transaction::{UnverifiedTransaction, SignedTransaction}; use receipt::Receipt; @@ -294,4 +295,10 @@ pub trait Engine : Sync + Send { /// Stops any services that the may hold the Engine and makes it safe to drop. fn stop(&self) {} + + /// Create a factory for building snapshot chunks and restoring from them. + /// Returning `None` indicates that this engine doesn't support snapshot creation. + fn snapshot_components(&self) -> Option> { + None + } } diff --git a/ethcore/src/engines/null_engine.rs b/ethcore/src/engines/null_engine.rs index 0611fc08e..9b9fb9469 100644 --- a/ethcore/src/engines/null_engine.rs +++ b/ethcore/src/engines/null_engine.rs @@ -60,4 +60,8 @@ impl Engine for NullEngine { fn schedule(&self, _env_info: &EnvInfo) -> Schedule { Schedule::new_homestead() } + + fn snapshot_components(&self) -> Option> { + Some(Box::new(::snapshot::PowSnapshot)) + } } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 548d0dbaa..0acea6f50 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -405,6 +405,10 @@ impl Engine for Arc { fn epoch_verifier(&self, _header: &Header, _proof: &[u8]) -> Result, Error> { Ok(Box::new(self.clone())) } + + fn snapshot_components(&self) -> Option> { + Some(Box::new(::snapshot::PowSnapshot)) + } } // Try to round gas_limit a bit so that: diff --git a/ethcore/src/snapshot/consensus/mod.rs b/ethcore/src/snapshot/consensus/mod.rs new file mode 100644 index 000000000..2e6b6b736 --- /dev/null +++ b/ethcore/src/snapshot/consensus/mod.rs @@ -0,0 +1,349 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Secondary chunk creation and restoration, implementations for different consensus +//! engines. + +use std::collections::VecDeque; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use blockchain::{BlockChain, BlockProvider}; +use engines::Engine; +use snapshot::{Error, ManifestData}; +use snapshot::block::AbridgedBlock; + +use util::{Bytes, H256}; +use util::kvdb::KeyValueDB; +use rand::OsRng; +use rlp::{RlpStream, UntrustedRlp}; + + +/// A sink for produced chunks. +pub type ChunkSink<'a> = FnMut(&[u8]) -> io::Result<()> + 'a; + +// How many blocks to include in a snapshot, starting from the head of the chain. +const SNAPSHOT_BLOCKS: u64 = 30000; + +/// Components necessary for snapshot creation and restoration. +pub trait SnapshotComponents: Send { + /// Create secondary snapshot chunks; these corroborate the state data + /// in the state chunks. + /// + /// Chunks shouldn't exceed the given preferred size, and should be fed + /// uncompressed into the sink. + /// + /// This will vary by consensus engine, so it's exposed as a trait. + fn chunk_all( + &mut self, + chain: &BlockChain, + block_at: H256, + chunk_sink: &mut ChunkSink, + preferred_size: usize, + ) -> Result<(), Error>; + + /// Create a rebuilder, which will have chunks fed into it in aribtrary + /// order and then be finalized. + /// + /// The manifest, a database, and fresh `BlockChain` are supplied. + // TODO: supply anything for state? + fn rebuilder( + &self, + chain: BlockChain, + db: Arc, + manifest: &ManifestData, + ) -> Result, ::error::Error>; +} + + +/// Restore from secondary snapshot chunks. +pub trait Rebuilder: Send { + /// Feed a chunk, potentially out of order. + /// + /// Check `abort_flag` periodically while doing heavy work. If set to `false`, should bail with + /// `Error::RestorationAborted`. + fn feed( + &mut self, + chunk: &[u8], + engine: &Engine, + abort_flag: &AtomicBool, + ) -> Result<(), ::error::Error>; + + /// Finalize the restoration. Will be done after all chunks have been + /// fed successfully. + /// This will apply the necessary "glue" between chunks. + fn finalize(&mut self) -> Result<(), Error>; +} + +/// Snapshot creation and restoration for PoW chains. +/// This includes blocks from the head of the chain as a +/// loose assurance that the chain is valid. +pub struct PowSnapshot; + +impl SnapshotComponents for PowSnapshot { + fn chunk_all( + &mut self, + chain: &BlockChain, + block_at: H256, + chunk_sink: &mut ChunkSink, + preferred_size: usize, + ) -> Result<(), Error> { + PowWorker { + chain: chain, + rlps: VecDeque::new(), + current_hash: block_at, + writer: chunk_sink, + preferred_size: preferred_size, + }.chunk_all() + } + + fn rebuilder( + &self, + chain: BlockChain, + db: Arc, + manifest: &ManifestData, + ) -> Result, ::error::Error> { + PowRebuilder::new(chain, db, manifest).map(|r| Box::new(r) as Box<_>) + } +} + +/// Used to build block chunks. +struct PowWorker<'a> { + chain: &'a BlockChain, + // block, receipt rlp pairs. + rlps: VecDeque, + current_hash: H256, + writer: &'a mut ChunkSink<'a>, + preferred_size: usize, +} + +impl<'a> PowWorker<'a> { + // Repeatedly fill the buffers and writes out chunks, moving backwards from starting block hash. + // Loops until we reach the first desired block, and writes out the remainder. + fn chunk_all(&mut self) -> Result<(), Error> { + let mut loaded_size = 0; + let mut last = self.current_hash; + + let genesis_hash = self.chain.genesis_hash(); + + for _ in 0..SNAPSHOT_BLOCKS { + if self.current_hash == genesis_hash { break } + + let (block, receipts) = self.chain.block(&self.current_hash) + .and_then(|b| self.chain.block_receipts(&self.current_hash).map(|r| (b, r))) + .ok_or(Error::BlockNotFound(self.current_hash))?; + + let abridged_rlp = AbridgedBlock::from_block_view(&block.view()).into_inner(); + + let pair = { + let mut pair_stream = RlpStream::new_list(2); + pair_stream.append_raw(&abridged_rlp, 1).append(&receipts); + pair_stream.out() + }; + + let new_loaded_size = loaded_size + pair.len(); + + // cut off the chunk if too large. + + if new_loaded_size > self.preferred_size && !self.rlps.is_empty() { + self.write_chunk(last)?; + loaded_size = pair.len(); + } else { + loaded_size = new_loaded_size; + } + + self.rlps.push_front(pair); + + last = self.current_hash; + self.current_hash = block.header_view().parent_hash(); + } + + if loaded_size != 0 { + self.write_chunk(last)?; + } + + Ok(()) + } + + // write out the data in the buffers to a chunk on disk + // + // we preface each chunk with the parent of the first block's details, + // obtained from the details of the last block written. + fn write_chunk(&mut self, last: H256) -> Result<(), Error> { + trace!(target: "snapshot", "prepared block chunk with {} blocks", self.rlps.len()); + + let (last_header, last_details) = self.chain.block_header(&last) + .and_then(|n| self.chain.block_details(&last).map(|d| (n, d))) + .ok_or(Error::BlockNotFound(last))?; + + let parent_number = last_header.number() - 1; + let parent_hash = last_header.parent_hash(); + let parent_total_difficulty = last_details.total_difficulty - *last_header.difficulty(); + + trace!(target: "snapshot", "parent last written block: {}", parent_hash); + + let num_entries = self.rlps.len(); + let mut rlp_stream = RlpStream::new_list(3 + num_entries); + rlp_stream.append(&parent_number).append(parent_hash).append(&parent_total_difficulty); + + for pair in self.rlps.drain(..) { + rlp_stream.append_raw(&pair, 1); + } + + let raw_data = rlp_stream.out(); + + (self.writer)(&raw_data)?; + + Ok(()) + } +} + +/// Rebuilder for proof-of-work chains. +/// Does basic verification for all blocks, but `PoW` verification for some. +/// Blocks must be fed in-order. +/// +/// The first block in every chunk is disconnected from the last block in the +/// chunk before it, as chunks may be submitted out-of-order. +/// +/// After all chunks have been submitted, we "glue" the chunks together. +pub struct PowRebuilder { + chain: BlockChain, + db: Arc, + rng: OsRng, + disconnected: Vec<(u64, H256)>, + best_number: u64, + best_hash: H256, + best_root: H256, + fed_blocks: u64, +} + +impl PowRebuilder { + /// Create a new PowRebuilder. + fn new(chain: BlockChain, db: Arc, manifest: &ManifestData) -> Result { + Ok(PowRebuilder { + chain: chain, + db: db, + rng: OsRng::new()?, + disconnected: Vec::new(), + best_number: manifest.block_number, + best_hash: manifest.block_hash, + best_root: manifest.state_root, + fed_blocks: 0, + }) + } +} + +impl Rebuilder for PowRebuilder { + /// Feed the rebuilder an uncompressed block chunk. + /// Returns the number of blocks fed or any errors. + fn feed(&mut self, chunk: &[u8], engine: &Engine, abort_flag: &AtomicBool) -> Result<(), ::error::Error> { + use basic_types::Seal::With; + use views::BlockView; + use snapshot::verify_old_block; + use util::U256; + use util::triehash::ordered_trie_root; + + let rlp = UntrustedRlp::new(chunk); + let item_count = rlp.item_count()?; + let num_blocks = (item_count - 3) as u64; + + trace!(target: "snapshot", "restoring block chunk with {} blocks.", item_count - 3); + + if self.fed_blocks + num_blocks > SNAPSHOT_BLOCKS { + return Err(Error::TooManyBlocks(SNAPSHOT_BLOCKS, self.fed_blocks).into()) + } + + // todo: assert here that these values are consistent with chunks being in order. + let mut cur_number = rlp.val_at::(0)? + 1; + let mut parent_hash = rlp.val_at::(1)?; + let parent_total_difficulty = rlp.val_at::(2)?; + + for idx in 3..item_count { + if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) } + + let pair = rlp.at(idx)?; + let abridged_rlp = pair.at(0)?.as_raw().to_owned(); + let abridged_block = AbridgedBlock::from_raw(abridged_rlp); + let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?; + let receipts_root = ordered_trie_root( + pair.at(1)?.iter().map(|r| r.as_raw().to_owned()) + ); + + let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; + let block_bytes = block.rlp_bytes(With); + let is_best = cur_number == self.best_number; + + if is_best { + if block.header.hash() != self.best_hash { + return Err(Error::WrongBlockHash(cur_number, self.best_hash, block.header.hash()).into()) + } + + if block.header.state_root() != &self.best_root { + return Err(Error::WrongStateRoot(self.best_root, *block.header.state_root()).into()) + } + } + + verify_old_block( + &mut self.rng, + &block.header, + engine, + &self.chain, + Some(&block_bytes), + is_best + )?; + + let mut batch = self.db.transaction(); + + // special-case the first block in each chunk. + if idx == 3 { + if self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, Some(parent_total_difficulty), is_best, false) { + self.disconnected.push((cur_number, block.header.hash())); + } + } else { + self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, is_best, false); + } + self.db.write_buffered(batch); + self.chain.commit(); + + parent_hash = BlockView::new(&block_bytes).hash(); + cur_number += 1; + } + + self.fed_blocks += num_blocks; + + Ok(()) + } + + /// Glue together any disconnected chunks and check that the chain is complete. + fn finalize(&mut self) -> Result<(), Error> { + let mut batch = self.db.transaction(); + + for (first_num, first_hash) in self.disconnected.drain(..) { + let parent_num = first_num - 1; + + // check if the parent is even in the chain. + // since we don't restore every single block in the chain, + // the first block of the first chunks has nothing to connect to. + if let Some(parent_hash) = self.chain.block_hash(parent_num) { + // if so, add the child to it. + self.chain.add_child(&mut batch, parent_hash, first_hash); + } + } + self.db.write_buffered(batch); + Ok(()) + } +} diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index c1391b300..7a3ffdca2 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -57,6 +57,8 @@ pub enum Error { VersionNotSupported(u64), /// Max chunk size is to small to fit basic account data. ChunkTooSmall, + /// Snapshots not supported by the consensus engine. + SnapshotsUnsupported, } impl fmt::Display for Error { @@ -79,6 +81,7 @@ impl fmt::Display for Error { Error::Trie(ref err) => err.fmt(f), Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver), Error::ChunkTooSmall => write!(f, "Chunk size is too small."), + Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."), } } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 1c3b4366b..db3bebde9 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -17,9 +17,9 @@ //! Snapshot creation, restoration, and network service. //! //! Documentation of the format can be found at -//! https://github.com/paritytech/parity/wiki/%22PV64%22-Snapshot-Format +//! https://github.com/paritytech/parity/wiki/Warp-Sync-Snapshot-Format -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -28,7 +28,6 @@ use blockchain::{BlockChain, BlockProvider}; use engines::Engine; use header::Header; use ids::BlockId; -use views::BlockView; use util::{Bytes, Hashable, HashDB, DBValue, snappy, U256, Uint}; use util::Mutex; @@ -40,7 +39,6 @@ use util::sha3::SHA3_NULL_RLP; use rlp::{RlpStream, UntrustedRlp}; use bloom_journal::Bloom; -use self::block::AbridgedBlock; use self::io::SnapshotWriter; use super::state_db::StateDB; @@ -51,6 +49,7 @@ use rand::{Rng, OsRng}; pub use self::error::Error; +pub use self::consensus::*; pub use self::service::{Service, DatabaseRestore}; pub use self::traits::SnapshotService; pub use self::watcher::Watcher; @@ -63,6 +62,7 @@ pub mod service; mod account; mod block; +mod consensus; mod error; mod watcher; @@ -83,9 +83,6 @@ mod traits { // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; -// How many blocks to include in a snapshot, starting from the head of the chain. -const SNAPSHOT_BLOCKS: u64 = 30000; - /// A progress indicator for snapshots. #[derive(Debug, Default)] pub struct Progress { @@ -122,6 +119,7 @@ impl Progress { } /// Take a snapshot using the given blockchain, starting block hash, and database, writing into the given writer. pub fn take_snapshot( + engine: &Engine, chain: &BlockChain, block_at: H256, state_db: &HashDB, @@ -136,9 +134,11 @@ pub fn take_snapshot( info!("Taking snapshot starting at block {}", number); let writer = Mutex::new(writer); + let chunker = engine.snapshot_components().ok_or(Error::SnapshotsUnsupported)?; let (state_hashes, block_hashes) = scope(|scope| { - let block_guard = scope.spawn(|| chunk_blocks(chain, block_at, &writer, p)); - let state_res = chunk_state(state_db, state_root, &writer, p); + let writer = &writer; + let block_guard = scope.spawn(move || chunk_secondary(chunker, chain, block_at, writer, p)); + let state_res = chunk_state(state_db, state_root, writer, p); state_res.and_then(|state_hashes| { block_guard.join().map(|block_hashes| (state_hashes, block_hashes)) @@ -163,128 +163,41 @@ pub fn take_snapshot( Ok(()) } -/// Used to build block chunks. -struct BlockChunker<'a> { - chain: &'a BlockChain, - // block, receipt rlp pairs. - rlps: VecDeque, - current_hash: H256, - hashes: Vec, - snappy_buffer: Vec, - writer: &'a Mutex, - progress: &'a Progress, -} - -impl<'a> BlockChunker<'a> { - // Repeatedly fill the buffers and writes out chunks, moving backwards from starting block hash. - // Loops until we reach the first desired block, and writes out the remainder. - fn chunk_all(&mut self) -> Result<(), Error> { - let mut loaded_size = 0; - let mut last = self.current_hash; - - let genesis_hash = self.chain.genesis_hash(); - - for _ in 0..SNAPSHOT_BLOCKS { - if self.current_hash == genesis_hash { break } - - let (block, receipts) = self.chain.block(&self.current_hash) - .and_then(|b| self.chain.block_receipts(&self.current_hash).map(|r| (b, r))) - .ok_or(Error::BlockNotFound(self.current_hash))?; - - let abridged_rlp = AbridgedBlock::from_block_view(&block.view()).into_inner(); - - let pair = { - let mut pair_stream = RlpStream::new_list(2); - pair_stream.append_raw(&abridged_rlp, 1).append(&receipts); - pair_stream.out() - }; - - let new_loaded_size = loaded_size + pair.len(); - - // cut off the chunk if too large. - - if new_loaded_size > PREFERRED_CHUNK_SIZE && !self.rlps.is_empty() { - self.write_chunk(last)?; - loaded_size = pair.len(); - } else { - loaded_size = new_loaded_size; - } - - self.rlps.push_front(pair); - - last = self.current_hash; - self.current_hash = block.header_view().parent_hash(); - } - - if loaded_size != 0 { - self.write_chunk(last)?; - } - - Ok(()) - } - - // write out the data in the buffers to a chunk on disk - // - // we preface each chunk with the parent of the first block's details, - // obtained from the details of the last block written. - fn write_chunk(&mut self, last: H256) -> Result<(), Error> { - trace!(target: "snapshot", "prepared block chunk with {} blocks", self.rlps.len()); - - let (last_header, last_details) = self.chain.block_header(&last) - .and_then(|n| self.chain.block_details(&last).map(|d| (n, d))) - .ok_or(Error::BlockNotFound(last))?; - - let parent_number = last_header.number() - 1; - let parent_hash = last_header.parent_hash(); - let parent_total_difficulty = last_details.total_difficulty - *last_header.difficulty(); - - trace!(target: "snapshot", "parent last written block: {}", parent_hash); - - let num_entries = self.rlps.len(); - let mut rlp_stream = RlpStream::new_list(3 + num_entries); - rlp_stream.append(&parent_number).append(parent_hash).append(&parent_total_difficulty); - - for pair in self.rlps.drain(..) { - rlp_stream.append_raw(&pair, 1); - } - - let raw_data = rlp_stream.out(); - - let size = snappy::compress_into(&raw_data, &mut self.snappy_buffer); - let compressed = &self.snappy_buffer[..size]; - let hash = compressed.sha3(); - - self.writer.lock().write_block_chunk(hash, compressed)?; - trace!(target: "snapshot", "wrote block chunk. hash: {}, size: {}, uncompressed size: {}", hash.hex(), size, raw_data.len()); - - self.progress.size.fetch_add(size, Ordering::SeqCst); - self.progress.blocks.fetch_add(num_entries, Ordering::SeqCst); - - self.hashes.push(hash); - Ok(()) - } -} - -/// Create and write out all block chunks to disk, returning a vector of all -/// the hashes of block chunks created. +/// Create and write out all secondary chunks to disk, returning a vector of all +/// the hashes of secondary chunks created. /// -/// The path parameter is the directory to store the block chunks in. -/// This function assumes the directory exists already. +/// Secondary chunks are engine-specific, but they intend to corroborate the state data +/// in the state chunks. /// Returns a list of chunk hashes, with the first having the blocks furthest from the genesis. -pub fn chunk_blocks<'a>(chain: &'a BlockChain, start_hash: H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { - let mut chunker = BlockChunker { - chain: chain, - rlps: VecDeque::new(), - current_hash: start_hash, - hashes: Vec::new(), - snappy_buffer: vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)], - writer: writer, - progress: progress, - }; +pub fn chunk_secondary<'a>(mut chunker: Box, chain: &'a BlockChain, start_hash: H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { + let mut chunk_hashes = Vec::new(); + let mut snappy_buffer = vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)]; - chunker.chunk_all()?; + { + let mut chunk_sink = |raw_data: &[u8]| { + let compressed_size = snappy::compress_into(raw_data, &mut snappy_buffer); + let compressed = &snappy_buffer[..compressed_size]; + let hash = compressed.sha3(); + let size = compressed.len(); - Ok(chunker.hashes) + writer.lock().write_block_chunk(hash, compressed)?; + trace!(target: "snapshot", "wrote secondary chunk. hash: {}, size: {}, uncompressed size: {}", + hash.hex(), size, raw_data.len()); + + progress.size.fetch_add(size, Ordering::SeqCst); + chunk_hashes.push(hash); + Ok(()) + }; + + chunker.chunk_all( + chain, + start_hash, + &mut chunk_sink, + PREFERRED_CHUNK_SIZE, + )?; + } + + Ok(chunk_hashes) } /// State trie chunker. @@ -564,158 +477,15 @@ const POW_VERIFY_RATE: f32 = 0.02; /// the fullest verification possible. If not, it will take a random sample to determine whether it will /// do heavy or light verification. pub fn verify_old_block(rng: &mut OsRng, header: &Header, engine: &Engine, chain: &BlockChain, body: Option<&[u8]>, always: bool) -> Result<(), ::error::Error> { + engine.verify_block_basic(header, body)?; + if always || rng.gen::() <= POW_VERIFY_RATE { + engine.verify_block_unordered(header, body)?; match chain.block_header(header.parent_hash()) { Some(parent) => engine.verify_block_family(header, &parent, body), - None => engine.verify_block_seal(header), // TODO: fetch validation proof as necessary. + None => Ok(()), } } else { - engine.verify_block_basic(header, body) - } -} - -/// Rebuilds the blockchain from chunks. -/// -/// Does basic verification for all blocks, but `PoW` verification for some. -/// Blocks must be fed in-order. -/// -/// The first block in every chunk is disconnected from the last block in the -/// chunk before it, as chunks may be submitted out-of-order. -/// -/// After all chunks have been submitted, we "glue" the chunks together. -pub struct BlockRebuilder { - chain: BlockChain, - db: Arc, - rng: OsRng, - disconnected: Vec<(u64, H256)>, - best_number: u64, - best_hash: H256, - best_root: H256, - fed_blocks: u64, -} - -impl BlockRebuilder { - /// Create a new BlockRebuilder. - pub fn new(chain: BlockChain, db: Arc, manifest: &ManifestData) -> Result { - Ok(BlockRebuilder { - chain: chain, - db: db, - rng: OsRng::new()?, - disconnected: Vec::new(), - best_number: manifest.block_number, - best_hash: manifest.block_hash, - best_root: manifest.state_root, - fed_blocks: 0, - }) - } - - /// Feed the rebuilder an uncompressed block chunk. - /// Returns the number of blocks fed or any errors. - pub fn feed(&mut self, chunk: &[u8], engine: &Engine, abort_flag: &AtomicBool) -> Result { - use basic_types::Seal::With; - use util::U256; - use util::triehash::ordered_trie_root; - - let rlp = UntrustedRlp::new(chunk); - let item_count = rlp.item_count()?; - let num_blocks = (item_count - 3) as u64; - - trace!(target: "snapshot", "restoring block chunk with {} blocks.", item_count - 3); - - if self.fed_blocks + num_blocks > SNAPSHOT_BLOCKS { - return Err(Error::TooManyBlocks(SNAPSHOT_BLOCKS, self.fed_blocks).into()) - } - - // todo: assert here that these values are consistent with chunks being in order. - let mut cur_number = rlp.val_at::(0)? + 1; - let mut parent_hash = rlp.val_at::(1)?; - let parent_total_difficulty = rlp.val_at::(2)?; - - for idx in 3..item_count { - if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) } - - let pair = rlp.at(idx)?; - let abridged_rlp = pair.at(0)?.as_raw().to_owned(); - let abridged_block = AbridgedBlock::from_raw(abridged_rlp); - let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?; - let receipts_root = ordered_trie_root( - pair.at(1)?.iter().map(|r| r.as_raw().to_owned()) - ); - - let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; - let block_bytes = block.rlp_bytes(With); - let is_best = cur_number == self.best_number; - - if is_best { - if block.header.hash() != self.best_hash { - return Err(Error::WrongBlockHash(cur_number, self.best_hash, block.header.hash()).into()) - } - - if block.header.state_root() != &self.best_root { - return Err(Error::WrongStateRoot(self.best_root, *block.header.state_root()).into()) - } - } - - verify_old_block( - &mut self.rng, - &block.header, - engine, - &self.chain, - Some(&block_bytes), - is_best - )?; - - let mut batch = self.db.transaction(); - - // special-case the first block in each chunk. - if idx == 3 { - if self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, Some(parent_total_difficulty), is_best, false) { - self.disconnected.push((cur_number, block.header.hash())); - } - } else { - self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, is_best, false); - } - self.db.write_buffered(batch); - self.chain.commit(); - - parent_hash = BlockView::new(&block_bytes).hash(); - cur_number += 1; - } - - self.fed_blocks += num_blocks; - - Ok(num_blocks) - } - - /// Glue together any disconnected chunks and check that the chain is complete. - pub fn finalize(self, canonical: HashMap) -> Result<(), Error> { - let mut batch = self.db.transaction(); - - for (first_num, first_hash) in self.disconnected { - let parent_num = first_num - 1; - - // check if the parent is even in the chain. - // since we don't restore every single block in the chain, - // the first block of the first chunks has nothing to connect to. - if let Some(parent_hash) = self.chain.block_hash(parent_num) { - // if so, add the child to it. - self.chain.add_child(&mut batch, parent_hash, first_hash); - } - } - self.db.write_buffered(batch); - - let best_number = self.best_number; - for num in (0..self.fed_blocks).map(|x| best_number - x) { - - let hash = self.chain.block_hash(num).ok_or(Error::IncompleteChain)?; - - if let Some(canon_hash) = canonical.get(&num).cloned() { - if canon_hash != hash { - return Err(Error::WrongBlockHash(num, canon_hash, hash)); - } - } - } - Ok(()) } } diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index 06e659bc1..dc92b5427 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -16,14 +16,14 @@ //! Snapshot network service implementation. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::io::ErrorKind; use std::fs; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use super::{ManifestData, StateRebuilder, BlockRebuilder, RestorationStatus, SnapshotService}; +use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService}; use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}; use blockchain::BlockChain; @@ -69,12 +69,11 @@ struct Restoration { state_chunks_left: HashSet, block_chunks_left: HashSet, state: StateRebuilder, - blocks: BlockRebuilder, + secondary: Box, writer: Option, snappy_buffer: Bytes, final_state_root: H256, guard: Guard, - canonical_hashes: HashMap, db: Arc, } @@ -86,6 +85,7 @@ struct RestorationParams<'a> { writer: Option, // writer for recovered snapshot. genesis: &'a [u8], // genesis block of the chain. guard: Guard, // guard for the restoration directory. + engine: &'a Engine, } impl Restoration { @@ -100,7 +100,10 @@ impl Restoration { .map_err(UtilError::SimpleString)?); let chain = BlockChain::new(Default::default(), params.genesis, raw_db.clone()); - let blocks = BlockRebuilder::new(chain, raw_db.clone(), &manifest)?; + let components = params.engine.snapshot_components() + .ok_or_else(|| ::snapshot::Error::SnapshotsUnsupported)?; + + let secondary = components.rebuilder(chain, raw_db.clone(), &manifest)?; let root = manifest.state_root.clone(); Ok(Restoration { @@ -108,12 +111,11 @@ impl Restoration { state_chunks_left: state_chunks, block_chunks_left: block_chunks, state: StateRebuilder::new(raw_db.clone(), params.pruning), - blocks: blocks, + secondary: secondary, writer: params.writer, snappy_buffer: Vec::new(), final_state_root: root, guard: params.guard, - canonical_hashes: HashMap::new(), db: raw_db, }) } @@ -138,7 +140,7 @@ impl Restoration { if self.block_chunks_left.remove(&hash) { let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; - self.blocks.feed(&self.snappy_buffer[..len], engine, flag)?; + self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?; if let Some(ref mut writer) = self.writer.as_mut() { writer.write_block_chunk(hash, chunk)?; } @@ -147,13 +149,8 @@ impl Restoration { Ok(()) } - // note canonical hashes. - fn note_canonical(&mut self, hashes: &[(u64, H256)]) { - self.canonical_hashes.extend(hashes.iter().cloned()); - } - // finish up restoration. - fn finalize(self) -> Result<(), Error> { + fn finalize(mut self) -> Result<(), Error> { use util::trie::TrieError; if !self.is_done() { return Ok(()) } @@ -169,7 +166,7 @@ impl Restoration { self.state.finalize(self.manifest.block_number, self.manifest.block_hash)?; // connect out-of-order chunks and verify chain integrity. - self.blocks.finalize(self.canonical_hashes)?; + self.secondary.finalize()?; if let Some(writer) = self.writer { writer.finish(self.manifest)?; @@ -425,6 +422,7 @@ impl Service { writer: writer, genesis: &self.genesis_block, guard: Guard::new(rest_dir), + engine: &*self.engine, }; let state_chunks = params.manifest.state_hashes.len(); @@ -593,14 +591,6 @@ impl SnapshotService for Service { trace!("Error sending snapshot service message: {:?}", e); } } - - fn provide_canon_hashes(&self, canonical: &[(u64, H256)]) { - let mut rest = self.restoration.lock(); - - if let Some(ref mut rest) = rest.as_mut() { - rest.note_canonical(canonical); - } - } } impl Drop for Service { diff --git a/ethcore/src/snapshot/snapshot_service_trait.rs b/ethcore/src/snapshot/snapshot_service_trait.rs index 7b53ee9b9..67e96398e 100644 --- a/ethcore/src/snapshot/snapshot_service_trait.rs +++ b/ethcore/src/snapshot/snapshot_service_trait.rs @@ -48,10 +48,6 @@ pub trait SnapshotService : Sync + Send { /// Feed a raw block chunk to the service to be processed asynchronously. /// no-op if currently restoring. fn restore_block_chunk(&self, hash: H256, chunk: Bytes); - - /// Give the restoration in-progress some canonical block hashes for - /// extra verification (performed at the end) - fn provide_canon_hashes(&self, canonical: &[(u64, H256)]); } impl IpcConfig for SnapshotService { } diff --git a/ethcore/src/snapshot/tests/blocks.rs b/ethcore/src/snapshot/tests/blocks.rs index ff63afdfc..fae9ae75e 100644 --- a/ethcore/src/snapshot/tests/blocks.rs +++ b/ethcore/src/snapshot/tests/blocks.rs @@ -21,13 +21,12 @@ use error::Error; use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer}; use blockchain::BlockChain; -use snapshot::{chunk_blocks, BlockRebuilder, Error as SnapshotError, Progress}; +use snapshot::{chunk_secondary, Error as SnapshotError, Progress, SnapshotComponents}; use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; use util::{Mutex, snappy}; -use util::kvdb::{Database, DatabaseConfig}; +use util::kvdb::{self, KeyValueDB, DBTransaction}; -use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -35,19 +34,18 @@ fn chunk_and_restore(amount: u64) { let mut canon_chain = ChainGenerator::default(); let mut finalizer = BlockFinalizer::default(); let genesis = canon_chain.generate(&mut finalizer).unwrap(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let components = ::snapshot::PowSnapshot; let engine = Arc::new(::engines::NullEngine::default()); - let orig_path = RandomTempPath::create_dir(); let new_path = RandomTempPath::create_dir(); let mut snapshot_path = new_path.as_path().to_owned(); snapshot_path.push("SNAP"); - let old_db = Arc::new(Database::open(&db_cfg, orig_path.as_str()).unwrap()); + let old_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0))); let bc = BlockChain::new(Default::default(), &genesis, old_db.clone()); // build the blockchain. - let mut batch = old_db.transaction(); + let mut batch = DBTransaction::new(); for _ in 0..amount { let block = canon_chain.generate(&mut finalizer).unwrap(); bc.insert_block(&mut batch, &block, vec![]); @@ -56,12 +54,18 @@ fn chunk_and_restore(amount: u64) { old_db.write(batch).unwrap(); - let best_hash = bc.best_block_hash(); // snapshot it. let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); - let block_hashes = chunk_blocks(&bc, best_hash, &writer, &Progress::default()).unwrap(); + let block_hashes = chunk_secondary( + Box::new(::snapshot::PowSnapshot), + &bc, + best_hash, + &writer, + &Progress::default() + ).unwrap(); + let manifest = ::snapshot::ManifestData { version: 2, state_hashes: Vec::new(), @@ -74,9 +78,10 @@ fn chunk_and_restore(amount: u64) { writer.into_inner().finish(manifest.clone()).unwrap(); // restore it. - let new_db = Arc::new(Database::open(&db_cfg, new_path.as_str()).unwrap()); + let new_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0))); let new_chain = BlockChain::new(Default::default(), &genesis, new_db.clone()); - let mut rebuilder = BlockRebuilder::new(new_chain, new_db.clone(), &manifest).unwrap(); + let mut rebuilder = components.rebuilder(new_chain, new_db.clone(), &manifest).unwrap(); + let reader = PackedReader::new(&snapshot_path).unwrap().unwrap(); let flag = AtomicBool::new(true); for chunk_hash in &reader.manifest().block_hashes { @@ -85,7 +90,8 @@ fn chunk_and_restore(amount: u64) { rebuilder.feed(&chunk, engine.as_ref(), &flag).unwrap(); } - rebuilder.finalize(HashMap::new()).unwrap(); + rebuilder.finalize().unwrap(); + drop(rebuilder); // and test it. let new_chain = BlockChain::new(Default::default(), &genesis, new_db); @@ -118,10 +124,8 @@ fn checks_flag() { }; let chunk = stream.out(); - let path = RandomTempPath::create_dir(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); - let db = Arc::new(Database::open(&db_cfg, path.as_str()).unwrap()); + let db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0))); let engine = Arc::new(::engines::NullEngine::default()); let chain = BlockChain::new(Default::default(), &genesis, db.clone()); @@ -134,7 +138,7 @@ fn checks_flag() { block_hash: H256::default(), }; - let mut rebuilder = BlockRebuilder::new(chain, db.clone(), &manifest).unwrap(); + let mut rebuilder = ::snapshot::PowSnapshot.rebuilder(chain, db.clone(), &manifest).unwrap(); match rebuilder.feed(&chunk, engine.as_ref(), &AtomicBool::new(false)) { Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {} diff --git a/rpc/src/v1/tests/helpers/snapshot_service.rs b/rpc/src/v1/tests/helpers/snapshot_service.rs index 2314cea10..f03eb3bc3 100644 --- a/rpc/src/v1/tests/helpers/snapshot_service.rs +++ b/rpc/src/v1/tests/helpers/snapshot_service.rs @@ -47,5 +47,4 @@ impl SnapshotService for TestSnapshotService { fn abort_restore(&self) { } fn restore_state_chunk(&self, _hash: H256, _chunk: Bytes) { } fn restore_block_chunk(&self, _hash: H256, _chunk: Bytes) { } - fn provide_canon_hashes(&self, _hashes: &[(u64, H256)]) { } -} \ No newline at end of file +} diff --git a/sync/src/tests/snapshot.rs b/sync/src/tests/snapshot.rs index 16114e216..0f97ec913 100644 --- a/sync/src/tests/snapshot.rs +++ b/sync/src/tests/snapshot.rs @@ -24,7 +24,6 @@ use SyncConfig; pub struct TestSnapshotService { manifest: Option, chunks: HashMap, - canon_hashes: Mutex>, restoration_manifest: Mutex>, state_restoration_chunks: Mutex>, @@ -36,7 +35,6 @@ impl TestSnapshotService { TestSnapshotService { manifest: None, chunks: HashMap::new(), - canon_hashes: Mutex::new(HashMap::new()), restoration_manifest: Mutex::new(None), state_restoration_chunks: Mutex::new(HashMap::new()), block_restoration_chunks: Mutex::new(HashMap::new()), @@ -61,7 +59,6 @@ impl TestSnapshotService { TestSnapshotService { manifest: Some(manifest), chunks: chunks, - canon_hashes: Mutex::new(HashMap::new()), restoration_manifest: Mutex::new(None), state_restoration_chunks: Mutex::new(HashMap::new()), block_restoration_chunks: Mutex::new(HashMap::new()), @@ -115,10 +112,6 @@ impl SnapshotService for TestSnapshotService { self.block_restoration_chunks.lock().insert(hash, chunk); } } - - fn provide_canon_hashes(&self, hashes: &[(u64, H256)]) { - self.canon_hashes.lock().extend(hashes.iter().cloned()); - } } #[test] From 240c111ebbfdf7870a949881fd55c84667c4f8ba Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 19 Apr 2017 20:44:11 +0200 Subject: [PATCH 17/22] fix indent --- ethcore/src/engines/authority_round.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index 2d158fcdb..937d6ccc4 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -585,7 +585,7 @@ mod tests { #[test] fn rejects_step_backwards() { - let tap = AccountProvider::transient_provider(); + let tap = AccountProvider::transient_provider(); let addr = tap.insert_account(Secret::from_slice(&"0".sha3()).unwrap(), "0").unwrap(); let mut parent_header: Header = Header::default(); From a33b4cc73b4c8199c3992ca731216a3424deec99 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 20 Apr 2017 15:04:07 +0200 Subject: [PATCH 18/22] fix tests --- util/src/kvdb.rs | 2 +- util/src/migration/tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 8006c4ccb..084969f30 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -910,7 +910,7 @@ mod tests { assert_eq!(&*db.get(None, &key1).unwrap().unwrap(), b"cat"); - let contents: Vec<_> = db.iter(None).collect(); + let contents: Vec<_> = db.iter(None).into_iter().flat_map(|inner| inner).collect(); assert_eq!(contents.len(), 2); assert_eq!(&*contents[0].0, &*key1); assert_eq!(&*contents[0].1, b"cat"); diff --git a/util/src/migration/tests.rs b/util/src/migration/tests.rs index 31226ec49..a246f65c8 100644 --- a/util/src/migration/tests.rs +++ b/util/src/migration/tests.rs @@ -94,7 +94,7 @@ impl Migration for AddsColumn { fn migrate(&mut self, source: Arc, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { let mut batch = Batch::new(config, col); - for (key, value) in source.iter(col) { + for (key, value) in source.iter(col).into_iter().flat_map(|inner| inner) { batch.insert(key.to_vec(), value.to_vec(), dest)?; } From 468a7a4a778490394b7494bd41b2ab599a49bfcf Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 20 Apr 2017 16:09:43 +0200 Subject: [PATCH 19/22] bloom check test --- .../engines/validator_set/safe_contract.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 39cd4f511..3f075330f 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -280,7 +280,7 @@ mod tests { use miner::MinerService; use tests::helpers::{generate_dummy_client_with_spec_and_accounts, generate_dummy_client_with_spec_and_data}; use super::super::ValidatorSet; - use super::ValidatorSafeContract; + use super::{ValidatorSafeContract, EVENT_NAME_HASH}; #[test] fn fetches_validators() { @@ -358,4 +358,35 @@ mod tests { sync_client.flush_queue(); assert_eq!(sync_client.chain_info().best_block_number, 3); } + + #[test] + fn detects_bloom() { + use header::Header; + use engines::{EpochChange, Unsure}; + use log_entry::LogEntry; + + let client = generate_dummy_client_with_spec_and_accounts(Spec::new_validator_safe_contract, None); + let engine = client.engine().clone(); + let validator_contract = Address::from_str("0000000000000000000000000000000000000005").unwrap(); + + let last_hash = client.best_block_header().hash(); + let mut new_header = Header::default(); + new_header.set_parent_hash(last_hash); + + // first, try without the parent hash. + let mut event = LogEntry { + address: validator_contract, + topics: vec![*EVENT_NAME_HASH], + data: Vec::new(), + }; + + new_header.set_log_bloom(event.bloom()); + assert_eq!(engine.is_epoch_end(&new_header, None, None), EpochChange::No); + + // with the last hash, it should need the receipts. + event.topics.push(last_hash); + new_header.set_log_bloom(event.bloom()); + assert_eq!(engine.is_epoch_end(&new_header, None, None), + EpochChange::Unsure(Unsure::NeedsReceipts)); + } } From 023c45f302c0bccd78b18f5ca7cab0af406a32e7 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 20 Apr 2017 17:34:04 +0200 Subject: [PATCH 20/22] checkout correct tests submodule --- ethcore/res/ethereum/tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index d52059307..ef191fdc6 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit d520593078fa0849dcd1f907e44ed0a616892e33 +Subproject commit ef191fdc61cf76cdb9cdc147465fb447304b0ed2 From 7ab92f0807a7de6cf3e841111421ab1c0d3c507b Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Sat, 22 Apr 2017 18:56:01 +0200 Subject: [PATCH 21/22] epoch generation proof fixes --- ethcore/src/client/client.rs | 24 ++++++++++--------- .../engines/validator_set/safe_contract.rs | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 44d7238b9..866742230 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -256,10 +256,8 @@ impl Client { // ensure genesis epoch proof in the DB. { - let mut batch = DBTransaction::new(); let chain = client.chain.read(); - client.generate_epoch_proof(&mut batch, &spec.genesis_header(), 0, &*chain); - client.db.read().write_buffered(batch); + client.generate_epoch_proof(&spec.genesis_header(), 0, &*chain); } if let Some(reg_addr) = client.additional_params().get("registrar").and_then(|s| Address::from_str(s).ok()) { @@ -611,10 +609,6 @@ impl Client { state.journal_under(&mut batch, number, hash).expect("DB commit failed"); let route = chain.insert_block(&mut batch, block_data, receipts); - if let Some((header, epoch)) = entering_new_epoch { - self.generate_epoch_proof(&mut batch, &header, epoch, &chain); - } - self.tracedb.read().import(&mut batch, TraceImportRequest { traces: traces.into(), block_hash: hash.clone(), @@ -634,14 +628,19 @@ impl Client { warn!("Failed to prune ancient state data: {}", e); } + if let Some((header, epoch)) = entering_new_epoch { + self.generate_epoch_proof(&header, epoch, &chain); + } + route } // generate an epoch transition proof at the given block, and write it into the given blockchain. - fn generate_epoch_proof(&self, batch: &mut DBTransaction, header: &Header, epoch_number: u64, chain: &BlockChain) { + fn generate_epoch_proof(&self, header: &Header, epoch_number: u64, chain: &BlockChain) { use std::cell::RefCell; use std::collections::BTreeSet; + let mut batch = DBTransaction::new(); let hash = header.hash(); debug!(target: "client", "Generating validation proof for block {}", hash); @@ -664,13 +663,16 @@ impl Client { // insert into database, using the generated proof. match proof { - Ok(proof) => - chain.insert_epoch_transition(batch, epoch_number, EpochTransition { + Ok(proof) => { + chain.insert_epoch_transition(&mut batch, epoch_number, EpochTransition { block_hash: hash.clone(), block_number: header.number(), proof: proof, state_proof: read_values.into_inner().into_iter().collect(), - }), + }); + + self.db.read().write_buffered(batch); + } Err(e) => { warn!(target: "client", "Error generating epoch change proof for block {}: {}", hash, e); warn!(target: "client", "Snapshots generated by this node will be incomplete."); diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 3f075330f..bfdab65b0 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -170,8 +170,8 @@ impl ValidatorSet for ValidatorSafeContract { Some(matched_event) => { // decode log manually until the native contract generator is // good enough to do it for us. - let &(_, _, ref nonce_token) = &matched_event.params[2]; - let &(_, _, ref validators_token) = &matched_event.params[3]; + let &(_, _, ref nonce_token) = &matched_event.params[1]; + let &(_, _, ref validators_token) = &matched_event.params[2]; let nonce: Option = nonce_token.clone().to_uint() .map(H256).map(Into::into); From 6a5702f27c64fc2cfbe46989df32df10b12fd707 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 24 Apr 2017 13:14:50 +0200 Subject: [PATCH 22/22] address grumbles --- ethcore/src/client/client.rs | 2 +- ethcore/src/engines/epoch_verifier.rs | 2 +- ethcore/src/engines/null_engine.rs | 2 +- ethcore/src/ethereum/ethash.rs | 6 +++++- ethcore/src/snapshot/consensus/mod.rs | 25 ++++++++++++++----------- ethcore/src/snapshot/tests/blocks.rs | 9 +++++---- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 866742230..cc8520cb7 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -414,7 +414,7 @@ impl Client { // Final Verification if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header()) { - warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + warn!(target: "client", "Stage 5 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); } diff --git a/ethcore/src/engines/epoch_verifier.rs b/ethcore/src/engines/epoch_verifier.rs index 62740fbd4..0d9c87e53 100644 --- a/ethcore/src/engines/epoch_verifier.rs +++ b/ethcore/src/engines/epoch_verifier.rs @@ -19,7 +19,7 @@ use error::Error; use header::Header; -/// Verifier for all blocks within an epoch without accessing +/// Verifier for all blocks within an epoch with self-contained state. /// /// See docs on `Engine` relating to proving functions for more details. pub trait EpochVerifier: Sync { diff --git a/ethcore/src/engines/null_engine.rs b/ethcore/src/engines/null_engine.rs index 09bd07607..3bdef480c 100644 --- a/ethcore/src/engines/null_engine.rs +++ b/ethcore/src/engines/null_engine.rs @@ -62,6 +62,6 @@ impl Engine for NullEngine { } fn snapshot_components(&self) -> Option> { - Some(Box::new(::snapshot::PowSnapshot)) + Some(Box::new(::snapshot::PowSnapshot(10000))) } } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index c7174e197..de53227de 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -32,6 +32,10 @@ use rlp::{self, UntrustedRlp}; /// Parity tries to round block.gas_limit to multiple of this constant pub const PARITY_GAS_LIMIT_DETERMINANT: U256 = U256([37, 0, 0, 0]); +/// Number of blocks in an ethash snapshot. +// make dependent on difficulty incrment divisor? +const SNAPSHOT_BLOCKS: u64 = 30000; + /// Ethash params. #[derive(Debug, PartialEq)] pub struct EthashParams { @@ -401,7 +405,7 @@ impl Engine for Arc { } fn snapshot_components(&self) -> Option> { - Some(Box::new(::snapshot::PowSnapshot)) + Some(Box::new(::snapshot::PowSnapshot(SNAPSHOT_BLOCKS))) } } diff --git a/ethcore/src/snapshot/consensus/mod.rs b/ethcore/src/snapshot/consensus/mod.rs index 2e6b6b736..4d853ca27 100644 --- a/ethcore/src/snapshot/consensus/mod.rs +++ b/ethcore/src/snapshot/consensus/mod.rs @@ -36,9 +36,6 @@ use rlp::{RlpStream, UntrustedRlp}; /// A sink for produced chunks. pub type ChunkSink<'a> = FnMut(&[u8]) -> io::Result<()> + 'a; -// How many blocks to include in a snapshot, starting from the head of the chain. -const SNAPSHOT_BLOCKS: u64 = 30000; - /// Components necessary for snapshot creation and restoration. pub trait SnapshotComponents: Send { /// Create secondary snapshot chunks; these corroborate the state data @@ -92,7 +89,11 @@ pub trait Rebuilder: Send { /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a /// loose assurance that the chain is valid. -pub struct PowSnapshot; +/// +/// The field is the number of blocks from the head of the chain +/// to include in the snapshot. +#[derive(Clone, Copy, PartialEq)] +pub struct PowSnapshot(pub u64); impl SnapshotComponents for PowSnapshot { fn chunk_all( @@ -108,7 +109,7 @@ impl SnapshotComponents for PowSnapshot { current_hash: block_at, writer: chunk_sink, preferred_size: preferred_size, - }.chunk_all() + }.chunk_all(self.0) } fn rebuilder( @@ -117,7 +118,7 @@ impl SnapshotComponents for PowSnapshot { db: Arc, manifest: &ManifestData, ) -> Result, ::error::Error> { - PowRebuilder::new(chain, db, manifest).map(|r| Box::new(r) as Box<_>) + PowRebuilder::new(chain, db, manifest, self.0).map(|r| Box::new(r) as Box<_>) } } @@ -134,13 +135,13 @@ struct PowWorker<'a> { impl<'a> PowWorker<'a> { // Repeatedly fill the buffers and writes out chunks, moving backwards from starting block hash. // Loops until we reach the first desired block, and writes out the remainder. - fn chunk_all(&mut self) -> Result<(), Error> { + fn chunk_all(&mut self, snapshot_blocks: u64) -> Result<(), Error> { let mut loaded_size = 0; let mut last = self.current_hash; let genesis_hash = self.chain.genesis_hash(); - for _ in 0..SNAPSHOT_BLOCKS { + for _ in 0..snapshot_blocks { if self.current_hash == genesis_hash { break } let (block, receipts) = self.chain.block(&self.current_hash) @@ -229,11 +230,12 @@ pub struct PowRebuilder { best_hash: H256, best_root: H256, fed_blocks: u64, + snapshot_blocks: u64, } impl PowRebuilder { /// Create a new PowRebuilder. - fn new(chain: BlockChain, db: Arc, manifest: &ManifestData) -> Result { + fn new(chain: BlockChain, db: Arc, manifest: &ManifestData, snapshot_blocks: u64) -> Result { Ok(PowRebuilder { chain: chain, db: db, @@ -243,6 +245,7 @@ impl PowRebuilder { best_hash: manifest.block_hash, best_root: manifest.state_root, fed_blocks: 0, + snapshot_blocks: snapshot_blocks, }) } } @@ -263,8 +266,8 @@ impl Rebuilder for PowRebuilder { trace!(target: "snapshot", "restoring block chunk with {} blocks.", item_count - 3); - if self.fed_blocks + num_blocks > SNAPSHOT_BLOCKS { - return Err(Error::TooManyBlocks(SNAPSHOT_BLOCKS, self.fed_blocks).into()) + if self.fed_blocks + num_blocks > self.snapshot_blocks { + return Err(Error::TooManyBlocks(self.snapshot_blocks, self.fed_blocks).into()) } // todo: assert here that these values are consistent with chunks being in order. diff --git a/ethcore/src/snapshot/tests/blocks.rs b/ethcore/src/snapshot/tests/blocks.rs index fae9ae75e..2769644e8 100644 --- a/ethcore/src/snapshot/tests/blocks.rs +++ b/ethcore/src/snapshot/tests/blocks.rs @@ -30,11 +30,12 @@ use util::kvdb::{self, KeyValueDB, DBTransaction}; use std::sync::Arc; use std::sync::atomic::AtomicBool; +const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot(30000); + fn chunk_and_restore(amount: u64) { let mut canon_chain = ChainGenerator::default(); let mut finalizer = BlockFinalizer::default(); let genesis = canon_chain.generate(&mut finalizer).unwrap(); - let components = ::snapshot::PowSnapshot; let engine = Arc::new(::engines::NullEngine::default()); let new_path = RandomTempPath::create_dir(); @@ -59,7 +60,7 @@ fn chunk_and_restore(amount: u64) { // snapshot it. let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); let block_hashes = chunk_secondary( - Box::new(::snapshot::PowSnapshot), + Box::new(SNAPSHOT_MODE), &bc, best_hash, &writer, @@ -80,7 +81,7 @@ fn chunk_and_restore(amount: u64) { // restore it. let new_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0))); let new_chain = BlockChain::new(Default::default(), &genesis, new_db.clone()); - let mut rebuilder = components.rebuilder(new_chain, new_db.clone(), &manifest).unwrap(); + let mut rebuilder = SNAPSHOT_MODE.rebuilder(new_chain, new_db.clone(), &manifest).unwrap(); let reader = PackedReader::new(&snapshot_path).unwrap().unwrap(); let flag = AtomicBool::new(true); @@ -138,7 +139,7 @@ fn checks_flag() { block_hash: H256::default(), }; - let mut rebuilder = ::snapshot::PowSnapshot.rebuilder(chain, db.clone(), &manifest).unwrap(); + let mut rebuilder = SNAPSHOT_MODE.rebuilder(chain, db.clone(), &manifest).unwrap(); match rebuilder.feed(&chunk, engine.as_ref(), &AtomicBool::new(false)) { Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {}