grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make ClientIoMessage generic over the Client (#10981)

Browse Source 
* Add client-traits crate
Move the BlockInfo trait to new crate

* New crate `machine`
Contains code extracted from ethcore that defines `Machine`, `Externalities` and other execution related code.

* Use new machine and client-traits crates in ethcore

* Use new crates machine and client-traits instead of ethcore where appropriate

* Fix tests

* Don't re-export so many types from ethcore::client

* Fixing more fallout from removing re-export

* fix test

* More fallout from not re-exporting types

* Add some docs

* cleanup

* import the macro edition style

* Tweak docs

* Add missing import

* remove unused ethabi_derive imports

* Use latest ethabi-contract

* Move many traits from ethcore/client/traits to client-traits crate
Initial version of extracted Engine trait

* Move snapshot related traits to the engine crate (eew)

* Move a few snapshot related types to common_types
Cleanup Executed as exported from machine crate

* fix warning

* Gradually introduce new engine crate: snapshot

* ethcore typechecks with new engine crate

* Sort out types outside ethcore

* Add an EpochVerifier to ethash and use that in Engine.epoch_verifier()
Cleanup

* Document pub members

* Sort out tests
Sort out default impls for EpochVerifier

* Add test-helpers feature and move EngineSigner impl to the right place

* Sort out tests

* Sort out tests and refactor verification types

* Fix missing traits

* More missing traits
Fix Histogram

* Fix tests and cleanup

* cleanup

* Put back needed logger import

* Don't rexport common_types from ethcore/src/client
Don't export ethcore::client::*

* Remove files no longer used
Use types from the engine crate
Explicit exports from engine::engine

* Get rid of itertools

* Move a few more traits from ethcore to client-traits: BlockChainReset, ScheduleInfo, StateClient

* Move ProvingBlockChainClient to client-traits

* Don't re-export ForkChoice and Transition from ethcore

* Address grumbles: sort imports, remove commented out code

* Fix merge resolution error

* Extract the Clique engine to own crate

* Extract NullEngine and the block_reward module from ethcore

* Extract InstantSeal engine to own crate

* Extract remaining engines

* Extract executive_state to own crate so it can be used by engine crates

* Remove snapshot stuff from the engine crate

* Put snapshot traits back in ethcore

* cleanup

* Remove stuff from ethcore

* Don't use itertools

* itertools in aura is legit-ish

* More post-merge fixes

* Re-export less types in client

* cleanup

* Extract spec to own crate

* Put back the test-helpers from basic-authority

* Fix ethcore benchmarks

* Reduce the public api of ethcore/verification

* WIP

* Add Cargo.toml

* Fix compilation outside ethcore

* Audit uses of import_verified_blocks() and remove unneeded calls
Cleanup

* cleanup

* Remove unused imports from ethcore

* Cleanup

* remove double semi-colons

* Add missing generic param

* More missing generics

* Update ethcore/block-reward/Cargo.toml

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update ethcore/engines/basic-authority/Cargo.toml

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update ethcore/engines/ethash/Cargo.toml

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update ethcore/engines/clique/src/lib.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* signers is already a ref

* Add an EngineType enum to tighten up Engine.name()

* Introduce Snapshotting enum to distinguish the type of snapshots a chain uses

* Rename supports_warp to snapshot_mode

* Missing import

* Update ethcore/src/snapshot/consensus/mod.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* missing import

* Fix import

* double semi

* Fix merge problem

* cleanup

* Parametrise `ClientIoMessage` with `()` for the light client

* Add impl Tick for ()

* Address review feedback

* Move ClientIoMessage to common-types

* remove superseeded fixme

* fix merge conflict errors
This commit is contained in:
David 2019-08-28 10:09:42 +02:00 committed by GitHub
parent 974b24549b
commit cd26526868
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 542 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Cargo.lock generated
View File

@ -1052,7 +1052,6 @@ dependencies = [
"kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"len-caching-lock 0.1.1",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"machine 0.1.0",
"macros 0.1.0",
@ -1062,7 +1061,6 @@ dependencies = [
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-runtime 0.1.0",
"parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"patricia-trie-ethereum 0.1.0",
"pod 0.1.0",
@ -1080,7 +1078,6 @@ dependencies = [
"state-db 0.1.0",
"stats 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"time-utils 0.1.0",
"trace 0.1.0",
"trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1089,6 +1086,7 @@ dependencies = [
"triehash-ethereum 0.2.0",
"unexpected 0.1.0",
"using_queue 0.1.0",
"verification 0.1.0",
"vm 0.1.0",
]
@ -1245,6 +1243,7 @@ dependencies = [
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
"triehash-ethereum 0.2.0",
"verification 0.1.0",
"vm 0.1.0",
]
@ -1451,6 +1450,7 @@ name = "ethcore-service"
version = "0.1.0"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"client-traits 0.1.0",
"common-types 0.1.0",
"ethcore 1.12.0",
"ethcore-blockchain 0.1.0",
@ -3021,6 +3021,7 @@ dependencies = [
"term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
"verification 0.1.0",
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -3171,6 +3172,7 @@ dependencies = [
"trace 0.1.0",
"transaction-pool 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"verification 0.1.0",
"vm 0.1.0",
]
@ -4947,6 +4949,35 @@ dependencies = [
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "verification"
version = "0.1.0"
dependencies = [
"client-traits 0.1.0",
"common-types 0.1.0",
"engine 0.1.0",
"ethcore 1.12.0",
"ethcore-blockchain 0.1.0",
"ethcore-call-contract 0.1.0",
"ethcore-io 1.12.0",
"ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ethkey 0.3.0",
"keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"len-caching-lock 0.1.1",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"machine 0.1.0",
"null-engine 0.1.0",
"num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-util-mem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"spec 0.1.0",
"time-utils 0.1.0",
"triehash-ethereum 0.2.0",
"unexpected 0.1.0",
]
[[package]]
name = "version_check"
version = "0.1.5"

1
Cargo.toml
View File

@ -73,6 +73,7 @@ spec = { path = "ethcore/spec" }
term_size = "0.3"
textwrap = "0.9"
toml = "0.4"
verification = { path = "ethcore/verification" }
[build-dependencies]
rustc_version = "0.2"

19
ethcore/Cargo.toml
View File

@ -35,16 +35,14 @@ evm = { path = "evm" }
executive-state = { path = "executive-state" }
futures = "0.1"
hash-db = "0.15.0"
parity-util-mem = "0.2.0"
itertools = "0.5"
journaldb = { path = "../util/journaldb" }
keccak-hash = "0.2.0"
keccak-hasher = { path = "../util/keccak-hasher" }
kvdb = "0.1"
kvdb-memorydb = "0.1"
kvdb-memorydb = { version = "0.1", optional = true }
kvdb-rocksdb = { version = "0.1.3", optional = true }
lazy_static = "1.3.0"
len-caching-lock = { path = "../util/len-caching-lock" }
lazy_static = { version = "1.3", optional = true }
log = "0.4"
macros = { path = "../util/macros", optional = true }
machine = { path = "./machine" }
@ -61,20 +59,19 @@ rand_xorshift = "0.1.1"
rayon = "1.1"
rlp = "0.4.0"
rlp_derive = { path = "../util/rlp-derive" }
rustc-hex = "1.0"
rustc-hex = { version = "1", optional = true }
serde = "1.0"
serde_derive = "1.0"
spec = { path = "spec" }
state-db = { path = "state-db" }
stats = { path = "../util/stats" }
tempdir = { version = "0.3", optional = true }
time-utils = { path = "../util/time-utils" }
trace = { path = "trace" }
trace-time = "0.1"
trie-vm-factories = { path = "trie-vm-factories" }
triehash-ethereum = { version = "0.2", path = "../util/triehash-ethereum" }
unexpected = { path = "../util/unexpected" }
using_queue = { path = "../miner/using-queue" }
verification = { path = "./verification" }
vm = { path = "vm" }
[dev-dependencies]
@ -88,14 +85,18 @@ ethcore-accounts = { path = "../accounts" }
ethjson = { path = "../json" }
ethkey = { path = "../accounts/ethkey" }
fetch = { path = "../util/fetch" }
kvdb-memorydb = "0.1"
kvdb-rocksdb = "0.1.3"
lazy_static = { version = "1.3" }
machine = { path = "./machine", features = ["test-helpers"] }
macros = { path = "../util/macros" }
null-engine = { path = "./engines/null-engine" }
parity-runtime = { path = "../util/runtime" }
pod = { path = "pod" }
rlp_compress = { path = "../util/rlp-compress" }
rustc-hex = "1"
serde_json = "1.0"
stats = { path = "../util/stats" }
tempdir = "0.3"
trie-standardmap = "0.15.0"
@ -121,14 +122,14 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
# EVM debug traces are printed.
slow-blocks = []
# Run JSON consensus tests.
json-tests = ["env_logger", "test-helpers", "machine/test-helpers"]
json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers"]
# Skip JSON consensus tests with pending issues.
ci-skip-tests = []
# Run memory/cpu heavy tests.
test-heavy = []
# Compile test helpers
# note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works
test-helpers = ["tempdir", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "basic-authority/test-helpers"]
test-helpers = ["tempdir", "kvdb-memorydb", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "rustc-hex", "basic-authority/test-helpers"]
[[bench]]
name = "builtin"

55
ethcore/client-traits/src/lib.rs
View File

@ -32,7 +32,7 @@ use common_types::{
client_types::Mode,
encoded,
engines::{epoch::Transition as EpochTransition, machine::Executed},
errors::EthcoreResult,
errors::{EthcoreError, EthcoreResult},
filter::Filter,
header::Header,
ids::{BlockId, TransactionId, TraceId, UncleId},
@ -56,6 +56,8 @@ use trace::{
};
use vm::{LastHashes, Schedule};
use common_types::snapshot::Progress;
/// State information to be used during client query
pub enum StateOrBlock {
/// State to be used, may be pending
@ -168,11 +170,14 @@ pub trait EngineClient: Sync + Send + ChainInfo {
fn block_header(&self, id: BlockId) -> Option<encoded::Header>;
}
// FIXME Why these methods belong to BlockChainClient and not MiningBlockChainClient?
/// Provides methods to import block into blockchain
pub trait ImportBlock {
/// Import a block into the blockchain.
fn import_block(&self, block: Unverified) -> EthcoreResult<H256>;
/// Triggered by a message from a block queue when the block is ready for insertion.
/// Returns the number of blocks imported.
fn import_verified_blocks(&self) -> usize;
}
/// IO operations that should off-load heavy work to another thread.
@ -187,6 +192,14 @@ pub trait IoClient: Sync + Send {
fn queue_consensus_message(&self, message: Bytes);
}
/// Implement this for clients that need logic to decide when/how to advance.
pub trait Tick {
/// Tick the client
fn tick(&self, _prevent_sleep: bool) {}
}
impl Tick for () {}
/// Provides recently seen bad blocks.
pub trait BadBlocks {
/// Returns a list of blocks that were recently not imported because they were invalid.
@ -377,6 +390,9 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
pub trait BlockChainReset {
/// reset to best_block - n
fn reset(&self, num: u32) -> Result<(), String>;
/// Number of eras kept in a journal before they are pruned
fn pruning_history(&self) -> u64;
}
@ -423,3 +439,38 @@ pub trait ProvingBlockChainClient: BlockChainClient {
/// Get an epoch change signal by block hash.
fn epoch_signal(&self, hash: H256) -> Option<Vec<u8>>;
}
/// External database restoration handler
pub trait DatabaseRestore: Send + Sync {
/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError>;
}
/// Snapshot related functionality
pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
/// Take a snapshot at the given block.
/// If the ID given is "latest", this will default to 1000 blocks behind.
fn take_snapshot<W: SnapshotWriter + Send>(
&self,
writer: W,
at: BlockId,
p: &Progress,
) -> Result<(), EthcoreError>;
}
// todo[dvdplm] move this back to snapshot once extracted from ethcore
/// Something which can write snapshots.
/// Writing the same chunk multiple times will lead to implementation-defined
/// behavior, and is not advised.
pub trait SnapshotWriter {
/// Write a compressed state chunk.
fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
/// Write a compressed block chunk.
fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
/// Complete writing. The manifest's chunk lists must be consistent
/// with the chunks written.
fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized;
}

2
ethcore/light/Cargo.toml
View File

@ -13,7 +13,6 @@ client-traits = { path = "../client-traits" }
common-types = { path = "../types" }
derive_more = "0.14.0"
engine = { path = "../engine" }
ethcore = { path = ".."}
ethcore-db = { path = "../db" }
ethcore-blockchain = { path = "../blockchain" }
ethereum-types = "0.6.0"
@ -49,6 +48,7 @@ kvdb = "0.1"
memory-cache = { path = "../../util/memory-cache" }
error-chain = { version = "0.12", default-features = false }
journaldb = { path = "../../util/journaldb" }
verification = { path = "../verification" }
[dev-dependencies]
ethcore = { path = "..", features = ["test-helpers"] }

11
ethcore/light/src/client/mod.rs
View File

@ -19,8 +19,7 @@
use std::sync::{Weak, Arc};
use engine::{Engine, EpochChange, Proof};
use ethcore::client::{ClientReport, ClientIoMessage};
use ethcore::verification::queue::{self, HeaderQueue};
use verification::queue::{self, HeaderQueue};
use spec::{Spec, SpecHardcodedSync};
use io::IoChannel;
use parking_lot::{Mutex, RwLock};
@ -30,12 +29,14 @@ use common_types::{
BlockNumber,
block_status::BlockStatus,
blockchain_info::BlockChainInfo,
client_types::ClientReport,
encoded,
engines::epoch::{Transition as EpochTransition, PendingTransition},
errors::EthcoreError as Error,
errors::EthcoreResult,
header::Header,
ids::BlockId,
io_message::ClientIoMessage,
verification::VerificationQueueInfo as BlockQueueInfo,
};
use kvdb::KeyValueDB;
@ -162,7 +163,7 @@ impl<T: LightChainClient> AsLightClient for T {
/// Light client implementation.
pub struct Client<T> {
queue: HeaderQueue,
queue: HeaderQueue<()>,
engine: Arc<dyn Engine>,
chain: HeaderChain,
report: RwLock<ClientReport>,
@ -183,7 +184,7 @@ impl<T: ChainDataFetcher> Client<T> {
chain_col: Option<u32>,
spec: &Spec,
fetcher: T,
io_channel: IoChannel<ClientIoMessage>,
io_channel: IoChannel<ClientIoMessage<()>>,
cache: Arc<Mutex<Cache>>
) -> Result<Self, Error> {
Ok(Self {
@ -649,3 +650,5 @@ impl<T: ChainDataFetcher> client_traits::EngineClient for Client<T> {
Client::block_header(self, id)
}
}
impl<T> client_traits::Tick for Client<T> {}

18
ethcore/light/src/client/service.rs
View File

@ -20,10 +20,12 @@
use std::fmt;
use std::sync::Arc;
use common_types::errors::EthcoreError as CoreError;
use common_types::{
errors::EthcoreError as CoreError,
io_message::ClientIoMessage,
};
use ethcore_db as db;
use ethcore_blockchain::BlockChainDB;
use ethcore::client::ClientIoMessage;
use spec::Spec;
use io::{IoContext, IoError, IoHandler, IoService};
@ -58,15 +60,15 @@ impl fmt::Display for Error {
}
/// Light client service.
pub struct Service<T> {
pub struct Service<T: 'static> {
client: Arc<Client<T>>,
io_service: IoService<ClientIoMessage>,
io_service: IoService<ClientIoMessage<()>>,
}
impl<T: ChainDataFetcher> Service<T> {
/// Start the service: initialize I/O workers and client itself.
pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, db: Arc<dyn BlockChainDB>, cache: Arc<Mutex<Cache>>) -> Result<Self, Error> {
let io_service = IoService::<ClientIoMessage>::start().map_err(Error::Io)?;
let io_service = IoService::<ClientIoMessage<()>>::start().map_err(Error::Io)?;
let client = Arc::new(Client::new(config,
db.key_value().clone(),
db::COL_LIGHT_CHAIN,
@ -90,7 +92,7 @@ impl<T: ChainDataFetcher> Service<T> {
}
/// Register an I/O handler on the service.
pub fn register_handler(&self, handler: Arc<dyn IoHandler<ClientIoMessage> + Send>) -> Result<(), IoError> {
pub fn register_handler(&self, handler: Arc<dyn IoHandler<ClientIoMessage<()>> + Send>) -> Result<(), IoError> {
self.io_service.register_handler(handler)
}
@ -102,8 +104,8 @@ impl<T: ChainDataFetcher> Service<T> {
struct ImportBlocks<T>(Arc<Client<T>>);
impl<T: ChainDataFetcher> IoHandler<ClientIoMessage> for ImportBlocks<T> {
fn message(&self, _io: &IoContext<ClientIoMessage>, message: &ClientIoMessage) {
impl<T: ChainDataFetcher> IoHandler<ClientIoMessage<()>> for ImportBlocks<T> {
fn message(&self, _io: &IoContext<ClientIoMessage<()>>, message: &ClientIoMessage<()>) {
if let ClientIoMessage::BlockVerified = *message {
self.0.import_verified();
}

4
ethcore/light/src/lib.rs
View File

@ -65,7 +65,6 @@ extern crate executive_state;
extern crate parity_bytes as bytes;
extern crate ethereum_types;
extern crate ethcore_miner as miner;
extern crate ethcore;
extern crate hash_db;
extern crate parity_util_mem;
extern crate parity_util_mem as mem;
@ -94,7 +93,10 @@ extern crate triehash_ethereum as triehash;
extern crate kvdb;
extern crate memory_cache;
extern crate derive_more;
extern crate verification;
#[cfg(test)]
extern crate ethcore;
#[cfg(test)]
extern crate kvdb_memorydb;
#[cfg(test)]

13
ethcore/private-tx/src/lib.rs
View File

@ -104,11 +104,12 @@ use machine::{
};
use types::{
ids::BlockId,
io_message::ClientIoMessage,
transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction},
engines::machine::Executed,
};
use ethcore::client::{
Client, ChainNotify, NewBlocks, ChainMessageType, ClientIoMessage, Call
Client, ChainNotify, NewBlocks, ChainMessageType, Call
};
use client_traits::BlockInfo;
use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache};
@ -213,7 +214,7 @@ pub struct Provider {
client: Arc<Client>,
miner: Arc<Miner>,
accounts: Arc<dyn Signer>,
channel: IoChannel<ClientIoMessage>,
channel: IoChannel<ClientIoMessage<Client>>,
keys_provider: Arc<dyn KeyProvider>,
logging: Option<Logging>,
use_offchain_storage: bool,
@ -236,7 +237,7 @@ impl Provider {
accounts: Arc<dyn Signer>,
encryptor: Box<dyn Encryptor>,
config: ProviderConfig,
channel: IoChannel<ClientIoMessage>,
channel: IoChannel<ClientIoMessage<Client>>,
keys_provider: Arc<dyn KeyProvider>,
db: Arc<dyn KeyValueDB>,
) -> Self {
@ -876,14 +877,14 @@ impl Provider {
}
}
impl IoHandler<ClientIoMessage> for Provider {
fn initialize(&self, io: &IoContext<ClientIoMessage>) {
impl IoHandler<ClientIoMessage<Client>> for Provider {
fn initialize(&self, io: &IoContext<ClientIoMessage<Client>>) {
if self.use_offchain_storage {
io.register_timer(STATE_RETRIEVAL_TIMER, STATE_RETRIEVAL_TICK).expect("Error registering state retrieval timer");
}
}
fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
fn timeout(&self, _io: &IoContext<ClientIoMessage<Client>>, timer: TimerToken) {
match timer {
STATE_RETRIEVAL_TIMER => self.state_storage.tick(&self.logging),
_ => warn!("IO service triggered unregistered timer '{}'", timer),

1
ethcore/service/Cargo.toml
View File

@ -7,6 +7,7 @@ authors = ["Parity Technologies <admin@parity.io>"]
[dependencies]
ansi_term = "0.11"
common-types = { path = "../types" }
client-traits = { path = "../client-traits" }
ethcore = { path = ".." }
ethcore-blockchain = { path = "../blockchain" }
ethcore-io = { path = "../../util/io" }

1
ethcore/service/src/lib.rs
View File

@ -16,6 +16,7 @@
extern crate ansi_term;
extern crate common_types;
extern crate client_traits;
extern crate ethcore;
extern crate ethcore_blockchain as blockchain;
extern crate ethcore_io as io;

31
ethcore/service/src/service.rs
View File

@ -26,15 +26,17 @@ use io::{IoContext, TimerToken, IoHandler, IoService, IoError};
use sync::PrivateTxHandler;
use blockchain::{BlockChainDB, BlockChainDBHandler};
use ethcore::client::{Client, ClientConfig, ChainNotify, ClientIoMessage};
use ethcore::client::{Client, ClientConfig, ChainNotify};
use ethcore::miner::Miner;
use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams};
use ethcore::snapshot::{SnapshotService as _SnapshotService};
use spec::Spec;
use common_types::{
io_message::ClientIoMessage,
errors::{EthcoreError, SnapshotError},
snapshot::RestorationStatus,
};
use client_traits::{ImportBlock, SnapshotClient, Tick};
use ethcore_private_tx::{self, Importer, Signer};
@ -90,7 +92,7 @@ impl PrivateTxHandler for PrivateTxService {
/// Client service setup. Creates and registers client and network services with the IO subsystem.
pub struct ClientService {
io_service: Arc<IoService<ClientIoMessage>>,
io_service: Arc<IoService<ClientIoMessage<Client>>>,
client: Arc<Client>,
snapshot: Arc<SnapshotService>,
private_tx: Arc<PrivateTxService>,
@ -113,7 +115,7 @@ impl ClientService {
private_encryptor_conf: ethcore_private_tx::EncryptorConfig,
) -> Result<ClientService, EthcoreError>
{
let io_service = IoService::<ClientIoMessage>::start()?;
let io_service = IoService::<ClientIoMessage<Client>>::start()?;
info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name().to_string()));
@ -165,15 +167,15 @@ impl ClientService {
Ok(ClientService {
io_service: Arc::new(io_service),
client: client,
snapshot: snapshot,
client,
snapshot,
private_tx,
database: blockchain_db,
})
}
/// Get general IO interface
pub fn register_io_handler(&self, handler: Arc<dyn IoHandler<ClientIoMessage> + Send>) -> Result<(), IoError> {
pub fn register_io_handler(&self, handler: Arc<dyn IoHandler<ClientIoMessage<Client>> + Send>) -> Result<(), IoError> {
self.io_service.register_handler(handler)
}
@ -193,7 +195,7 @@ impl ClientService {
}
/// Get network service component
pub fn io(&self) -> Arc<IoService<ClientIoMessage>> {
pub fn io(&self) -> Arc<IoService<ClientIoMessage<Client>>> {
self.io_service.clone()
}
@ -213,8 +215,8 @@ impl ClientService {
}
/// IO interface for the Client handler
struct ClientIoHandler {
client: Arc<Client>,
struct ClientIoHandler<C> {
client: Arc<C>,
snapshot: Arc<SnapshotService>,
}
@ -224,13 +226,16 @@ const SNAPSHOT_TICK_TIMER: TimerToken = 1;
const CLIENT_TICK: Duration = Duration::from_secs(5);
const SNAPSHOT_TICK: Duration = Duration::from_secs(10);
impl IoHandler<ClientIoMessage> for ClientIoHandler {
fn initialize(&self, io: &IoContext<ClientIoMessage>) {
impl<C> IoHandler<ClientIoMessage<C>> for ClientIoHandler<C>
where
C: ImportBlock + SnapshotClient + Tick + 'static,
{
fn initialize(&self, io: &IoContext<ClientIoMessage<C>>) {
io.register_timer(CLIENT_TICK_TIMER, CLIENT_TICK).expect("Error registering client timer");
io.register_timer(SNAPSHOT_TICK_TIMER, SNAPSHOT_TICK).expect("Error registering snapshot timer");
}
fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
fn timeout(&self, _io: &IoContext<ClientIoMessage<C>>, timer: TimerToken) {
trace_time!("service::read");
match timer {
CLIENT_TICK_TIMER => {
@ -243,7 +248,7 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
}
}
fn message(&self, _io: &IoContext<ClientIoMessage>, net_message: &ClientIoMessage) {
fn message(&self, _io: &IoContext<ClientIoMessage<C>>, net_message: &ClientIoMessage<C>) {
trace_time!("service::message");
use std::thread;

214
ethcore/src/client/client.rs
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
use std::{cmp, ops};
use std::cmp;
use std::collections::{HashSet, BTreeMap, VecDeque};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
use std::sync::{Arc, Weak};
@ -42,14 +42,15 @@ use client::ancient_import::AncientVerifier;
use client::{
ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock,
Call, BlockProducer, SealedBlockImporter, ChainNotify, EngineInfo,
ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks, ClientIoMessage,
ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks,
};
use client_traits::{
BlockInfo, ScheduleInfo, StateClient, BlockChainReset,
Nonce, Balance, ChainInfo, TransactionInfo, ImportBlock,
AccountData, BlockChain as BlockChainTrait, BlockChainClient,
IoClient, BadBlocks, ProvingBlockChainClient,
StateOrBlock
IoClient, BadBlocks, ProvingBlockChainClient, SnapshotClient,
DatabaseRestore, SnapshotWriter, Tick,
StateOrBlock,
};
use engine::Engine;
use machine::{
@ -59,7 +60,7 @@ use machine::{
};
use trie_vm_factories::{Factories, VmFactory};
use miner::{Miner, MinerService};
use snapshot::{self, io as snapshot_io, SnapshotClient};
use snapshot;
use spec::Spec;
use account_state::State;
use executive_state;
@ -71,6 +72,8 @@ use types::{
block::PreverifiedBlock,
block_status::BlockStatus,
blockchain_info::BlockChainInfo,
client_types::ClientReport,
io_message::ClientIoMessage,
encoded,
engines::{
ForkChoice,
@ -111,44 +114,6 @@ const MAX_ANCIENT_BLOCKS_TO_IMPORT: usize = 4;
const MAX_QUEUE_SIZE_TO_SLEEP_ON: usize = 2;
const MIN_HISTORY_SIZE: u64 = 8;
/// Report on the status of a client.
#[derive(Default, Clone, Debug, Eq, PartialEq)]
pub struct ClientReport {
/// How many blocks have been imported so far.
pub blocks_imported: usize,
/// How many transactions have been applied so far.
pub transactions_applied: usize,
/// How much gas has been processed so far.
pub gas_processed: U256,
/// Memory used by state DB
pub state_db_mem: usize,
}
impl ClientReport {
/// Alter internal reporting to reflect the additional `block` has been processed.
pub fn accrue_block(&mut self, header: &Header, transactions: usize) {
self.blocks_imported += 1;
self.transactions_applied += transactions;
self.gas_processed = self.gas_processed + *header.gas_used();
}
}
impl<'a> ops::Sub<&'a ClientReport> for ClientReport {
type Output = Self;
fn sub(mut self, other: &'a ClientReport) -> Self {
let higher_mem = cmp::max(self.state_db_mem, other.state_db_mem);
let lower_mem = cmp::min(self.state_db_mem, other.state_db_mem);
self.blocks_imported -= other.blocks_imported;
self.transactions_applied -= other.transactions_applied;
self.gas_processed = self.gas_processed - other.gas_processed;
self.state_db_mem = higher_mem - lower_mem;
self
}
}
struct SleepState {
last_activity: Option<Instant>,
last_autosleep: Option<Instant>,
@ -171,7 +136,7 @@ struct Importer {
pub verifier: Box<dyn Verifier<Client>>,
/// Queue containing pending blocks
pub block_queue: BlockQueue,
pub block_queue: BlockQueue<Client>,
/// Handles block sealing
pub miner: Arc<Miner>,
@ -222,7 +187,7 @@ pub struct Client {
/// Flag changed by `sleep` and `wake_up` methods. Not to be confused with `enabled`.
liveness: AtomicBool,
io_channel: RwLock<IoChannel<ClientIoMessage>>,
io_channel: RwLock<IoChannel<ClientIoMessage<Self>>>,
/// List of actors to be notified on certain chain events
notify: RwLock<Vec<Weak<dyn ChainNotify>>>,
@ -261,7 +226,7 @@ impl Importer {
pub fn new(
config: &ClientConfig,
engine: Arc<dyn Engine>,
message_channel: IoChannel<ClientIoMessage>,
message_channel: IoChannel<ClientIoMessage<Client>>,
miner: Arc<Miner>,
) -> Result<Importer, EthcoreError> {
let block_queue = BlockQueue::new(
@ -723,7 +688,7 @@ impl Client {
spec: &Spec,
db: Arc<dyn BlockChainDB>,
miner: Arc<Miner>,
message_channel: IoChannel<ClientIoMessage>,
message_channel: IoChannel<ClientIoMessage<Self>>,
) -> Result<Arc<Client>, EthcoreError> {
let trie_spec = match config.fat_db {
true => TrieSpec::Fat,
@ -948,11 +913,6 @@ impl Client {
Arc::new(last_hashes)
}
/// This is triggered by a message coming from a block queue when the block is ready for insertion
pub fn import_verified_blocks(&self) -> usize {
self.importer.import_verified_blocks(self)
}
// use a state-proving closure for the given block.
fn with_proving_caller<F, T>(&self, id: BlockId, with_call: F) -> T
where F: FnOnce(&MachineCall) -> T
@ -1037,7 +997,7 @@ impl Client {
}
/// Replace io channel. Useful for testing.
pub fn set_io_channel(&self, io_channel: IoChannel<ClientIoMessage>) {
pub fn set_io_channel(&self, io_channel: IoChannel<ClientIoMessage<Self>>) {
*self.io_channel.write() = io_channel;
}
@ -1112,15 +1072,6 @@ impl Client {
report
}
/// Tick the client.
// TODO: manage by real events.
pub fn tick(&self, prevent_sleep: bool) {
self.check_garbage();
if !prevent_sleep {
self.check_snooze();
}
}
fn check_garbage(&self) {
self.chain.read().collect_garbage();
self.importer.block_queue.collect_garbage();
@ -1161,64 +1112,6 @@ impl Client {
}
}
/// Take a snapshot at the given block.
/// If the ID given is "latest", this will default to 1000 blocks behind.
pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(
&self,
writer: W,
at: BlockId,
p: &Progress,
) -> Result<(), EthcoreError> {
if let Snapshotting::Unsupported = self.engine.snapshot_mode() {
return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported));
}
let db = self.state_db.read().journal_db().boxed_clone();
let best_block_number = self.chain_info().best_block_number;
let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
if db.is_prunable() && self.pruning_info().earliest_state > block_number {
return Err(SnapshotError::OldBlockPrunedDB.into());
}
let history = cmp::min(self.history, 1000);
let start_hash = match at {
BlockId::Latest => {
let start_num = match db.earliest_era() {
Some(era) => cmp::max(era, best_block_number.saturating_sub(history)),
None => best_block_number.saturating_sub(history),
};
match self.block_hash(BlockId::Number(start_num)) {
Some(h) => h,
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
}
}
_ => match self.block_hash(at) {
Some(hash) => hash,
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
},
};
let processing_threads = self.config.snapshot.processing_threads;
let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
snapshot::take_snapshot(
chunker,
&self.chain.read(),
start_hash,
db.as_hash_db(),
writer,
p,
processing_threads,
)?;
Ok(())
}
/// Ask the client what the history parameter is.
pub fn pruning_history(&self) -> u64 {
self.history
}
fn block_hash(chain: &BlockChain, id: BlockId) -> Option<H256> {
match id {
BlockId::Hash(hash) => Some(hash),
@ -1342,7 +1235,7 @@ impl Client {
}
}
impl snapshot::DatabaseRestore for Client {
impl DatabaseRestore for Client {
/// Restart the client with a new backend
fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError> {
trace!(target: "snapshot", "Replacing client database with {:?}", new_db);
@ -1426,6 +1319,11 @@ impl BlockChainReset for Client {
Ok(())
}
/// Ask the client what the history parameter is.
fn pruning_history(&self) -> u64 {
self.history
}
}
impl Nonce for Client {
@ -1547,6 +1445,11 @@ impl ImportBlock for Client {
Err((_, e)) => Err(e),
}
}
/// Triggered by a message from a block queue when the block is ready for insertion
fn import_verified_blocks(&self) -> usize {
self.importer.import_verified_blocks(self)
}
}
impl StateClient for Client {
@ -2341,6 +2244,18 @@ impl IoClient for Client {
}
}
}
}
impl Tick for Client {
/// Tick the client.
// TODO: manage by real events.
fn tick(&self, prevent_sleep: bool) {
self.check_garbage();
if !prevent_sleep {
self.check_snooze();
}
}
}
impl ReopenBlock for Client {
@ -2581,7 +2496,60 @@ impl ProvingBlockChainClient for Client {
}
}
impl SnapshotClient for Client {}
impl SnapshotClient for Client {
fn take_snapshot<W: SnapshotWriter + Send>(
&self,
writer: W,
at: BlockId,
p: &Progress,
) -> Result<(), EthcoreError> {
if let Snapshotting::Unsupported = self.engine.snapshot_mode() {
return Err(EthcoreError::Snapshot(SnapshotError::SnapshotsUnsupported));
}
let db = self.state_db.read().journal_db().boxed_clone();
let best_block_number = self.chain_info().best_block_number;
let block_number = self.block_number(at).ok_or_else(|| SnapshotError::InvalidStartingBlock(at))?;
if db.is_prunable() && self.pruning_info().earliest_state > block_number {
return Err(SnapshotError::OldBlockPrunedDB.into());
}
let history = cmp::min(self.history, 1000);
let start_hash = match at {
BlockId::Latest => {
let start_num = match db.earliest_era() {
Some(era) => cmp::max(era, best_block_number.saturating_sub(history)),
None => best_block_number.saturating_sub(history),
};
match self.block_hash(BlockId::Number(start_num)) {
Some(h) => h,
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
}
}
_ => match self.block_hash(at) {
Some(hash) => hash,
None => return Err(SnapshotError::InvalidStartingBlock(at).into()),
},
};
let processing_threads = self.config.snapshot.processing_threads;
let chunker = snapshot::chunker(self.engine.snapshot_mode()).ok_or_else(|| SnapshotError::SnapshotsUnsupported)?;
snapshot::take_snapshot(
chunker,
&self.chain.read(),
start_hash,
db.as_hash_db(),
writer,
p,
processing_threads,
)?;
Ok(())
}
}
/// Returns `LocalizedReceipt` given `LocalizedTransaction`
/// and a vector of receipts from given block up to transaction index.
@ -2641,7 +2609,7 @@ impl IoChannelQueue {
}
}
pub fn queue<F>(&self, channel: &IoChannel<ClientIoMessage>, count: usize, fun: F) -> EthcoreResult<()> where
pub fn queue<F>(&self, channel: &IoChannel<ClientIoMessage<Client>>, count: usize, fun: F) -> EthcoreResult<()> where
F: Fn(&Client) + Send + Sync + 'static,
{
let queue_size = self.currently_queued.load(AtomicOrdering::Relaxed);

4
ethcore/src/client/mod.rs
View File

@ -22,15 +22,13 @@ mod client;
mod config;
#[cfg(any(test, feature = "test-helpers"))]
mod evm_test_client;
mod io_message;
#[cfg(any(test, feature = "test-helpers"))]
mod test_client;
pub use self::client::{Client, ClientReport};
pub use self::client::Client;
pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType};
#[cfg(any(test, feature = "test-helpers"))]
pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess};
pub use self::io_message::ClientIoMessage;
#[cfg(any(test, feature = "test-helpers"))]
pub use self::test_client::{TestBlockChainClient, EachBlockWith, TestState};
pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType};

4
ethcore/src/client/test_client.rs
View File

@ -592,6 +592,10 @@ impl ImportBlock for TestBlockChainClient {
}
Ok(h)
}
fn import_verified_blocks(&self) -> usize {
unimplemented!("TestClient does not implement import_verified_blocks()")
}
}
impl Call for TestBlockChainClient {

1
ethcore/src/json_tests/chain.rs
View File

@ -100,7 +100,6 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
if let Ok(block) = Unverified::from_rlp(b) {
let _ = client.import_block(block);
client.flush_queue();
client.import_verified_blocks();
}
}
fail_unless(client.chain_info().best_block_hash == blockchain.best_block.into());

21
ethcore/src/lib.rs
View File

@ -69,7 +69,6 @@ extern crate ethcore_io as io;
extern crate ethcore_miner;
extern crate ethereum_types;
extern crate executive_state;
extern crate trie_vm_factories;
extern crate futures;
extern crate hash_db;
extern crate itertools;
@ -77,10 +76,6 @@ extern crate journaldb;
extern crate keccak_hash as hash;
extern crate keccak_hasher;
extern crate kvdb;
#[cfg(any(test, feature = "test-helpers"))]
extern crate kvdb_memorydb;
extern crate len_caching_lock;
extern crate machine;
extern crate memory_cache;
extern crate num_cpus;
@ -92,18 +87,15 @@ extern crate patricia_trie_ethereum as ethtrie;
extern crate rand;
extern crate rayon;
extern crate rlp;
extern crate parity_util_mem;
extern crate parity_util_mem as malloc_size_of;
#[cfg(any(test, feature = "test-helpers"))]
extern crate rustc_hex;
extern crate serde;
extern crate spec;
extern crate state_db;
extern crate time_utils;
extern crate trace;
extern crate trie_vm_factories;
extern crate triehash_ethereum as triehash;
extern crate unexpected;
extern crate using_queue;
extern crate verification;
extern crate vm;
#[cfg(test)]
@ -119,8 +111,8 @@ extern crate ethash;
extern crate ethkey;
#[cfg(any(test, feature = "test-helpers"))]
extern crate ethjson;
#[cfg(any(test, feature = "tempdir"))]
extern crate tempdir;
#[cfg(any(test, feature = "test-helpers"))]
extern crate kvdb_memorydb;
#[cfg(any(test, feature = "kvdb-rocksdb"))]
extern crate kvdb_rocksdb;
#[cfg(any(test, feature = "json-tests"))]
@ -137,8 +129,12 @@ extern crate pod;
extern crate blooms_db;
#[cfg(any(test, feature = "env_logger"))]
extern crate env_logger;
#[cfg(any(test, feature = "test-helpers"))]
extern crate rustc_hex;
#[cfg(test)]
extern crate serde_json;
#[cfg(any(test, feature = "tempdir"))]
extern crate tempdir;
#[macro_use]
extern crate ethabi_contract;
@ -162,7 +158,6 @@ pub mod block;
pub mod client;
pub mod miner;
pub mod snapshot;
pub mod verification;
#[cfg(test)]
mod tests;

36
ethcore/src/miner/miner.rs
View File

@ -36,36 +36,30 @@ use miner::pool_client::{PoolClient, CachedNonceClient, NonceCache};
use miner;
use parking_lot::{Mutex, RwLock};
use rayon::prelude::*;
use types::transaction::{
self,
Action,
UnverifiedTransaction,
SignedTransaction,
PendingTransaction,
};
use types::{
BlockNumber,
ids::TransactionId,
block::Block,
header::Header,
ids::BlockId,
io_message::ClientIoMessage,
engines::{Seal, SealingState},
errors::{EthcoreError as Error, ExecutionError},
receipt::RichReceipt,
transaction::{
self,
Action,
UnverifiedTransaction,
SignedTransaction,
PendingTransaction,
},
};
use using_queue::{UsingQueue, GetAction};
use block::{ClosedBlock, SealedBlock};
use client::{
BlockProducer, SealedBlockImporter, ClientIoMessage,
};
use client_traits::{
BlockChain, ChainInfo, Nonce, TransactionInfo,
};
use engine::{
Engine,
signer::EngineSigner
};
use client::{BlockProducer, SealedBlockImporter, Client};
use client_traits::{BlockChain, ChainInfo, Nonce, TransactionInfo};
use engine::{Engine, signer::EngineSigner};
use machine::executive::contract_address;
use spec::Spec;
use account_state::State;
@ -262,7 +256,7 @@ pub struct Miner {
transaction_queue: Arc<TransactionQueue>,
engine: Arc<dyn Engine>,
accounts: Arc<dyn LocalAccounts>,
io_channel: RwLock<Option<IoChannel<ClientIoMessage>>>,
io_channel: RwLock<Option<IoChannel<ClientIoMessage<Client>>>>,
service_transaction_checker: Option<ServiceTransactionChecker>,
}
@ -346,7 +340,7 @@ impl Miner {
}
/// Sets `IoChannel`
pub fn set_io_channel(&self, io_channel: IoChannel<ClientIoMessage>) {
pub fn set_io_channel(&self, io_channel: IoChannel<ClientIoMessage<Client>>) {
*self.io_channel.write() = Some(io_channel);
}
@ -1426,7 +1420,7 @@ impl miner::MinerService for Miner {
let accounts = self.accounts.clone();
let service_transaction_checker = self.service_transaction_checker.clone();
let cull = move |chain: &::client::Client| {
let cull = move |chain: &Client| {
let client = PoolClient::new(
chain,
&nonce_cache,
@ -1437,7 +1431,7 @@ impl miner::MinerService for Miner {
queue.cull(client);
};
if let Err(e) = channel.send(ClientIoMessage::execute(cull)) {
if let Err(e) = channel.send(ClientIoMessage::<Client>::execute(cull)) {
warn!(target: "miner", "Error queueing cull: {:?}", e);
}
} else {

15
ethcore/src/snapshot/io.rs
View File

@ -26,6 +26,7 @@ use std::fs::{self, File};
use std::path::{Path, PathBuf};
use bytes::Bytes;
use client_traits::SnapshotWriter;
use ethereum_types::H256;
use rlp::{RlpStream, Rlp};
use types::{
@ -35,20 +36,6 @@ use types::{
const SNAPSHOT_VERSION: u64 = 2;
/// Something which can write snapshots.
/// Writing the same chunk multiple times will lead to implementation-defined
/// behavior, and is not advised.
pub trait SnapshotWriter {
/// Write a compressed state chunk.
fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> io::Result<()>;
/// Write a compressed block chunk.
fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> io::Result<()>;
/// Complete writing. The manifest's chunk lists must be consistent
/// with the chunks written.
fn finish(self, manifest: ManifestData) -> io::Result<()> where Self: Sized;
}
// (hash, len, offset)
#[derive(RlpEncodable, RlpDecodable)]

5
ethcore/src/snapshot/mod.rs
View File

@ -48,7 +48,8 @@ use bloom_journal::Bloom;
use num_cpus;
use types::snapshot::ManifestData;
use self::io::SnapshotWriter;
// todo[dvdplm] put back in snapshots once it's extracted
use client_traits::SnapshotWriter;
use super::state_db::StateDB;
use account_state::Account as StateAccount;
@ -58,7 +59,7 @@ use crossbeam_utils::thread;
use rand::{Rng, rngs::OsRng};
pub use self::consensus::*;
pub use self::service::{SnapshotClient, Service, DatabaseRestore};
pub use self::service::Service;
pub use self::traits::{SnapshotService, SnapshotComponents, Rebuilder};
pub use self::watcher::Watcher;
pub use types::basic_account::BasicAccount;

42
ethcore/src/snapshot/service.rs
View File

@ -29,16 +29,21 @@ use super::{
SnapshotService,
Rebuilder,
MAX_CHUNK_SIZE,
io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter},
io::{SnapshotReader, LooseReader, LooseWriter},
chunker,
};
use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler};
use client::{Client, ClientIoMessage};
use client_traits::{BlockInfo, BlockChainClient, ChainInfo};
use client::Client;
// todo[dvdplm] put SnapshotWriter back in snapshots once extracted
use client_traits::{
BlockInfo, BlockChainClient, ChainInfo,
SnapshotClient, SnapshotWriter, DatabaseRestore,
};
use engine::Engine;
use hash::keccak;
use types::{
io_message::ClientIoMessage,
errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain},
ids::BlockId,
snapshot::{ManifestData, Progress, RestorationStatus},
@ -73,12 +78,6 @@ impl Drop for Guard {
}
}
/// External database restoration handler
pub trait DatabaseRestore: Send + Sync {
/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
fn restore_db(&self, new_db: &str) -> Result<(), Error>;
}
/// State restoration manager.
struct Restoration {
manifest: ManifestData,
@ -213,10 +212,7 @@ impl Restoration {
}
/// Type alias for client io channel.
pub type Channel = IoChannel<ClientIoMessage>;
/// Trait alias for the Client Service used
pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore {}
pub type Channel = IoChannel<ClientIoMessage<Client>>;
/// Snapshot service parameters.
pub struct ServiceParams {
@ -234,7 +230,7 @@ pub struct ServiceParams {
/// Usually "<chain hash>/snapshot"
pub snapshot_root: PathBuf,
/// A handle for database restoration.
pub client: Arc<dyn SnapshotClient>,
pub client: Arc<Client>,
}
/// `SnapshotService` implementation.
@ -251,7 +247,7 @@ pub struct Service {
genesis_block: Bytes,
state_chunks: AtomicUsize,
block_chunks: AtomicUsize,
client: Arc<dyn SnapshotClient>,
client: Arc<Client>,
progress: Progress,
taking_snapshot: AtomicBool,
restoring_snapshot: AtomicBool,
@ -483,7 +479,10 @@ impl Service {
/// calling this while a restoration is in progress or vice versa
/// will lead to a race condition where the first one to finish will
/// have their produced snapshot overwritten.
pub fn take_snapshot(&self, client: &Client, num: u64) -> Result<(), Error> {
pub fn take_snapshot<C>(&self, client: &C, num: u64) -> Result<(), Error>
where
C: ChainInfo + SnapshotClient
{
if self.taking_snapshot.compare_and_swap(false, true, Ordering::SeqCst) {
info!("Skipping snapshot at #{} as another one is currently in-progress.", num);
return Ok(());
@ -905,13 +904,16 @@ impl Drop for Service {
#[cfg(test)]
mod tests {
use client::ClientIoMessage;
use client::Client;
use io::{IoService};
use spec;
use journaldb::Algorithm;
use snapshot::SnapshotService;
use super::*;
use types::snapshot::{ManifestData, RestorationStatus};
use types::{
io_message::ClientIoMessage,
snapshot::{ManifestData, RestorationStatus}
};
use tempdir::TempDir;
use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler};
@ -919,7 +921,7 @@ mod tests {
fn sends_async_messages() {
let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()];
let client = generate_dummy_client_with_spec_and_data(spec::new_null, 400, 5, &gas_prices);
let service = IoService::<ClientIoMessage>::start().unwrap();
let service = IoService::<ClientIoMessage<Client>>::start().unwrap();
let spec = spec::new_test();
let tempdir = TempDir::new("").unwrap();
@ -932,7 +934,7 @@ mod tests {
pruning: Algorithm::Archive,
channel: service.channel(),
snapshot_root: dir,
client: client,
client,
};
let service = Service::new(snapshot_params).unwrap();

2
ethcore/src/snapshot/tests/helpers.rs
View File

@ -26,7 +26,7 @@ use account_db::AccountDBMut;
use types::basic_account::BasicAccount;
use blockchain::{BlockChain, BlockChainDB};
use client::Client;
use client_traits::ChainInfo;
use client_traits::{ChainInfo, SnapshotClient};
use engine::Engine;
use snapshot::{StateRebuilder};
use snapshot::io::{SnapshotReader, PackedWriter, PackedReader};

3
ethcore/src/snapshot/tests/proof_of_work.rs
View File

@ -26,8 +26,9 @@ use types::{
use blockchain::generator::{BlockGenerator, BlockBuilder};
use blockchain::{BlockChain, ExtrasInsert};
use client_traits::SnapshotWriter;
use snapshot::{chunk_secondary, Error as SnapshotError, SnapshotComponents};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader};
use parking_lot::Mutex;
use snappy;

9
ethcore/src/snapshot/tests/service.rs
View File

@ -22,14 +22,14 @@ use std::sync::Arc;
use tempdir::TempDir;
use blockchain::BlockProvider;
use client::{Client, ClientConfig};
use client_traits::{BlockInfo, ImportBlock};
use client_traits::{BlockInfo, ImportBlock, SnapshotWriter};
use types::{
ids::BlockId,
snapshot::Progress,
verification::Unverified,
snapshot::{ManifestData, RestorationStatus},
};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader};
use snapshot::service::{Service, ServiceParams};
use snapshot::{chunk_state, chunk_secondary, SnapshotService};
use spec;
@ -77,7 +77,7 @@ fn restored_is_equivalent() {
};
let service = Service::new(service_params).unwrap();
service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap();
service.take_snapshot(&*client, NUM_BLOCKS as u64).unwrap();
let manifest = service.manifest().unwrap();
@ -226,7 +226,6 @@ fn keep_ancient_blocks() {
client2.import_block(Unverified::from_rlp(block.into_inner()).unwrap()).unwrap();
}
client2.import_verified_blocks();
client2.flush_queue();
// Restore the Snapshot
@ -304,7 +303,7 @@ fn recover_aborted_recovery() {
};
let service = Service::new(service_params).unwrap();
service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap();
service.take_snapshot(&*client, NUM_BLOCKS as u64).unwrap();
let manifest = service.manifest().unwrap();
service.init_restore(manifest.clone(), true).unwrap();

3
ethcore/src/snapshot/tests/state.rs
View File

@ -24,9 +24,10 @@ use types::{
basic_account::BasicAccount,
errors::EthcoreError as Error,
};
use client_traits::SnapshotWriter;
use snapshot::account;
use snapshot::{chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
use snapshot::io::{PackedReader, PackedWriter, SnapshotReader};
use super::helpers::StateProducer;
use rand::SeedableRng;
use rand_xorshift::XorShiftRng;

11
ethcore/src/snapshot/watcher.rs
View File

@ -17,9 +17,12 @@
//! Watcher for snapshot-related chain events.
use parking_lot::Mutex;
use client::{Client, ChainNotify, NewBlocks, ClientIoMessage};
use client::{Client, ChainNotify, NewBlocks};
use client_traits::BlockInfo;
use types::ids::BlockId;
use types::{
ids::BlockId,
io_message::ClientIoMessage,
};
use io::IoChannel;
use ethereum_types::H256;
@ -55,7 +58,7 @@ trait Broadcast: Send + Sync {
fn take_at(&self, num: Option<u64>);
}
impl Broadcast for Mutex<IoChannel<ClientIoMessage>> {
impl Broadcast for Mutex<IoChannel<ClientIoMessage<Client>>> {
fn take_at(&self, num: Option<u64>) {
let num = match num {
Some(n) => n,
@ -83,7 +86,7 @@ impl Watcher {
/// Create a new `Watcher` which will trigger a snapshot event
/// once every `period` blocks, but only after that block is
/// `history` blocks old.
pub fn new<F>(client: Arc<Client>, sync_status: F, channel: IoChannel<ClientIoMessage>, period: u64, history: u64) -> Self
pub fn new<F>(client: Arc<Client>, sync_status: F, channel: IoChannel<ClientIoMessage<Client>>, period: u64, history: u64) -> Self
where F: 'static + Send + Sync + Fn() -> bool
{
Watcher {

3
ethcore/src/test_helpers.rs
View File

@ -186,7 +186,6 @@ pub fn generate_dummy_client_with_spec_and_data<F>(test_spec: F, block_number: u
db = b.drain().state.drop().1;
}
client.flush_queue();
client.import_verified_blocks();
client
}
@ -239,7 +238,6 @@ pub fn push_block_with_transactions(client: &Arc<Client>, transactions: &[Signed
}
client.flush_queue();
client.import_verified_blocks();
}
/// Creates dummy client (not test client) with corresponding blocks
@ -261,7 +259,6 @@ pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> Arc<Client> {
}
}
client.flush_queue();
client.import_verified_blocks();
client
}

7
ethcore/src/tests/client.rs
View File

@ -32,7 +32,10 @@ use types::{
};
use client::{Client, ClientConfig, PrepareOpenBlock, ImportSealedBlock};
use client_traits::{BlockInfo, BlockChainClient, BlockChainReset, ChainInfo, ImportBlock};
use client_traits::{
BlockInfo, BlockChainClient, BlockChainReset, ChainInfo,
ImportBlock, Tick,
};
use spec;
use machine::executive::{Executive, TransactOptions};
use miner::{Miner, PendingOrdering, MinerService};
@ -55,7 +58,6 @@ fn imports_from_empty() {
Arc::new(Miner::new_for_tests(&spec, None)),
IoChannel::disconnected(),
).unwrap();
client.import_verified_blocks();
client.flush_queue();
}
@ -102,7 +104,6 @@ fn imports_good_block() {
panic!("error importing block being good by definition");
}
client.flush_queue();
client.import_verified_blocks();
let block = client.block_header(BlockId::Number(1)).unwrap();
assert!(!block.into_inner().is_empty());

1
ethcore/src/tests/trace.rs
View File

@ -181,7 +181,6 @@ fn can_trace_block_and_uncle_reward() {
block.drain();
client.flush_queue();
client.import_verified_blocks();
// Test0. Check overall filter
let filter = TraceFilter {

13
ethcore/sync/src/tests/consensus.rs
View File

@ -20,11 +20,14 @@ use ethereum_types::{U256, Address};
use io::{IoHandler, IoChannel};
use client_traits::ChainInfo;
use engine::signer;
use ethcore::client::{ClientIoMessage};
use spec;
use ethcore::client::Client;
use ethcore::miner::{self, MinerService};
use ethkey::{KeyPair, Secret};
use types::transaction::{Action, PendingTransaction, Transaction};
use types::{
io_message::ClientIoMessage,
transaction::{Action, PendingTransaction, Transaction}
};
use super::helpers::*;
use SyncConfig;
@ -47,8 +50,8 @@ fn authority_round() {
let chain_id = spec::new_test_round().chain_id();
let mut net = TestNet::with_spec(2, SyncConfig::default(), spec::new_test_round);
let io_handler0: Arc<dyn IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<dyn IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
let io_handler0: Arc<dyn IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<dyn IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
// Push transaction to both clients. Only one of them gets lucky to produce a block.
net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone())));
net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone())));
@ -115,7 +118,7 @@ fn authority_round() {
net.peer(1).chain.engine().step();
net.peer(1).chain.engine().step();
assert_eq!(net.peer(1).chain.chain_info().best_block_number, 5);
// Reorg to the longest chain one not ealier view one.
// Reorg to the longest chain one not earlier view one.
net.sync();
let ci0 = net.peer(0).chain.chain_info();
let ci1 = net.peer(1).chain.chain_info();

7
ethcore/sync/src/tests/helpers.rs
View File

@ -22,9 +22,10 @@ use bytes::Bytes;
use network::{self, PeerId, ProtocolId, PacketId, SessionInfo};
use network::client_version::ClientVersion;
use tests::snapshot::*;
use types::io_message::ClientIoMessage;
use client_traits::BlockChainClient;
use ethcore::client::{TestBlockChainClient, Client as EthcoreClient,
ClientConfig, ChainNotify, NewBlocks, ChainMessageType, ClientIoMessage};
ClientConfig, ChainNotify, NewBlocks, ChainMessageType};
use ethcore::snapshot::SnapshotService;
use spec::{self, Spec};
use ethcore_private_tx::PrivateStateDB;
@ -554,8 +555,8 @@ impl TestIoHandler {
}
}
impl IoHandler<ClientIoMessage> for TestIoHandler {
fn message(&self, _io: &IoContext<ClientIoMessage>, net_message: &ClientIoMessage) {
impl IoHandler<ClientIoMessage<EthcoreClient>> for TestIoHandler {
fn message(&self, _io: &IoContext<ClientIoMessage<EthcoreClient>>, net_message: &ClientIoMessage<EthcoreClient>) {
match *net_message {
ClientIoMessage::Execute(ref exec) => {
*self.private_tx_queued.lock() += 1;

17
ethcore/sync/src/tests/private.rs
View File

@ -18,12 +18,15 @@ use std::sync::Arc;
use hash::keccak;
use io::{IoHandler, IoChannel};
use types::transaction::{Transaction, Action};
use types::ids::BlockId;
use types::{
ids::BlockId,
io_message::ClientIoMessage,
};
use client_traits::BlockChainClient;
use engine::signer;
use ethcore::{
client::Client,
CreateContractAddress,
client::ClientIoMessage,
miner::{self, MinerService},
test_helpers::{push_block_with_transactions, new_db},
};
@ -52,8 +55,8 @@ fn send_private_transaction() {
let mut net = TestNet::with_spec(2, SyncConfig::default(), seal_spec);
let client0 = net.peer(0).chain.clone();
let client1 = net.peer(1).chain.clone();
let io_handler0: Arc<dyn IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<dyn IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
let io_handler0: Arc<dyn IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<dyn IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone())));
net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone())));
@ -65,7 +68,7 @@ fn send_private_transaction() {
let (address, _) = contract_address(CreateContractAddress::FromSenderAndNonce, &s0.address(), &0.into(), &[]);
let chain_id = client0.signing_chain_id();
// Exhange statuses
// Exchange statuses
net.sync();
// Setup private providers
@ -173,8 +176,8 @@ fn sync_private_state() {
let mut net = TestNet::with_spec(2, SyncConfig::default(), seal_spec);
let client0 = net.peer(0).chain.clone();
let client1 = net.peer(1).chain.clone();
let io_handler0: Arc<IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<IoHandler<ClientIoMessage>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
let io_handler0: Arc<IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(0).chain.clone()));
let io_handler1: Arc<IoHandler<ClientIoMessage<Client>>> = Arc::new(TestIoHandler::new(net.peer(1).chain.clone()));
net.peer(0).miner.set_author(miner::Author::Sealer(signer::from_keypair(s0.clone())));
net.peer(1).miner.set_author(miner::Author::Sealer(signer::from_keypair(s1.clone())));

42
ethcore/types/src/client_types.rs
View File

@ -18,8 +18,12 @@
use std::{
fmt::{Display, Formatter, Error as FmtError},
ops,
cmp,
time::Duration,
};
use ethereum_types::U256;
use crate::header::Header;
/// Operating mode for the client.
#[derive(Debug, Eq, PartialEq, Clone)]
@ -47,3 +51,41 @@ impl Display for Mode {
}
}
/// Report on the status of a client.
#[derive(Default, Clone, Debug, Eq, PartialEq)]
pub struct ClientReport {
/// How many blocks have been imported so far.
pub blocks_imported: usize,
/// How many transactions have been applied so far.
pub transactions_applied: usize,
/// How much gas has been processed so far.
pub gas_processed: U256,
/// Memory used by state DB
pub state_db_mem: usize,
}
impl ClientReport {
/// Alter internal reporting to reflect the additional `block` has been processed.
pub fn accrue_block(&mut self, header: &Header, transactions: usize) {
self.blocks_imported += 1;
self.transactions_applied += transactions;
self.gas_processed = self.gas_processed + *header.gas_used();
}
}
impl<'a> ops::Sub<&'a ClientReport> for ClientReport {
type Output = Self;
fn sub(mut self, other: &'a ClientReport) -> Self {
let higher_mem = cmp::max(self.state_db_mem, other.state_db_mem);
let lower_mem = cmp::min(self.state_db_mem, other.state_db_mem);
self.blocks_imported -= other.blocks_imported;
self.transactions_applied -= other.transactions_applied;
self.gas_processed = self.gas_processed - other.gas_processed;
self.state_db_mem = higher_mem - lower_mem;
self
}
}

18
ethcore/src/client/io_message.rs → ethcore/types/src/io_message.rs
View File

@ -14,15 +14,17 @@
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
//! Defines the `ClientIoMessage` type, used pervasively throughout various parts of the project to
//! communicate between each other.
use std::fmt;
use bytes::Bytes;
use client::Client;
use ethereum_types::H256;
use types::snapshot::ManifestData;
use crate::snapshot::ManifestData;
/// Message type for external and internal events
#[derive(Debug)]
pub enum ClientIoMessage {
pub enum ClientIoMessage<C> {
/// Best Block Hash in chain has been changed
NewChainHead,
/// A block is ready
@ -36,20 +38,20 @@ pub enum ClientIoMessage {
/// Take a snapshot for the block with given number.
TakeSnapshot(u64),
/// Execute wrapped closure
Execute(Callback),
Execute(Callback<C>),
}
impl ClientIoMessage {
impl<C> ClientIoMessage<C> {
/// Create new `ClientIoMessage` that executes given procedure.
pub fn execute<F: Fn(&Client) + Send + Sync + 'static>(fun: F) -> Self {
pub fn execute<F: Fn(&C) + Send + Sync + 'static>(fun: F) -> Self {
ClientIoMessage::Execute(Callback(Box::new(fun)))
}
}
/// A function to invoke in the client thread.
pub struct Callback(pub Box<dyn Fn(&Client) + Send + Sync>);
pub struct Callback<C>(pub Box<dyn Fn(&C) + Send + Sync>);
impl fmt::Debug for Callback {
impl<C> fmt::Debug for Callback<C> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "<callback>")
}

1
ethcore/types/src/lib.rs
View File

@ -71,6 +71,7 @@ pub mod errors;
pub mod filter;
pub mod header;
pub mod ids;
pub mod io_message;
pub mod log_entry;
pub mod pruning_info;
pub mod receipt;

34
ethcore/verification/Cargo.toml Normal file
View File

@ -0,0 +1,34 @@
[package]
description = "Block verification utilities."
name = "verification"
version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
license = "GPL-3.0"
[dependencies]
blockchain = { package = "ethcore-blockchain", path = "../blockchain" }
call-contract = { package = "ethcore-call-contract", path = "../call-contract" }
client-traits = { path = "../client-traits" }
common-types = { path = "../types" }
engine = { path = "../engine" }
ethcore-io = { path = "../../util/io" }
ethereum-types = "0.6.0"
keccak-hash = "0.2.0"
len-caching-lock = { path = "../../util/len-caching-lock" }
log = "0.4"
num_cpus = "1.2"
parity-bytes = "0.1.0"
parity-util-mem = "0.2.0"
parking_lot = "0.8.0"
rlp = "0.4.2"
time-utils = { path = "../../util/time-utils" }
triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" }
unexpected = { path = "../../util/unexpected" }
[dev-dependencies]
ethcore = { path = "../", features = ["test-helpers"] }
ethkey = { path = "../../accounts/ethkey" }
machine = { path = "../machine" }
null-engine = { path = "../engines/null-engine" }
spec = { path = "../spec" }

2
ethcore/src/verification/canon_verifier.rs → ethcore/verification/src/canon_verifier.rs
View File

@ -19,7 +19,7 @@
use call_contract::CallContract;
use client_traits::BlockInfo;
use engine::Engine;
use types::{
use common_types::{
header::Header,
errors::EthcoreError as Error,
};

7
ethcore/src/verification/mod.rs → ethcore/verification/src/lib.rs
View File

@ -18,6 +18,8 @@
use call_contract::CallContract;
use client_traits::BlockInfo;
// The MallocSizeOf derive looks for this in the root
use parity_util_mem as malloc_size_of;
mod verification;
mod verifier;
@ -29,11 +31,6 @@ pub use self::verification::FullFamilyParams;
pub use self::verifier::Verifier;
pub use self::queue::{BlockQueue, Config as QueueConfig};
use self::verification::{
verify_block_basic,
verify_block_unordered,
verify_header_params,
};
use self::canon_verifier::CanonVerifier;
use self::noop_verifier::NoopVerifier;

4
ethcore/src/verification/noop_verifier.rs → ethcore/verification/src/noop_verifier.rs
View File

@ -18,11 +18,11 @@
use call_contract::CallContract;
use client_traits::BlockInfo;
use engine::Engine;
use types::{
use common_types::{
header::Header,
errors::EthcoreError as Error
};
use engine::Engine;
use super::{verification, Verifier};
/// A no-op verifier -- this will verify everything it's given immediately.

11
ethcore/src/verification/queue/kind.rs → ethcore/verification/src/queue/kind.rs
View File

@ -21,7 +21,7 @@ use engine::Engine;
use parity_util_mem::MallocSizeOf;
use ethereum_types::{H256, U256};
use types::errors::EthcoreError as Error;
use common_types::errors::EthcoreError as Error;
pub use self::blocks::Blocks;
pub use self::headers::Headers;
@ -70,12 +70,13 @@ pub mod blocks {
use super::{Kind, BlockLike};
use engine::Engine;
use types::{
use common_types::{
block::PreverifiedBlock,
errors::{EthcoreError as Error, BlockError},
verification::Unverified,
};
use verification::{verify_block_basic, verify_block_unordered};
use log::{debug, warn};
use crate::verification::{verify_block_basic, verify_block_unordered};
use ethereum_types::{H256, U256};
@ -147,11 +148,11 @@ pub mod headers {
use super::{Kind, BlockLike};
use engine::Engine;
use types::{
use common_types::{
header::Header,
errors::EthcoreError as Error,
};
use verification::verify_header_params;
use crate::verification::verify_header_params;
use ethereum_types::{H256, U256};

94
ethcore/src/verification/queue/mod.rs → ethcore/verification/src/queue/mod.rs
View File

@ -22,17 +22,19 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
use std::sync::Arc;
use std::cmp;
use std::collections::{VecDeque, HashSet, HashMap};
use parity_util_mem::{MallocSizeOf, MallocSizeOfExt};
use ethereum_types::{H256, U256};
use parking_lot::{Condvar, Mutex, RwLock};
use io::*;
use engine::Engine;
use client::ClientIoMessage;
use len_caching_lock::LenCachingMutex;
use types::{
use common_types::{
block_status::BlockStatus,
io_message::ClientIoMessage,
errors::{BlockError, EthcoreError as Error, ImportError},
verification::VerificationQueueInfo as QueueInfo,
};
use ethcore_io::*;
use ethereum_types::{H256, U256};
use engine::Engine;
use len_caching_lock::LenCachingMutex;
use log::{debug, trace};
use parity_util_mem::{MallocSizeOf, MallocSizeOfExt};
use parking_lot::{Condvar, Mutex, RwLock};
use self::kind::{BlockLike, Kind};
@ -42,10 +44,10 @@ const MIN_MEM_LIMIT: usize = 16384;
const MIN_QUEUE_LIMIT: usize = 512;
/// Type alias for block queue convenience.
pub type BlockQueue = VerificationQueue<self::kind::Blocks>;
pub type BlockQueue<C> = VerificationQueue<self::kind::Blocks, C>;
/// Type alias for header queue convenience.
pub type HeaderQueue = VerificationQueue<self::kind::Headers>;
pub type HeaderQueue<C> = VerificationQueue<self::kind::Headers, C>;
/// Verification queue configuration
#[derive(Debug, PartialEq, Clone)]
@ -84,7 +86,7 @@ impl Default for VerifierSettings {
fn default() -> Self {
VerifierSettings {
scale_verifiers: false,
num_verifiers: ::num_cpus::get(),
num_verifiers: num_cpus::get(),
}
}
}
@ -113,9 +115,8 @@ pub enum Status {
Unknown,
}
impl Into<::types::block_status::BlockStatus> for Status {
fn into(self) -> ::types::block_status::BlockStatus {
use ::types::block_status::BlockStatus;
impl Into<BlockStatus> for Status {
fn into(self) -> BlockStatus {
match self {
Status::Queued => BlockStatus::Queued,
Status::Bad => BlockStatus::Bad,
@ -133,12 +134,12 @@ struct Sizes {
/// A queue of items to be verified. Sits between network or other I/O and the `BlockChain`.
/// Keeps them in the same order as inserted, minus invalid items.
pub struct VerificationQueue<K: Kind> {
pub struct VerificationQueue<K: Kind, C: 'static> {
engine: Arc<dyn Engine>,
more_to_verify: Arc<Condvar>,
verification: Arc<Verification<K>>,
deleting: Arc<AtomicBool>,
ready_signal: Arc<QueueSignal>,
ready_signal: Arc<QueueSignal<C>>,
empty: Arc<Condvar>,
processing: RwLock<HashMap<H256, U256>>, // hash to difficulty
ticks_since_adjustment: AtomicUsize,
@ -150,13 +151,13 @@ pub struct VerificationQueue<K: Kind> {
total_difficulty: RwLock<U256>,
}
struct QueueSignal {
struct QueueSignal<C: 'static> {
deleting: Arc<AtomicBool>,
signalled: AtomicBool,
message_channel: Mutex<IoChannel<ClientIoMessage>>,
message_channel: Mutex<IoChannel<ClientIoMessage<C>>>,
}
impl QueueSignal {
impl<C> QueueSignal<C> {
fn set_sync(&self) {
// Do not signal when we are about to close
if self.deleting.load(AtomicOrdering::Relaxed) {
@ -200,9 +201,9 @@ struct Verification<K: Kind> {
check_seal: bool,
}
impl<K: Kind> VerificationQueue<K> {
impl<K: Kind, C> VerificationQueue<K, C> {
/// Creates a new queue instance.
pub fn new(config: Config, engine: Arc<dyn Engine>, message_channel: IoChannel<ClientIoMessage>, check_seal: bool) -> Self {
pub fn new(config: Config, engine: Arc<dyn Engine>, message_channel: IoChannel<ClientIoMessage<C>>, check_seal: bool) -> Self {
let verification = Arc::new(Verification {
unverified: LenCachingMutex::new(VecDeque::new()),
verifying: LenCachingMutex::new(VecDeque::new()),
@ -213,7 +214,7 @@ impl<K: Kind> VerificationQueue<K> {
verifying: AtomicUsize::new(0),
verified: AtomicUsize::new(0),
},
check_seal: check_seal,
check_seal,
});
let more_to_verify = Arc::new(Condvar::new());
let deleting = Arc::new(AtomicBool::new(false));
@ -270,19 +271,19 @@ impl<K: Kind> VerificationQueue<K> {
}
VerificationQueue {
engine: engine,
ready_signal: ready_signal,
more_to_verify: more_to_verify,
verification: verification,
deleting: deleting,
engine,
ready_signal,
more_to_verify,
verification,
deleting,
processing: RwLock::new(HashMap::new()),
empty: empty,
empty,
ticks_since_adjustment: AtomicUsize::new(0),
max_queue_size: cmp::max(config.max_queue_size, MIN_QUEUE_LIMIT),
max_mem_use: cmp::max(config.max_mem_use, MIN_MEM_LIMIT),
scale_verifiers: scale_verifiers,
verifier_handles: verifier_handles,
state: state,
scale_verifiers,
verifier_handles,
state,
total_difficulty: RwLock::new(0.into()),
}
}
@ -291,7 +292,7 @@ impl<K: Kind> VerificationQueue<K> {
verification: Arc<Verification<K>>,
engine: Arc<dyn Engine>,
wait: Arc<Condvar>,
ready: Arc<QueueSignal>,
ready: Arc<QueueSignal<C>>,
empty: Arc<Condvar>,
state: Arc<(Mutex<State>, Condvar)>,
id: usize,
@ -373,7 +374,7 @@ impl<K: Kind> VerificationQueue<K> {
// we're next!
let mut verified = verification.verified.lock();
let mut bad = verification.bad.lock();
VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
VerificationQueue::<_, C>::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
true
} else {
false
@ -388,7 +389,7 @@ impl<K: Kind> VerificationQueue<K> {
verifying.retain(|e| e.hash != hash);
if verifying.front().map_or(false, |x| x.output.is_some()) {
VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
VerificationQueue::<_, C>::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
true
} else {
false
@ -706,7 +707,7 @@ impl<K: Kind> VerificationQueue<K> {
}
}
impl<K: Kind> Drop for VerificationQueue<K> {
impl<K: Kind, C> Drop for VerificationQueue<K, C> {
fn drop(&mut self) {
trace!(target: "shutdown", "[VerificationQueue] Closing...");
self.clear();
@ -734,11 +735,12 @@ impl<K: Kind> Drop for VerificationQueue<K> {
#[cfg(test)]
mod tests {
use io::*;
use ethcore_io::*;
use super::{BlockQueue, Config, State};
use test_helpers::{get_good_dummy_block_seq, get_good_dummy_block};
use bytes::Bytes;
use types::{
use ethcore::test_helpers::{get_good_dummy_block_seq, get_good_dummy_block};
use ethcore::client::Client;
use parity_bytes::Bytes;
use common_types::{
errors::{EthcoreError, ImportError},
verification::Unverified,
view,
@ -748,7 +750,7 @@ mod tests {
// create a test block queue.
// auto_scaling enables verifier adjustment.
fn get_test_queue(auto_scale: bool) -> BlockQueue {
fn get_test_queue(auto_scale: bool) -> BlockQueue<Client> {
let spec = spec::new_test();
let engine = spec.engine;
@ -773,7 +775,7 @@ mod tests {
// TODO better test
let spec = spec::new_test();
let engine = spec.engine;
let _ = BlockQueue::new(Config::default(), engine, IoChannel::disconnected(), true);
let _ = BlockQueue::<Client>::new(Config::default(), engine, IoChannel::disconnected(), true);
}
#[test]
@ -853,7 +855,7 @@ mod tests {
let engine = spec.engine;
let mut config = Config::default();
config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000
let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true);
let queue = BlockQueue::<Client>::new(config, engine, IoChannel::disconnected(), true);
assert!(!queue.queue_info().is_full());
let mut blocks = get_good_dummy_block_seq(50);
for b in blocks.drain(..) {
@ -903,7 +905,7 @@ mod tests {
let spec = spec::new_test();
let engine = spec.engine;
let config = get_test_config(1, false);
let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true);
let queue = BlockQueue::<Client>::new(config, engine, IoChannel::disconnected(), true);
assert_eq!(queue.num_verifiers(), 1);
}
@ -913,7 +915,7 @@ mod tests {
let spec = spec::new_test();
let engine = spec.engine;
let config = get_test_config(0, false);
let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true);
let queue = BlockQueue::<Client>::new(config, engine, IoChannel::disconnected(), true);
assert_eq!(queue.num_verifiers(), 1);
}
@ -923,7 +925,7 @@ mod tests {
let spec = spec::new_test();
let engine = spec.engine;
let config = get_test_config(10_000, false);
let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true);
let queue = BlockQueue::<Client>::new(config, engine, IoChannel::disconnected(), true);
let num_cpus = ::num_cpus::get();
assert_eq!(queue.num_verifiers(), num_cpus);
@ -937,7 +939,7 @@ mod tests {
let spec = spec::new_test();
let engine = spec.engine;
let config = get_test_config(num_cpus - 1, true);
let queue = BlockQueue::new(config, engine, IoChannel::disconnected(), true);
let queue = BlockQueue::<Client>::new(config, engine, IoChannel::disconnected(), true);
queue.scale_verifiers(num_cpus);
assert_eq!(queue.num_verifiers(), num_cpus);

25
ethcore/src/verification/verification.rs → ethcore/verification/src/verification.rs
View File

@ -24,7 +24,7 @@
use std::collections::HashSet;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use hash::keccak;
use keccak_hash::keccak;
use rlp::Rlp;
use triehash::ordered_trie_root;
use unexpected::{Mismatch, OutOfBounds};
@ -33,7 +33,7 @@ use blockchain::*;
use call_contract::CallContract;
use client_traits::BlockInfo;
use engine::Engine;
use types::{
use common_types::{
BlockNumber,
header::Header,
errors::{EthcoreError as Error, BlockError},
@ -366,15 +366,19 @@ mod tests {
use std::collections::{BTreeMap, HashMap};
use std::time::{SystemTime, UNIX_EPOCH};
use ethereum_types::{H256, BloomRef, U256, Address};
use blockchain::{BlockDetails, TransactionAddress, BlockReceipts};
use bytes::Bytes;
use hash::keccak;
use parity_bytes::Bytes;
use keccak_hash::keccak;
use engine::Engine;
use ethkey::{Random, Generator};
use spec;
use test_helpers::{create_test_block_with_data, create_test_block};
use types::{
use ethcore::{
client::TestBlockChainClient,
test_helpers::{create_test_block_with_data, create_test_block}
};
use common_types::{
encoded,
engines::params::CommonParams,
errors::BlockError::*,
@ -383,6 +387,8 @@ mod tests {
};
use rlp;
use triehash::ordered_trie_root;
use machine::Machine;
use null_engine::NullEngine;
fn check_ok(result: Result<(), Error>) {
result.unwrap_or_else(|e| panic!("Block verification failed: {:?}", e));
@ -518,7 +524,7 @@ mod tests {
// additions that need access to state (tx filter in specific)
// no existing tests need access to test, so having this not function
// is fine.
let client = ::client::TestBlockChainClient::default();
let client = TestBlockChainClient::default();
let parent = bc.block_header_data(header.parent_hash())
.ok_or(BlockError::UnknownParent(*header.parent_hash()))?
.decode()?;
@ -779,11 +785,6 @@ mod tests {
#[test]
fn dust_protection() {
use ethkey::{Generator, Random};
use types::transaction::{Transaction, Action};
use machine::Machine;
use null_engine::NullEngine;
let mut params = CommonParams::default();
params.dust_protection_transition = 0;
params.nonce_cap_increment = 2;

5
ethcore/src/verification/verifier.rs → ethcore/verification/src/verifier.rs
View File

@ -18,11 +18,12 @@
use call_contract::CallContract;
use client_traits::BlockInfo;
use engine::Engine;
use types::{
use common_types::{
header::Header,
errors::EthcoreError as Error,
};
use engine::Engine;
use super::verification;
/// Should be used to verify blocks.

4
parity/blockchain.rs
View File

@ -30,7 +30,6 @@ use client_traits::{BlockInfo, BlockChainReset, Nonce, Balance, BlockChainClient
use ethcore::{
client::{DatabaseCompactionProfile, VMType},
miner::Miner,
verification::queue::VerifierSettings,
};
use ethcore_service::ClientService;
use cache::CacheConfig;
@ -48,6 +47,7 @@ use types::{
client_types::Mode,
verification::Unverified,
};
use verification::queue::VerifierSettings;
#[derive(Debug, PartialEq)]
pub enum DataFormat {
@ -234,7 +234,7 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> {
&cmd.cache_config,
&cmd.compaction).map_err(|e| format!("Failed to open database: {:?}", e))?;
// TODO: could epoch signals be avilable at the end of the file?
// TODO: could epoch signals be available at the end of the file?
let fetch = ::light::client::fetch::unavailable();
let service = LightClientService::start(config, &spec, fetch, db, cache)
.map_err(|e| format!("Failed to start client: {}", e))?;

2
parity/configuration.rs
View File

@ -32,8 +32,8 @@ use ethkey::{Secret, Public};
use ethcore::client::{VMType};
use ethcore::miner::{stratum, MinerOptions};
use ethcore::snapshot::SnapshotConfiguration;
use ethcore::verification::queue::VerifierSettings;
use miner::pool;
use verification::queue::VerifierSettings;
use rpc::{IpcConfiguration, HttpConfiguration, WsConfiguration};
use parity_rpc::NetworkSettings;

18
parity/informant.rs
View File

@ -23,13 +23,13 @@ use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
use std::time::{Instant, Duration};
use atty;
use ethcore::client::{
ChainNotify, NewBlocks, ClientReport, Client, ClientIoMessage
};
use client_traits::{BlockInfo, ChainInfo, BlockChainClient};
use ethcore::client::{ChainNotify, NewBlocks, Client};
use client_traits::{BlockInfo, ChainInfo, BlockChainClient, IoClient};
use types::{
BlockNumber,
client_types::ClientReport,
ids::BlockId,
io_message::ClientIoMessage,
blockchain_info::BlockChainInfo,
verification::VerificationQueueInfo as BlockQueueInfo,
snapshot::RestorationStatus,
@ -451,12 +451,16 @@ impl LightChainNotify for Informant<LightNodeInformantData> {
const INFO_TIMER: TimerToken = 0;
impl<T: InformantData> IoHandler<ClientIoMessage> for Informant<T> {
fn initialize(&self, io: &IoContext<ClientIoMessage>) {
impl<T, C> IoHandler<ClientIoMessage<C>> for Informant<T>
where
T: InformantData,
C: client_traits::Tick + 'static,
{
fn initialize(&self, io: &IoContext<ClientIoMessage<C>>) {
io.register_timer(INFO_TIMER, Duration::from_secs(5)).expect("Error registering timer");
}
fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
fn timeout(&self, _io: &IoContext<ClientIoMessage<C>>, timer: TimerToken) {
if timer == INFO_TIMER && !self.in_shutdown.load(AtomicOrdering::SeqCst) {
self.tick();
}

1
parity/lib.rs
View File

@ -74,6 +74,7 @@ extern crate parity_updater as updater;
extern crate parity_version;
extern crate registrar;
extern crate spec;
extern crate verification;
#[macro_use]
extern crate log as rlog;

12
parity/run.rs
View File

@ -27,7 +27,7 @@ use ethcore::client::{Client, DatabaseCompactionProfile, VMType};
use ethcore::miner::{self, stratum, Miner, MinerService, MinerOptions};
use ethcore::snapshot::{self, SnapshotConfiguration};
use spec::SpecParams;
use ethcore::verification::queue::VerifierSettings;
use verification::queue::VerifierSettings;
use ethcore_logger::{Config as LogConfig, RotatingLogger};
use ethcore_service::ClientService;
use ethereum_types::Address;
@ -307,18 +307,18 @@ fn execute_light_impl<Cr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq
// start RPCs
let deps_for_rpc_apis = Arc::new(rpc_apis::LightDependencies {
signer_service: signer_service,
signer_service,
client: client.clone(),
sync: light_sync.clone(),
net: light_sync.clone(),
accounts: account_provider,
logger: logger,
logger,
settings: Arc::new(cmd.net_settings),
on_demand: on_demand,
on_demand,
cache: cache.clone(),
transaction_queue: txq,
ws_address: cmd.ws_conf.address(),
fetch: fetch,
fetch,
geth_compatibility: cmd.geth_compatibility,
experimental_rpcs: cmd.experimental_rpcs,
executor: runtime.executor(),
@ -344,7 +344,7 @@ fn execute_light_impl<Cr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq
LightNodeInformantData {
client: client.clone(),
sync: light_sync.clone(),
cache: cache,
cache,
},
None,
Some(rpc_stats),

1
parity/snapshot.rs
View File

@ -20,6 +20,7 @@ use std::time::Duration;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use client_traits::SnapshotClient;
use hash::keccak;
use ethcore::snapshot::{SnapshotConfiguration, SnapshotService as SS};
use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter};

1
rpc/Cargo.toml
View File

@ -67,6 +67,7 @@ account-state = { path = "../ethcore/account-state" }
stats = { path = "../util/stats" }
trace = { path = "../ethcore/trace" }
vm = { path = "../ethcore/vm" }
verification = { path = "../ethcore/verification" }
[dev-dependencies]
client-traits = { path = "../ethcore/client-traits" }

1
rpc/src/lib.rs
View File

@ -89,6 +89,7 @@ extern crate account_state;
extern crate stats;
extern crate tempdir;
extern crate trace;
extern crate verification;
extern crate vm;
#[cfg(any(test, feature = "ethcore-accounts"))]

3
rpc/src/v1/tests/eth.rs
View File

@ -24,7 +24,7 @@ use ethcore::client::{Client, ClientConfig};
use ethcore::miner::Miner;
use spec::{Genesis, Spec, self};
use ethcore::test_helpers;
use ethcore::verification::VerifierType;
use verification::VerifierType;
use ethereum_types::{Address, H256, U256};
use ethjson::blockchain::BlockChain;
use ethjson::spec::ForkSpec;
@ -98,7 +98,6 @@ impl EthTester {
if let Ok(block) = Unverified::from_rlp(b) {
let _ = tester.client.import_block(block);
tester.client.flush_queue();
tester.client.import_verified_blocks();
}
}