From d193ddde19dbf3574aca8638c1a4041cc06472d7 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 3 Sep 2019 11:29:25 +0200 Subject: [PATCH] Extract snapshot to own crate (#11010) * Move snapshot to own crate Sort out imports * WIP cargo toml * Make snapshotting generic over the client Sort out tests * Sort out types from blockchain and client * Sort out sync * Sort out imports and generics * Sort out main binary * Fix sync test-helpers * Sort out import for secret-store * Sort out more imports * Fix easy todos * cleanup * Cleanup * remove unneded workspace member * cleanup * Sort out test-helpers dependency on account-db * Update ethcore/client-traits/src/lib.rs Co-Authored-By: Niklas Adolfsson * Update ethcore/snapshot/Cargo.toml --- Cargo.lock | 58 ++++++++++- Cargo.toml | 1 + ethcore/Cargo.toml | 24 +++-- ethcore/blockchain/src/block_info.rs | 54 ---------- ethcore/blockchain/src/blockchain.rs | 31 +++--- ethcore/blockchain/src/lib.rs | 3 - ethcore/blockchain/src/update.rs | 11 ++- ethcore/client-traits/src/lib.rs | 40 +++++++- ethcore/node-filter/src/lib.rs | 10 +- ethcore/private-tx/src/lib.rs | 7 +- ethcore/service/Cargo.toml | 7 +- ethcore/service/src/lib.rs | 1 + ethcore/service/src/service.rs | 18 ++-- ethcore/snapshot/Cargo.toml | 54 ++++++++++ .../{src/snapshot => snapshot/src}/account.rs | 24 ++--- .../{src/snapshot => snapshot/src}/block.rs | 28 +++--- .../src}/consensus/authority.rs | 18 ++-- .../src}/consensus/mod.rs | 4 +- .../src}/consensus/work.rs | 32 +++--- ethcore/{src/snapshot => snapshot/src}/io.rs | 17 ++-- .../snapshot/mod.rs => snapshot/src/lib.rs} | 52 +++++----- .../{src/snapshot => snapshot/src}/service.rs | 99 +++++++++---------- .../src}/tests/helpers.rs | 29 +++--- .../snapshot => snapshot/src}/tests/mod.rs | 2 +- .../src}/tests/proof_of_authority.rs | 24 +++-- .../src}/tests/proof_of_work.rs | 28 +++--- .../src}/tests/service.rs | 39 ++++---- .../snapshot => snapshot/src}/tests/state.rs | 37 +++---- .../src}/tests/test_validator_contract.json | 0 .../{src/snapshot => snapshot/src}/traits.rs | 7 +- .../{src/snapshot => snapshot/src}/watcher.rs | 48 ++++----- ethcore/src/client/client.rs | 16 +-- ethcore/src/client/mod.rs | 2 - ethcore/src/lib.rs | 10 +- ethcore/src/test_helpers.rs | 5 +- ethcore/state-db/src/lib.rs | 6 +- ethcore/sync/Cargo.toml | 11 ++- ethcore/sync/src/api.rs | 13 +-- ethcore/sync/src/chain/handler.rs | 10 +- ethcore/sync/src/chain/mod.rs | 6 +- ethcore/sync/src/lib.rs | 7 +- .../src/{snapshot.rs => snapshot_sync.rs} | 2 +- ethcore/sync/src/sync_io.rs | 2 +- ethcore/sync/src/tests/helpers.rs | 16 ++- ethcore/sync/src/tests/snapshot.rs | 2 +- ethcore/types/src/block.rs | 39 ++++++++ .../{src/client => types/src}/chain_notify.rs | 50 ++-------- ethcore/types/src/client_types.rs | 4 +- .../{blockchain => types}/src/import_route.rs | 6 +- ethcore/types/src/lib.rs | 2 + parity/configuration.rs | 8 +- parity/informant.rs | 21 ++-- parity/lib.rs | 5 +- parity/modules.rs | 5 +- parity/rpc_apis.rs | 2 +- parity/run.rs | 2 +- parity/{snapshot.rs => snapshot_cmd.rs} | 10 +- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/impls/eth.rs | 2 +- rpc/src/v1/impls/eth_pubsub.rs | 4 +- rpc/src/v1/impls/parity.rs | 2 +- rpc/src/v1/tests/helpers/snapshot_service.rs | 2 +- rpc/src/v1/tests/mocked/eth_pubsub.rs | 5 +- secret-store/src/acl_storage.rs | 7 +- secret-store/src/key_server_set.rs | 9 +- .../src/listener/service_contract_listener.rs | 7 +- updater/src/updater.rs | 4 +- 68 files changed, 627 insertions(+), 486 deletions(-) delete mode 100644 ethcore/blockchain/src/block_info.rs create mode 100644 ethcore/snapshot/Cargo.toml rename ethcore/{src/snapshot => snapshot/src}/account.rs (97%) rename ethcore/{src/snapshot => snapshot/src}/block.rs (94%) rename ethcore/{src/snapshot => snapshot/src}/consensus/authority.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/consensus/mod.rs (94%) rename ethcore/{src/snapshot => snapshot/src}/consensus/work.rs (95%) rename ethcore/{src/snapshot => snapshot/src}/io.rs (98%) rename ethcore/{src/snapshot/mod.rs => snapshot/src/lib.rs} (98%) rename ethcore/{src/snapshot => snapshot/src}/service.rs (96%) rename ethcore/{src/snapshot => snapshot/src}/tests/helpers.rs (91%) rename ethcore/{src/snapshot => snapshot/src}/tests/mod.rs (96%) rename ethcore/{src/snapshot => snapshot/src}/tests/proof_of_authority.rs (94%) rename ethcore/{src/snapshot => snapshot/src}/tests/proof_of_work.rs (88%) rename ethcore/{src/snapshot => snapshot/src}/tests/service.rs (91%) rename ethcore/{src/snapshot => snapshot/src}/tests/state.rs (87%) rename ethcore/{src/snapshot => snapshot/src}/tests/test_validator_contract.json (100%) rename ethcore/{src/snapshot => snapshot/src}/traits.rs (99%) rename ethcore/{src/snapshot => snapshot/src}/watcher.rs (87%) rename ethcore/sync/src/{snapshot.rs => snapshot_sync.rs} (99%) rename ethcore/{src/client => types/src}/chain_notify.rs (79%) rename ethcore/{blockchain => types}/src/import_route.rs (95%) rename parity/{snapshot.rs => snapshot_cmd.rs} (96%) diff --git a/Cargo.lock b/Cargo.lock index 98940990f..59ef08be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,7 +1020,6 @@ dependencies = [ "client-traits 0.1.0", "common-types 0.1.0", "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1029,7 +1028,6 @@ dependencies = [ "ethash 1.12.0", "ethcore-accounts 0.1.0", "ethcore-blockchain 0.1.0", - "ethcore-bloom-journal 0.1.0", "ethcore-builtin 0.1.0", "ethcore-call-contract 0.1.0", "ethcore-db 0.1.0", @@ -1047,7 +1045,6 @@ dependencies = [ "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1057,10 +1054,8 @@ dependencies = [ "macros 0.1.0", "memory-cache 0.1.0", "null-engine 0.1.0", - "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", - "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "pod 0.1.0", @@ -1074,6 +1069,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "state-db 0.1.0", "stats 0.1.0", @@ -1462,6 +1458,7 @@ dependencies = [ "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1518,6 +1515,7 @@ dependencies = [ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", @@ -3034,6 +3032,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3182,6 +3181,7 @@ dependencies = [ "serde 1.0.99 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "snapshot 0.1.0", "spec 0.1.0", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4202,6 +4202,54 @@ name = "smallvec" version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "snapshot" +version = "0.1.0" +dependencies = [ + "account-db 0.1.0", + "account-state 0.1.0", + "client-traits 0.1.0", + "common-types 0.1.0", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "engine 0.1.0", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-contract 8.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethabi-derive 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore 1.12.0", + "ethcore-accounts 0.1.0", + "ethcore-blockchain 0.1.0", + "ethcore-bloom-journal 0.1.0", + "ethcore-db 0.1.0", + "ethcore-io 1.12.0", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethkey 0.3.0", + "hash-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "journaldb 0.2.0", + "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.1.1", + "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie-ethereum 0.1.0", + "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp_derive 0.1.0", + "spec 0.1.0", + "state-db 0.1.0", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "triehash-ethereum 0.2.0", +] + [[package]] name = "socket2" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index 298c1542e..3a18c6c9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ semver = "0.9" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +snapshot = { path = "ethcore/snapshot" } spec = { path = "ethcore/spec" } term_size = "0.3" textwrap = "0.9" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index cf27c1a65..b2ac1c8e9 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -7,14 +7,13 @@ version = "1.12.0" authors = ["Parity Technologies "] [dependencies] -account-db = { path = "account-db" } +account-db = { path = "account-db", optional = true } account-state = { path = "account-state" } ansi_term = "0.11" basic-authority = { path = "./engines/basic-authority", optional = true} # used by test-helpers feature blooms-db = { path = "../util/blooms-db", optional = true } client-traits = { path = "./client-traits" } common-types = { path = "types" } -crossbeam-utils = "0.6" engine = { path = "./engine" } env_logger = { version = "0.5", optional = true } ethabi = "8.0" @@ -24,7 +23,6 @@ ethash = { path = "../ethash", optional = true } ethjson = { path = "../json", optional = true } ethkey = { path = "../accounts/ethkey", optional = true } ethcore-blockchain = { path = "./blockchain" } -ethcore-bloom-journal = { path = "../util/bloom" } ethcore-call-contract = { path = "./call-contract" } ethcore-db = { path = "./db" } ethcore-io = { path = "../util/io" } @@ -38,7 +36,6 @@ hash-db = "0.15.0" itertools = "0.5" journaldb = { path = "../util/journaldb" } keccak-hash = "0.2.0" -keccak-hasher = { path = "../util/keccak-hasher" } kvdb = "0.1" kvdb-memorydb = { version = "0.1", optional = true } kvdb-rocksdb = { version = "0.1.3", optional = true } @@ -47,9 +44,7 @@ log = "0.4" macros = { path = "../util/macros", optional = true } machine = { path = "./machine" } memory-cache = { path = "../util/memory-cache" } -num_cpus = "1.2" parity-bytes = "0.1" -parity-snappy = "0.1" parking_lot = "0.8" pod = { path = "pod", optional = true } trie-db = "0.15.0" @@ -62,6 +57,7 @@ rlp_derive = { path = "../util/rlp-derive" } rustc-hex = { version = "1", optional = true } serde = "1.0" serde_derive = "1.0" +snapshot = { path = "snapshot" } spec = { path = "spec" } state-db = { path = "state-db" } tempdir = { version = "0.3", optional = true } @@ -75,6 +71,7 @@ verification = { path = "./verification" } vm = { path = "vm" } [dev-dependencies] +account-db = { path = "account-db" } blooms-db = { path = "../util/blooms-db" } ethcore-builtin = { path = "./builtin" } criterion = "0.2" @@ -129,7 +126,20 @@ ci-skip-tests = [] test-heavy = [] # Compile test helpers # note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works -test-helpers = ["tempdir", "kvdb-memorydb", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "rustc-hex", "basic-authority/test-helpers"] +test-helpers = [ + "account-db", + "blooms-db", + "ethash", + "ethjson", + "ethkey", + "kvdb-memorydb", + "kvdb-rocksdb", + "macros", + "pod", + "rustc-hex", + "tempdir", + "basic-authority/test-helpers" + ] [[bench]] name = "builtin" diff --git a/ethcore/blockchain/src/block_info.rs b/ethcore/blockchain/src/block_info.rs deleted file mode 100644 index 15f71ecb8..000000000 --- a/ethcore/blockchain/src/block_info.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -use ethereum_types::{H256, U256}; -use common_types::BlockNumber; - -/// Brief info about inserted block. -#[derive(Clone)] -pub struct BlockInfo { - /// Block hash. - pub hash: H256, - /// Block number. - pub number: BlockNumber, - /// Total block difficulty. - pub total_difficulty: U256, - /// Block location in blockchain. - pub location: BlockLocation -} - -/// Describes location of newly inserted block. -#[derive(Debug, Clone, PartialEq)] -pub enum BlockLocation { - /// It's part of the canon chain. - CanonChain, - /// It's not a part of the canon chain. - Branch, - /// It's part of the fork which should become canon chain, - /// because its total difficulty is higher than current - /// canon chain difficulty. - BranchBecomingCanonChain(BranchBecomingCanonChainData), -} - -#[derive(Debug, Clone, PartialEq)] -pub struct BranchBecomingCanonChainData { - /// Hash of the newest common ancestor with old canon chain. - pub ancestor: H256, - /// Hashes of the blocks between ancestor and this block. - pub enacted: Vec, - /// Hashes of the blocks which were invalidated. - pub retracted: Vec, -} diff --git a/ethcore/blockchain/src/blockchain.rs b/ethcore/blockchain/src/blockchain.rs index a85de546d..05aae018a 100644 --- a/ethcore/blockchain/src/blockchain.rs +++ b/ethcore/blockchain/src/blockchain.rs @@ -23,18 +23,22 @@ use std::sync::Arc; use ansi_term::Colour; use blooms_db; -use common_types::BlockNumber; -use common_types::blockchain_info::BlockChainInfo; -use common_types::encoded; -use common_types::engines::ForkChoice; -use common_types::engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}; -use common_types::header::{Header, ExtendedHeader}; -use common_types::log_entry::{LogEntry, LocalizedLogEntry}; -use common_types::receipt::Receipt; -use common_types::transaction::LocalizedTransaction; -use common_types::tree_route::TreeRoute; -use common_types::view; -use common_types::views::{BlockView, HeaderView}; +use common_types::{ + BlockNumber, + blockchain_info::BlockChainInfo, + block::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}, + encoded, + engines::ForkChoice, + engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}, + header::{Header, ExtendedHeader}, + import_route::ImportRoute, + log_entry::{LogEntry, LocalizedLogEntry}, + receipt::Receipt, + transaction::LocalizedTransaction, + tree_route::TreeRoute, + view, + views::{BlockView, HeaderView}, +}; use ethcore_db::cache_manager::CacheManager; use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions}; use ethcore_db::{self as db, Writable, Readable, CacheUpdatePolicy}; @@ -50,9 +54,8 @@ use rlp::RlpStream; use rlp_compress::{compress, decompress, blocks_swapper}; use crate::best_block::{BestBlock, BestAncientBlock}; -use crate::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use crate::update::{ExtrasUpdate, ExtrasInsert}; -use crate::{CacheSize, ImportRoute, Config}; +use crate::{CacheSize, Config}; /// Database backing `BlockChain`. pub trait BlockChainDB: Send + Sync { diff --git a/ethcore/blockchain/src/lib.rs b/ethcore/blockchain/src/lib.rs index 77933922e..ceada247c 100644 --- a/ethcore/blockchain/src/lib.rs +++ b/ethcore/blockchain/src/lib.rs @@ -22,11 +22,9 @@ extern crate parity_util_mem as util_mem; extern crate parity_util_mem as malloc_size_of; mod best_block; -mod block_info; mod blockchain; mod cache; mod config; -mod import_route; mod update; pub mod generator; @@ -35,7 +33,6 @@ pub use crate::{ blockchain::{BlockProvider, BlockChain, BlockChainDB, BlockChainDBHandler}, cache::CacheSize, config::Config, - import_route::ImportRoute, update::ExtrasInsert, }; pub use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, BlockNumberKey}; diff --git a/ethcore/blockchain/src/update.rs b/ethcore/blockchain/src/update.rs index 959f55fdf..24d04cab8 100644 --- a/ethcore/blockchain/src/update.rs +++ b/ethcore/blockchain/src/update.rs @@ -16,14 +16,15 @@ use std::collections::HashMap; -use common_types::BlockNumber; -use common_types::encoded::Block; -use common_types::engines::ForkChoice; +use common_types::{ + BlockNumber, + encoded::Block, + engines::ForkChoice, + block::BlockInfo, +}; use ethcore_db::keys::{BlockDetails, BlockReceipts, TransactionAddress}; use ethereum_types::{H256, Bloom}; -use crate::block_info::BlockInfo; - /// Block extras update info. pub struct ExtrasUpdate { /// Block info. diff --git a/ethcore/client-traits/src/lib.rs b/ethcore/client-traits/src/lib.rs index 896f9515b..2a04a7cec 100644 --- a/ethcore/client-traits/src/lib.rs +++ b/ethcore/client-traits/src/lib.rs @@ -29,6 +29,7 @@ use common_types::{ blockchain_info::BlockChainInfo, BlockNumber, call_analytics::CallAnalytics, + chain_notify::{NewBlocks, ChainMessageType}, client_types::Mode, encoded, engines::{epoch::Transition as EpochTransition, machine::Executed}, @@ -40,7 +41,7 @@ use common_types::{ pruning_info::PruningInfo, receipt::LocalizedReceipt, trace_filter::Filter as TraceFilter, - transaction::{self, LocalizedTransaction, CallError, SignedTransaction}, + transaction::{self, LocalizedTransaction, CallError, SignedTransaction, UnverifiedTransaction}, tree_route::TreeRoute, verification::{VerificationQueueInfo, Unverified}, }; @@ -119,7 +120,7 @@ pub trait ChainInfo { } /// Provides various information on a block by it's ID -pub trait BlockInfo { +pub trait BlockInfo: Send + Sync { /// Get raw block header data by block id. fn block_header(&self, id: BlockId) -> Option; @@ -474,3 +475,38 @@ pub trait SnapshotWriter { /// with the chunks written. fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized; } + + +/// Represents what has to be handled by actor listening to chain events +pub trait ChainNotify: Send + Sync { + /// fires when chain has new blocks. + fn new_blocks(&self, _new_blocks: NewBlocks) { + // does nothing by default + } + + /// fires when chain achieves active mode + fn start(&self) { + // does nothing by default + } + + /// fires when chain achieves passive mode + fn stop(&self) { + // does nothing by default + } + + /// fires when chain broadcasts a message + fn broadcast(&self, _message_type: ChainMessageType) { + // does nothing by default + } + + /// fires when new block is about to be imported + /// implementations should be light + fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { + // does nothing by default + } + + /// fires when new transactions are received from a peer + fn transactions_received(&self, _txs: &[UnverifiedTransaction], _peer_id: usize) { + // does nothing by default + } +} diff --git a/ethcore/node-filter/src/lib.rs b/ethcore/node-filter/src/lib.rs index f34b6571b..332b93fc1 100644 --- a/ethcore/node-filter/src/lib.rs +++ b/ethcore/node-filter/src/lib.rs @@ -42,9 +42,11 @@ extern crate log; use std::collections::{HashMap, VecDeque}; use std::sync::Weak; -use common_types::ids::BlockId; -use ethcore::client::{ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; +use common_types::{ + ids::BlockId, + chain_notify::NewBlocks, +}; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::{H256, Address}; use ethabi::FunctionOutputDecoder; use network::{ConnectionFilter, ConnectionDirection}; @@ -159,7 +161,7 @@ mod test { Arc::new(Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); - let filter = NodeFilter::new(Arc::downgrade(&client) as Weak, contract_addr); + let filter = NodeFilter::new(Arc::downgrade(&client) as Weak, contract_addr); let self1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002").unwrap(); let self2 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003").unwrap(); let node1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012").unwrap(); diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index 8673d4be7..fc0686541 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -103,15 +103,14 @@ use machine::{ executed::Executed as FlatExecuted, }; use types::{ + chain_notify::{NewBlocks, ChainMessageType}, ids::BlockId, io_message::ClientIoMessage, transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction}, engines::machine::Executed, }; -use ethcore::client::{ - Client, ChainNotify, NewBlocks, ChainMessageType, Call -}; -use client_traits::BlockInfo; +use ethcore::client::{Client, Call}; +use client_traits::{BlockInfo, ChainNotify}; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; use state_db::StateDB; use account_state::State; diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index f333e979d..4956d9c75 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -6,8 +6,8 @@ authors = ["Parity Technologies "] [dependencies] ansi_term = "0.11" -common-types = { path = "../types" } client-traits = { path = "../client-traits" } +common-types = { path = "../types" } ethcore = { path = ".." } ethcore-blockchain = { path = "../blockchain" } ethcore-io = { path = "../../util/io" } @@ -16,11 +16,12 @@ ethcore-sync = { path = "../sync" } ethereum-types = "0.6.0" kvdb = "0.1" log = "0.4" +snapshot = { path = "../snapshot" } spec = { path = "../spec" } trace-time = "0.1" [dev-dependencies] -ethcore-db = { path = "../db" } ethcore = { path = "..", features = ["test-helpers"] } -tempdir = "0.3" +ethcore-db = { path = "../db" } kvdb-rocksdb = "0.1.3" +tempdir = "0.3" diff --git a/ethcore/service/src/lib.rs b/ethcore/service/src/lib.rs index 662a25676..ed82c7dfa 100644 --- a/ethcore/service/src/lib.rs +++ b/ethcore/service/src/lib.rs @@ -25,6 +25,7 @@ extern crate ethcore_sync as sync; extern crate ethereum_types; extern crate kvdb; extern crate spec; +extern crate snapshot; #[macro_use] extern crate log; diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index add9eb1d0..786b8b8db 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -23,13 +23,13 @@ use std::time::Duration; use ansi_term::Colour; use ethereum_types::H256; use io::{IoContext, TimerToken, IoHandler, IoService, IoError}; - +use client_traits::ChainNotify; use sync::PrivateTxHandler; use blockchain::{BlockChainDB, BlockChainDBHandler}; -use ethcore::client::{Client, ClientConfig, ChainNotify}; +use ethcore::client::{Client, ClientConfig}; use ethcore::miner::Miner; -use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; -use ethcore::snapshot::{SnapshotService as _SnapshotService}; +use snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams}; +use snapshot::{SnapshotService as _SnapshotService}; use spec::Spec; use common_types::{ io_message::ClientIoMessage, @@ -94,7 +94,7 @@ impl PrivateTxHandler for PrivateTxService { pub struct ClientService { io_service: Arc>>, client: Arc, - snapshot: Arc, + snapshot: Arc>, private_tx: Arc, database: Arc, } @@ -185,7 +185,7 @@ impl ClientService { } /// Get snapshot interface. - pub fn snapshot_service(&self) -> Arc { + pub fn snapshot_service(&self) -> Arc> { self.snapshot.clone() } @@ -215,9 +215,9 @@ impl ClientService { } /// IO interface for the Client handler -struct ClientIoHandler { +struct ClientIoHandler { client: Arc, - snapshot: Arc, + snapshot: Arc>, } const CLIENT_TICK_TIMER: TimerToken = 0; @@ -239,7 +239,7 @@ where trace_time!("service::read"); match timer { CLIENT_TICK_TIMER => { - use ethcore::snapshot::SnapshotService; + use snapshot::SnapshotService; let snapshot_restoration = if let RestorationStatus::Ongoing{..} = self.snapshot.status() { true } else { false }; self.client.tick(snapshot_restoration) }, diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml new file mode 100644 index 000000000..57722f13a --- /dev/null +++ b/ethcore/snapshot/Cargo.toml @@ -0,0 +1,54 @@ +[package] +description = "Take and restore snapshots of the blockchain and read/write it in chunks from/to disk" +name = "snapshot" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" + +[dependencies] +account-db = { path = "../account-db" } +account-state = { path = "../account-state" } +blockchain = { package = "ethcore-blockchain", path = "../blockchain" } +bloom-journal = { package = "ethcore-bloom-journal", path = "../../util/bloom" } +bytes = { package = "parity-bytes", version = "0.1.0" } +client-traits = { path = "../client-traits" } +common-types = { path = "../types" } +crossbeam-utils = "0.6" +engine = { path = "../engine" } +ethcore-db = { path = "../db" } +ethcore-io = { path = "../../util/io" } +ethereum-types = "0.6.0" +ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" } +hash-db = "0.15.0" +itertools = "0.5" +journaldb = { path = "../../util/journaldb" } +keccak-hash = "0.2.0" +keccak-hasher = { path = "../../util/keccak-hasher" } +kvdb = "0.1.0" +log = "0.4.8" +num_cpus = "1.10.1" +parking_lot = "0.8.0" +rand = "0.6" +rand_xorshift = "0.1.1" +rlp = "0.4.2" +rlp_derive = { path = "../../util/rlp-derive" } +snappy = { package = "parity-snappy", version ="0.1.0" } +state-db = { path = "../state-db" } +trie-db = "0.15.0" +triehash = { package = "triehash-ethereum", version = "0.2", path = "../../util/triehash-ethereum" } + +[dev-dependencies] +accounts = { package = "ethcore-accounts", path = "../../accounts" } +engine = { path = "../engine", features = ["test-helpers"] } +env_logger = "0.5" +ethabi = "8.0" +ethabi-contract = "8.0" +ethabi-derive = "8.0" +ethcore = { path = "..", features = ["test-helpers"] } +ethkey = { path = "../../accounts/ethkey" } +kvdb-rocksdb = { version = "0.1.3" } +lazy_static = { version = "1.3" } +spec = { path = "../spec" } +tempdir = "0.3" +trie-standardmap = "0.15.0" diff --git a/ethcore/src/snapshot/account.rs b/ethcore/snapshot/src/account.rs similarity index 97% rename from ethcore/src/snapshot/account.rs rename to ethcore/snapshot/src/account.rs index 62d9ac71d..760576e37 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/snapshot/src/account.rs @@ -16,21 +16,23 @@ //! Account state encoding and decoding +use std::collections::HashSet; +use std::sync::atomic::Ordering; + use account_db::{AccountDB, AccountDBMut}; -use types::{ +use bytes::Bytes; +use common_types::{ basic_account::BasicAccount, snapshot::Progress, errors::SnapshotError as Error, }; -use bytes::Bytes; use ethereum_types::{H256, U256}; use ethtrie::{TrieDB, TrieDBMut}; -use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; use hash_db::HashDB; +use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; +use log::{trace, warn}; use rlp::{RlpStream, Rlp}; -use std::collections::HashSet; -use trie::{Trie, TrieMut}; -use std::sync::atomic::Ordering; +use trie_db::{Trie, TrieMut}; // An empty account -- these were replaced with RLP null data for a space optimization in v1. const ACC_EMPTY: BasicAccount = BasicAccount { @@ -246,12 +248,12 @@ pub fn from_fat_rlp( #[cfg(test)] mod tests { use account_db::{AccountDB, AccountDBMut}; - use types::basic_account::BasicAccount; - use test_helpers::get_temp_state_db; - use snapshot::tests::helpers::fill_storage; - use snapshot::Progress; + use common_types::basic_account::BasicAccount; + use ethcore::test_helpers::get_temp_state_db; + use crate::tests::helpers::fill_storage; + use common_types::snapshot::Progress; - use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; + use keccak_hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, Address}; use hash_db::{HashDB, EMPTY_PREFIX}; use kvdb::DBValue; diff --git a/ethcore/src/snapshot/block.rs b/ethcore/snapshot/src/block.rs similarity index 94% rename from ethcore/src/snapshot/block.rs rename to ethcore/snapshot/src/block.rs index f19b2611b..fbc5d3207 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/snapshot/src/block.rs @@ -17,18 +17,20 @@ //! Block RLP compression. use bytes::Bytes; +use common_types::{ + block::Block, + header::Header, + views::BlockView, +}; use ethereum_types::H256; -use hash::keccak; +use keccak_hash::keccak; use rlp::{DecoderError, RlpStream, Rlp}; use triehash::ordered_trie_root; -use types::block::Block; -use types::header::Header; -use types::views::BlockView; const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; -pub struct AbridgedBlock { +pub(crate) struct AbridgedBlock { rlp: Bytes, } @@ -122,11 +124,7 @@ impl AbridgedBlock { header.set_seal(seal_fields); - Ok(Block { - header: header, - transactions: transactions, - uncles: uncles, - }) + Ok(Block { header, transactions, uncles }) } } @@ -136,10 +134,12 @@ mod tests { use bytes::Bytes; use ethereum_types::{H256, U256, Address}; - use types::transaction::{Action, Transaction}; - use types::block::Block; - use types::view; - use types::views::BlockView; + use common_types::{ + transaction::{Action, Transaction}, + block::Block, + view, + views::BlockView, + }; fn encode_block(b: &Block) -> Bytes { b.rlp_bytes() diff --git a/ethcore/src/snapshot/consensus/authority.rs b/ethcore/snapshot/src/consensus/authority.rs similarity index 99% rename from ethcore/src/snapshot/consensus/authority.rs rename to ethcore/snapshot/src/consensus/authority.rs index 426ca1e54..6135ba882 100644 --- a/ethcore/src/snapshot/consensus/authority.rs +++ b/ethcore/snapshot/src/consensus/authority.rs @@ -22,14 +22,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use engine::{Engine, EpochVerifier}; use blockchain::{BlockChain, BlockChainDB, BlockProvider}; use bytes::Bytes; -use ethereum_types::{H256, U256}; -use itertools::{Position, Itertools}; -use kvdb::KeyValueDB; -use rlp::{RlpStream, Rlp}; -use types::{ +use common_types::{ encoded, engines::epoch::Transition as EpochTransition, header::Header, @@ -38,8 +33,15 @@ use types::{ receipt::Receipt, snapshot::{ChunkSink, Progress, ManifestData} }; +use engine::{Engine, EpochVerifier}; +use ethereum_types::{H256, U256}; +use itertools::{Position, Itertools}; +use kvdb::KeyValueDB; +use log::trace; +use rlp::{RlpStream, Rlp}; + +use crate::{SnapshotComponents, Rebuilder}; -use snapshot::{SnapshotComponents, Rebuilder}; /// Snapshot creation and restoration for PoA chains. /// Chunk format: @@ -320,7 +322,7 @@ impl Rebuilder for ChunkRebuilder { } if is_last_chunk { - use types::block::Block; + use common_types::block::Block; let last_rlp = rlp.at(num_items - 1)?; let block = Block { diff --git a/ethcore/src/snapshot/consensus/mod.rs b/ethcore/snapshot/src/consensus/mod.rs similarity index 94% rename from ethcore/src/snapshot/consensus/mod.rs rename to ethcore/snapshot/src/consensus/mod.rs index aa323b6af..280ebd8a7 100644 --- a/ethcore/src/snapshot/consensus/mod.rs +++ b/ethcore/snapshot/src/consensus/mod.rs @@ -23,8 +23,8 @@ mod work; pub use self::authority::*; pub use self::work::*; -use snapshot::SnapshotComponents; -use types::snapshot::Snapshotting::{self, *}; +use crate::SnapshotComponents; +use common_types::snapshot::Snapshotting::{self, *}; /// Create a factory for building snapshot chunks and restoring from them. /// `None` indicates that the engine doesn't support snapshot creation. diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/snapshot/src/consensus/work.rs similarity index 95% rename from ethcore/src/snapshot/consensus/work.rs rename to ethcore/snapshot/src/consensus/work.rs index ee52f85ba..eb035fb77 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/snapshot/src/consensus/work.rs @@ -25,21 +25,27 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use blockchain::{BlockChain, BlockChainDB, BlockProvider}; -use engine::Engine; -use snapshot::block::AbridgedBlock; -use ethereum_types::H256; -use kvdb::KeyValueDB; use bytes::Bytes; -use rlp::{RlpStream, Rlp}; -use rand::rngs::OsRng; -use types::{ +use common_types::{ encoded, engines::epoch::Transition as EpochTransition, errors::{SnapshotError, EthcoreError}, snapshot::{ChunkSink, ManifestData, Progress}, + receipt::Receipt, }; +use engine::Engine; +use ethereum_types::{H256, U256}; +use kvdb::KeyValueDB; +use log::trace; +use rand::rngs::OsRng; +use rlp::{RlpStream, Rlp}; +use triehash::ordered_trie_root; -use snapshot::{SnapshotComponents, Rebuilder}; +use crate::{ + SnapshotComponents, Rebuilder, + block::AbridgedBlock, + verify_old_block +}; /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a @@ -93,8 +99,8 @@ impl SnapshotComponents for PowSnapshot { ).map(|r| Box::new(r) as Box<_>) } - fn min_supported_version(&self) -> u64 { ::snapshot::MIN_SUPPORTED_STATE_CHUNK_VERSION } - fn current_version(&self) -> u64 { ::snapshot::STATE_CHUNK_VERSION } + fn min_supported_version(&self) -> u64 { crate::MIN_SUPPORTED_STATE_CHUNK_VERSION } + fn current_version(&self) -> u64 { crate::STATE_CHUNK_VERSION } } /// Used to build block chunks. @@ -231,10 +237,6 @@ impl Rebuilder for PowRebuilder { /// Feed the rebuilder an uncompressed block chunk. /// Returns the number of blocks fed or any errors. fn feed(&mut self, chunk: &[u8], engine: &dyn Engine, abort_flag: &AtomicBool) -> Result<(), EthcoreError> { - use snapshot::verify_old_block; - use ethereum_types::U256; - use triehash::ordered_trie_root; - let rlp = Rlp::new(chunk); let item_count = rlp.item_count()?; let num_blocks = (item_count - 3) as u64; @@ -256,7 +258,7 @@ impl Rebuilder for PowRebuilder { let pair = rlp.at(idx)?; let abridged_rlp = pair.at(0)?.as_raw().to_owned(); let abridged_block = AbridgedBlock::from_raw(abridged_rlp); - let receipts: Vec<::types::receipt::Receipt> = pair.list_at(1)?; + let receipts: Vec = pair.list_at(1)?; let receipts_root = ordered_trie_root(pair.at(1)?.iter().map(|r| r.as_raw())); let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; diff --git a/ethcore/src/snapshot/io.rs b/ethcore/snapshot/src/io.rs similarity index 98% rename from ethcore/src/snapshot/io.rs rename to ethcore/snapshot/src/io.rs index b996651bc..1afba21fb 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/snapshot/src/io.rs @@ -27,16 +27,17 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use client_traits::SnapshotWriter; -use ethereum_types::H256; -use rlp::{RlpStream, Rlp}; -use types::{ +use common_types::{ errors::{SnapshotError, EthcoreError}, snapshot::ManifestData, }; +use ethereum_types::H256; +use log::trace; +use rlp::{RlpStream, Rlp}; +use rlp_derive::*; const SNAPSHOT_VERSION: u64 = 2; - // (hash, len, offset) #[derive(RlpEncodable, RlpDecodable)] struct ChunkInfo(H256, u64, u64); @@ -320,9 +321,9 @@ impl SnapshotReader for LooseReader { #[cfg(test)] mod tests { use tempdir::TempDir; - use hash::keccak; + use keccak_hash::keccak; - use snapshot::ManifestData; + use common_types::snapshot::ManifestData; use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; @@ -351,8 +352,8 @@ mod tests { let manifest = ManifestData { version: SNAPSHOT_VERSION, - state_hashes: state_hashes, - block_hashes: block_hashes, + state_hashes, + block_hashes, state_root: keccak(b"notarealroot"), block_number: 12345678987654321, block_hash: keccak(b"notarealblock"), diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/snapshot/src/lib.rs similarity index 98% rename from ethcore/src/snapshot/mod.rs rename to ethcore/snapshot/src/lib.rs index 66a977196..76d48ba56 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/snapshot/src/lib.rs @@ -23,46 +23,44 @@ use std::collections::{HashMap, HashSet}; use std::cmp; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY}; + +use keccak_hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY}; use account_db::{AccountDB, AccountDBMut}; +use account_state::Account as StateAccount; use blockchain::{BlockChain, BlockProvider}; -use types::{ +use bloom_journal::Bloom; +use bytes::Bytes; +// todo[dvdplm] put back in snapshots once it's extracted +use client_traits::SnapshotWriter; +use common_types::{ ids::BlockId, header::Header, errors::{SnapshotError as Error, EthcoreError}, - snapshot::Progress, + snapshot::{Progress, ManifestData}, }; -use ethereum_types::{H256, U256}; -use hash_db::HashDB; -use keccak_hasher::KeccakHasher; -use snappy; -use bytes::Bytes; -use parking_lot::Mutex; -use journaldb::{self, Algorithm, JournalDB}; -use kvdb::{KeyValueDB, DBValue}; -use trie::{Trie, TrieMut}; -use ethtrie::{TrieDB, TrieDBMut}; -use rlp::{RlpStream, Rlp}; -use bloom_journal::Bloom; -use num_cpus; -use types::snapshot::ManifestData; - -// todo[dvdplm] put back in snapshots once it's extracted -use client_traits::SnapshotWriter; - -use super::state_db::StateDB; -use account_state::Account as StateAccount; -use engine::Engine; - use crossbeam_utils::thread; +use engine::Engine; +use ethereum_types::{H256, U256}; +use ethtrie::{TrieDB, TrieDBMut}; +use hash_db::HashDB; +use journaldb::{self, Algorithm, JournalDB}; +use keccak_hasher::KeccakHasher; +use parking_lot::Mutex; +use kvdb::{KeyValueDB, DBValue}; +use log::{debug, info, trace}; +use num_cpus; use rand::{Rng, rngs::OsRng}; +use rlp::{RlpStream, Rlp}; +use snappy; +use state_db::StateDB; +use trie_db::{Trie, TrieMut}; pub use self::consensus::*; pub use self::service::Service; pub use self::traits::{SnapshotService, SnapshotComponents, Rebuilder}; pub use self::watcher::Watcher; -pub use types::basic_account::BasicAccount; +pub use common_types::basic_account::BasicAccount; pub mod io; pub mod service; @@ -377,7 +375,7 @@ impl StateRebuilder { /// Create a new state rebuilder to write into the given backing DB. pub fn new(db: Arc, pruning: Algorithm) -> Self { StateRebuilder { - db: journaldb::new(db.clone(), pruning, ::db::COL_STATE), + db: journaldb::new(db.clone(), pruning, ethcore_db::COL_STATE), state_root: KECCAK_NULL_RLP, known_code: HashMap::new(), missing_code: HashMap::new(), diff --git a/ethcore/src/snapshot/service.rs b/ethcore/snapshot/src/service.rs similarity index 96% rename from ethcore/src/snapshot/service.rs rename to ethcore/snapshot/src/service.rs index 6ca50d706..747c987db 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/snapshot/src/service.rs @@ -24,6 +24,27 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::cmp; +use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; +use bytes::Bytes; +use common_types::{ + io_message::ClientIoMessage, + errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, + ids::BlockId, + snapshot::{ManifestData, Progress, RestorationStatus}, +}; +// todo[dvdplm] put SnapshotWriter back in snapshots once extracted +use client_traits::{ChainInfo, SnapshotClient, SnapshotWriter}; +use engine::Engine; +use ethereum_types::H256; +use ethcore_io::IoChannel; +use journaldb::Algorithm; +use keccak_hash::keccak; +use kvdb::DBTransaction; +use log::{error, info, trace, warn}; +use parking_lot::{Mutex, RwLock, RwLockReadGuard}; +use snappy; +use trie_db::TrieError; + use super::{ StateRebuilder, SnapshotService, @@ -33,31 +54,6 @@ use super::{ chunker, }; -use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; -use client::Client; -// todo[dvdplm] put SnapshotWriter back in snapshots once extracted -use client_traits::{ - BlockInfo, BlockChainClient, ChainInfo, - SnapshotClient, SnapshotWriter, DatabaseRestore, -}; -use engine::Engine; -use hash::keccak; -use types::{ - io_message::ClientIoMessage, - errors::{EthcoreError as Error, SnapshotError, SnapshotError::UnlinkedAncientBlockChain}, - ids::BlockId, - snapshot::{ManifestData, Progress, RestorationStatus}, -}; - -use io::IoChannel; - -use ethereum_types::H256; -use parking_lot::{Mutex, RwLock, RwLockReadGuard}; -use bytes::Bytes; -use journaldb::Algorithm; -use kvdb::DBTransaction; -use snappy; - /// Helper for removing directories in case of error. struct Guard(bool, PathBuf); @@ -140,7 +136,7 @@ impl Restoration { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); - return Err(::snapshot::Error::ChunkTooLarge.into()); + return Err(SnapshotError::ChunkTooLarge.into()); } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; @@ -162,7 +158,7 @@ impl Restoration { let expected_len = snappy::decompressed_len(chunk)?; if expected_len > MAX_CHUNK_SIZE { trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE); - return Err(::snapshot::Error::ChunkTooLarge.into()); + return Err(SnapshotError::ChunkTooLarge.into()); } let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?; @@ -179,8 +175,6 @@ impl Restoration { // finish up restoration. fn finalize(mut self) -> Result<(), Error> { - use trie::TrieError; - if !self.is_done() { return Ok(()) } // verify final state root. @@ -212,10 +206,10 @@ impl Restoration { } /// Type alias for client io channel. -pub type Channel = IoChannel>; +pub type Channel = IoChannel>; /// Snapshot service parameters. -pub struct ServiceParams { +pub struct ServiceParams { /// The consensus engine this is built on. pub engine: Arc, /// The chain's genesis block. @@ -225,21 +219,21 @@ pub struct ServiceParams { /// Handler for opening a restoration DB. pub restoration_db_handler: Box, /// Async IO channel for sending messages. - pub channel: Channel, + pub channel: Channel, /// The directory to put snapshots in. /// Usually "/snapshot" pub snapshot_root: PathBuf, /// A handle for database restoration. - pub client: Arc, + pub client: Arc, } /// `SnapshotService` implementation. /// This controls taking snapshots and restoring from them. -pub struct Service { +pub struct Service { restoration: Mutex>, restoration_db_handler: Box, snapshot_root: PathBuf, - io_channel: Mutex, + io_channel: Mutex>, pruning: Algorithm, status: Mutex, reader: RwLock>, @@ -247,15 +241,15 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, - client: Arc, + client: Arc, progress: Progress, taking_snapshot: AtomicBool, restoring_snapshot: AtomicBool, } -impl Service { +impl Service where C: SnapshotClient + ChainInfo { /// Create a new snapshot service from the given parameters. - pub fn new(params: ServiceParams) -> Result { + pub fn new(params: ServiceParams) -> Result { let mut service = Service { restoration: Mutex::new(None), restoration_db_handler: params.restoration_db_handler, @@ -479,10 +473,7 @@ impl Service { /// calling this while a restoration is in progress or vice versa /// will lead to a race condition where the first one to finish will /// have their produced snapshot overwritten. - pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> - where - C: ChainInfo + SnapshotClient - { + pub fn take_snapshot(&self, client: &C, num: u64) -> Result<(), Error> { if self.taking_snapshot.compare_and_swap(false, true, Ordering::SeqCst) { info!("Skipping snapshot at #{} as another one is currently in-progress.", num); return Ok(()); @@ -797,7 +788,7 @@ impl Service { } } -impl SnapshotService for Service { +impl SnapshotService for Service { fn manifest(&self) -> Option { self.reader.read().as_ref().map(|r| r.manifest().clone()) } @@ -807,10 +798,6 @@ impl SnapshotService for Service { .map(|c| (c.min_supported_version(), c.current_version())) } - fn chunk(&self, hash: H256) -> Option { - self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok()) - } - fn completed_chunks(&self) -> Option> { let restoration = self.restoration.lock(); @@ -833,6 +820,10 @@ impl SnapshotService for Service { } } + fn chunk(&self, hash: H256) -> Option { + self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok()) + } + fn status(&self) -> RestorationStatus { let mut cur_status = self.status.lock(); @@ -892,7 +883,7 @@ impl SnapshotService for Service { } } -impl Drop for Service { +impl Drop for Service { fn drop(&mut self) { trace!(target: "shutdown", "Dropping Service"); self.abort_restore(); @@ -904,18 +895,18 @@ impl Drop for Service { #[cfg(test)] mod tests { - use client::Client; - use io::{IoService}; + use ethcore::client::Client; + use ethcore_io::IoService; use spec; use journaldb::Algorithm; - use snapshot::SnapshotService; + use crate::SnapshotService; use super::*; - use types::{ + use common_types::{ io_message::ClientIoMessage, snapshot::{ManifestData, RestorationStatus} }; use tempdir::TempDir; - use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; + use ethcore::test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; #[test] fn sends_async_messages() { @@ -968,7 +959,7 @@ mod tests { let state_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); let block_hashes: Vec<_> = (0..5).map(|_| H256::random()).collect(); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let gb = spec.genesis_block(); let flag = ::std::sync::atomic::AtomicBool::new(true); diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/snapshot/src/tests/helpers.rs similarity index 91% rename from ethcore/src/snapshot/tests/helpers.rs rename to ethcore/snapshot/src/tests/helpers.rs index 3f23377a5..98b206ed4 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/snapshot/src/tests/helpers.rs @@ -17,32 +17,33 @@ //! Snapshot test helpers. These are used to build blockchains and state tries //! which can be queried before and after a full snapshot/restore cycle. -extern crate trie_standardmap; - use std::sync::Arc; -use hash::{KECCAK_NULL_RLP}; +use std::sync::atomic::AtomicBool; +use keccak_hash::{KECCAK_NULL_RLP}; use account_db::AccountDBMut; -use types::basic_account::BasicAccount; +use common_types::basic_account::BasicAccount; use blockchain::{BlockChain, BlockChainDB}; -use client::Client; +use ethcore::client::Client; use client_traits::{ChainInfo, SnapshotClient}; use engine::Engine; -use snapshot::{StateRebuilder}; -use snapshot::io::{SnapshotReader, PackedWriter, PackedReader}; +use crate::{ + StateRebuilder, + io::{SnapshotReader, PackedWriter, PackedReader}, +}; use tempdir::TempDir; use rand::Rng; - +use log::trace; use kvdb::DBValue; use ethereum_types::H256; use hash_db::HashDB; use keccak_hasher::KeccakHasher; use journaldb; -use trie::{TrieMut, Trie}; +use trie_db::{TrieMut, Trie}; use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; -use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; -use types::errors::EthcoreError; +use trie_standardmap::{Alphabet, StandardMap, ValueMode}; +use common_types::errors::EthcoreError; // the proportion of accounts we will alter each tick. const ACCOUNT_CHURN: f32 = 0.01; @@ -135,7 +136,7 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { /// Take a snapshot from the given client into a temporary file. /// Return a snapshot reader for it. pub fn snap(client: &Client) -> (Box, TempDir) { - use types::ids::BlockId; + use common_types::ids::BlockId; let tempdir = TempDir::new("").unwrap(); let path = tempdir.path().join("file"); @@ -158,10 +159,8 @@ pub fn restore( reader: &dyn SnapshotReader, genesis: &[u8], ) -> Result<(), EthcoreError> { - use std::sync::atomic::AtomicBool; - let flag = AtomicBool::new(true); - let chunker = crate::snapshot::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); + let chunker = crate::chunker(engine.snapshot_mode()).expect("the engine used here supports snapshots"); let manifest = reader.manifest(); let mut state = StateRebuilder::new(db.key_value().clone(), journaldb::Algorithm::Archive); diff --git a/ethcore/src/snapshot/tests/mod.rs b/ethcore/snapshot/src/tests/mod.rs similarity index 96% rename from ethcore/src/snapshot/tests/mod.rs rename to ethcore/snapshot/src/tests/mod.rs index f25fd03b2..c1139e7b5 100644 --- a/ethcore/src/snapshot/tests/mod.rs +++ b/ethcore/snapshot/src/tests/mod.rs @@ -23,7 +23,7 @@ mod service; pub mod helpers; -use super::ManifestData; +use common_types::snapshot::ManifestData; #[test] fn manifest_rlp() { diff --git a/ethcore/src/snapshot/tests/proof_of_authority.rs b/ethcore/snapshot/src/tests/proof_of_authority.rs similarity index 94% rename from ethcore/src/snapshot/tests/proof_of_authority.rs rename to ethcore/snapshot/src/tests/proof_of_authority.rs index a0161acdf..7b9437625 100644 --- a/ethcore/src/snapshot/tests/proof_of_authority.rs +++ b/ethcore/snapshot/src/tests/proof_of_authority.rs @@ -21,26 +21,32 @@ use std::sync::Arc; use std::str::FromStr; use accounts::AccountProvider; -use client::Client; +use ethcore::client::Client; use client_traits::{BlockChainClient, ChainInfo}; use ethkey::Secret; -use snapshot::tests::helpers as snapshot_helpers; +use crate::tests::helpers as snapshot_helpers; use spec::Spec; -use test_helpers::generate_dummy_client_with_spec; -use types::transaction::{Transaction, Action, SignedTransaction}; +use ethcore::test_helpers::generate_dummy_client_with_spec; +use common_types::transaction::{Transaction, Action, SignedTransaction}; use tempdir::TempDir; - +use log::trace; use ethereum_types::Address; -use test_helpers; +use ethcore::{ + test_helpers, + miner::{self, MinerService}, +}; +use keccak_hash::keccak; +use ethabi_contract::use_contract; +use lazy_static::lazy_static; -use_contract!(test_validator_set, "res/contracts/test_validator_set.json"); +use_contract!(test_validator_set, "../res/contracts/test_validator_set.json"); const PASS: &'static str = ""; const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated. const TRANSITION_BLOCK_2: usize = 10; // block at which the second contract activates. macro_rules! secret { - ($e: expr) => { Secret::from($crate::hash::keccak($e).0) } + ($e: expr) => { Secret::from(keccak($e).0) } } lazy_static! { @@ -100,8 +106,6 @@ fn make_chain(accounts: Arc, blocks_beyond: usize, transitions: { // push a block with given number, signed by one of the signers, with given transactions. let push_block = |signers: &[Address], n, txs: Vec| { - use miner::{self, MinerService}; - let idx = n as usize % signers.len(); trace!(target: "snapshot", "Pushing block #{}, {} txs, author={}", n, txs.len(), signers[idx]); diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/snapshot/src/tests/proof_of_work.rs similarity index 88% rename from ethcore/src/snapshot/tests/proof_of_work.rs rename to ethcore/snapshot/src/tests/proof_of_work.rs index 3b6106313..53dc07142 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/snapshot/src/tests/proof_of_work.rs @@ -18,25 +18,29 @@ use std::sync::atomic::AtomicBool; use tempdir::TempDir; -use types::{ +use common_types::{ errors::EthcoreError as Error, engines::ForkChoice, - snapshot::Progress, + snapshot::{Progress, ManifestData}, }; use blockchain::generator::{BlockGenerator, BlockBuilder}; use blockchain::{BlockChain, ExtrasInsert}; use client_traits::SnapshotWriter; -use snapshot::{chunk_secondary, Error as SnapshotError, SnapshotComponents}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; - +use crate::{ + chunk_secondary, + Error as SnapshotError, SnapshotComponents, + io::{PackedReader, PackedWriter, SnapshotReader}, + PowSnapshot, +}; use parking_lot::Mutex; use snappy; +use keccak_hash::KECCAK_NULL_RLP; use kvdb::DBTransaction; -use test_helpers; +use ethcore::test_helpers; use spec; -const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: 30000, max_restore_blocks: 30000 }; +const SNAPSHOT_MODE: PowSnapshot = PowSnapshot { blocks: 30000, max_restore_blocks: 30000 }; fn chunk_and_restore(amount: u64) { let genesis = BlockBuilder::genesis(); @@ -75,11 +79,11 @@ fn chunk_and_restore(amount: u64) { &Progress::default() ).unwrap(); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes: Vec::new(), - block_hashes: block_hashes, - state_root: ::hash::KECCAK_NULL_RLP, + block_hashes, + state_root: KECCAK_NULL_RLP, block_number: amount, block_hash: best_hash, }; @@ -137,11 +141,11 @@ fn checks_flag() { let engine = spec::new_test().engine; let chain = BlockChain::new(Default::default(), genesis.last().encoded().raw(), db.clone()); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes: Vec::new(), block_hashes: Vec::new(), - state_root: ::hash::KECCAK_NULL_RLP, + state_root: KECCAK_NULL_RLP, block_number: 102, block_hash: H256::zero(), }; diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/snapshot/src/tests/service.rs similarity index 91% rename from ethcore/src/snapshot/tests/service.rs rename to ethcore/snapshot/src/tests/service.rs index aed14e9b4..05a2322b8 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/snapshot/src/tests/service.rs @@ -21,22 +21,28 @@ use std::sync::Arc; use tempdir::TempDir; use blockchain::BlockProvider; -use client::{Client, ClientConfig}; +use ethcore::client::{Client, ClientConfig}; use client_traits::{BlockInfo, ImportBlock, SnapshotWriter}; -use types::{ +use common_types::{ ids::BlockId, snapshot::Progress, verification::Unverified, snapshot::{ManifestData, RestorationStatus}, }; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; -use snapshot::service::{Service, ServiceParams}; -use snapshot::{chunk_state, chunk_secondary, SnapshotService}; +use crate::{ + chunk_state, chunk_secondary, SnapshotService, + io::{PackedReader, PackedWriter, SnapshotReader}, + service::{Service, ServiceParams}, + PowSnapshot, +}; use spec; -use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use ethcore::{ + miner, + test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler} +}; use parking_lot::Mutex; -use io::IoChannel; +use ethcore_io::IoChannel; use kvdb_rocksdb::DatabaseConfig; #[test] @@ -53,7 +59,7 @@ fn restored_is_equivalent() { let client_db = tempdir.path().join("client_db"); let path = tempdir.path().join("snapshot"); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let restoration = restoration_db_handler(db_config); let blockchain_db = restoration.open(&client_db).unwrap(); @@ -62,7 +68,7 @@ fn restored_is_equivalent() { Default::default(), &spec, blockchain_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); @@ -118,7 +124,7 @@ fn guards_delete_folders() { let service_params = ServiceParams { engine: spec.engine.clone(), genesis_block: spec.genesis_block(), - restoration_db_handler: restoration_db_handler(DatabaseConfig::with_columns(::db::NUM_COLUMNS)), + restoration_db_handler: restoration_db_handler(DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS)), pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: tempdir.path().to_owned(), @@ -157,11 +163,10 @@ fn guards_delete_folders() { #[test] fn keep_ancient_blocks() { let _ = ::env_logger::try_init(); - // Test variables const NUM_BLOCKS: u64 = 500; const NUM_SNAPSHOT_BLOCKS: u64 = 300; - const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; + const SNAPSHOT_MODE: PowSnapshot = PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; // Temporary folders let tempdir = TempDir::new("").unwrap(); @@ -197,7 +202,7 @@ fn keep_ancient_blocks() { 0 ).unwrap(); - let manifest = ::snapshot::ManifestData { + let manifest = ManifestData { version: 2, state_hashes, state_root, @@ -209,13 +214,13 @@ fn keep_ancient_blocks() { writer.into_inner().finish(manifest.clone()).unwrap(); // Initialize the Client - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let client_db = new_temp_db(&tempdir.path()); let client2 = Client::new( ClientConfig::default(), &spec, client_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); @@ -283,13 +288,13 @@ fn recover_aborted_recovery() { let spec = spec::new_null(); let tempdir = TempDir::new("").unwrap(); - let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let client_db = new_db(); let client2 = Client::new( Default::default(), &spec, client_db, - Arc::new(::miner::Miner::new_for_tests(&spec, None)), + Arc::new(miner::Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ).unwrap(); let service_params = ServiceParams { diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/snapshot/src/tests/state.rs similarity index 87% rename from ethcore/src/snapshot/tests/state.rs rename to ethcore/snapshot/src/tests/state.rs index 50a505e3e..0be365549 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/snapshot/src/tests/state.rs @@ -18,17 +18,20 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; -use hash::{KECCAK_NULL_RLP, keccak}; +use keccak_hash::{KECCAK_NULL_RLP, keccak}; -use types::{ +use common_types::{ basic_account::BasicAccount, errors::EthcoreError as Error, + snapshot::ManifestData, }; use client_traits::SnapshotWriter; -use snapshot::account; -use snapshot::{chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}; -use snapshot::io::{PackedReader, PackedWriter, SnapshotReader}; -use super::helpers::StateProducer; +use crate::{ + account, + {chunk_state, Error as SnapshotError, Progress, StateRebuilder, SNAPSHOT_SUBPARTS}, + io::{PackedReader, PackedWriter, SnapshotReader}, + tests::helpers::StateProducer, +}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use ethereum_types::H256; @@ -45,7 +48,7 @@ fn snap_and_restore() { let mut producer = StateProducer::new(); let mut rng = XorShiftRng::from_seed(RNG_SEED); let mut old_db = journaldb::new_memory_db(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); for _ in 0..150 { producer.tick(&mut rng, &mut old_db); @@ -63,11 +66,11 @@ fn snap_and_restore() { state_hashes.append(&mut hashes); } - writer.into_inner().finish(::snapshot::ManifestData { + writer.into_inner().finish(ManifestData { version: 2, - state_hashes: state_hashes, + state_hashes, block_hashes: Vec::new(), - state_root: state_root, + state_root, block_number: 1000, block_hash: H256::zero(), }).unwrap(); @@ -82,7 +85,7 @@ fn snap_and_restore() { for chunk_hash in &reader.manifest().state_hashes { let raw = reader.chunk(*chunk_hash).unwrap(); - let chunk = ::snappy::decompress(&raw).unwrap(); + let chunk = snappy::decompress(&raw).unwrap(); rebuilder.feed(&chunk, &flag).unwrap(); } @@ -93,7 +96,7 @@ fn snap_and_restore() { new_db }; - let new_db = journaldb::new(db, Algorithm::OverlayRecent, ::db::COL_STATE); + let new_db = journaldb::new(db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); assert_eq!(new_db.earliest_era(), Some(1000)); let keys = old_db.keys(); @@ -141,7 +144,7 @@ fn get_code_from_prev_chunk() { let chunk2 = make_chunk(acc, h2); let tempdir = TempDir::new("").unwrap(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); let new_db = Arc::new(Database::open(&db_cfg, tempdir.path().to_str().unwrap()).unwrap()); { @@ -154,7 +157,7 @@ fn get_code_from_prev_chunk() { rebuilder.finalize(1000, H256::random()).unwrap(); } - let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ::db::COL_STATE); + let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ethcore_db::COL_STATE); assert_eq!(state_db.earliest_era(), Some(1000)); } @@ -163,7 +166,7 @@ fn checks_flag() { let mut producer = StateProducer::new(); let mut rng = XorShiftRng::from_seed(RNG_SEED); let mut old_db = journaldb::new_memory_db(); - let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let db_cfg = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS); for _ in 0..10 { producer.tick(&mut rng, &mut old_db); @@ -177,7 +180,7 @@ fn checks_flag() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default(), None, 0).unwrap(); - writer.into_inner().finish(::snapshot::ManifestData { + writer.into_inner().finish(ManifestData { version: 2, state_hashes, block_hashes: Vec::new(), @@ -197,7 +200,7 @@ fn checks_flag() { for chunk_hash in &reader.manifest().state_hashes { let raw = reader.chunk(*chunk_hash).unwrap(); - let chunk = ::snappy::decompress(&raw).unwrap(); + let chunk = snappy::decompress(&raw).unwrap(); match rebuilder.feed(&chunk, &flag) { Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {}, diff --git a/ethcore/src/snapshot/tests/test_validator_contract.json b/ethcore/snapshot/src/tests/test_validator_contract.json similarity index 100% rename from ethcore/src/snapshot/tests/test_validator_contract.json rename to ethcore/snapshot/src/tests/test_validator_contract.json diff --git a/ethcore/src/snapshot/traits.rs b/ethcore/snapshot/src/traits.rs similarity index 99% rename from ethcore/src/snapshot/traits.rs rename to ethcore/snapshot/src/traits.rs index 3eb2542bc..7425ada63 100644 --- a/ethcore/src/snapshot/traits.rs +++ b/ethcore/snapshot/src/traits.rs @@ -18,11 +18,12 @@ use std::sync::{Arc, atomic::AtomicBool}; use blockchain::{BlockChain, BlockChainDB}; use bytes::Bytes; -use ethereum_types::H256; -use types::{ +use common_types::{ errors::{EthcoreError as Error, SnapshotError}, snapshot::{ManifestData, ChunkSink, Progress, RestorationStatus}, }; +use engine::Engine; +use ethereum_types::H256; /// The interface for a snapshot network service. @@ -69,8 +70,6 @@ pub trait SnapshotService : Sync + Send { fn shutdown(&self); } -use crate::engine::Engine; - /// Restore from secondary snapshot chunks. pub trait Rebuilder: Send { /// Feed a chunk, potentially out of order. diff --git a/ethcore/src/snapshot/watcher.rs b/ethcore/snapshot/src/watcher.rs similarity index 87% rename from ethcore/src/snapshot/watcher.rs rename to ethcore/snapshot/src/watcher.rs index 828fb8d4d..864efd356 100644 --- a/ethcore/src/snapshot/watcher.rs +++ b/ethcore/snapshot/src/watcher.rs @@ -16,18 +16,18 @@ //! Watcher for snapshot-related chain events. -use parking_lot::Mutex; -use client::{Client, ChainNotify, NewBlocks}; -use client_traits::BlockInfo; -use types::{ +use std::sync::Arc; + +use client_traits::{BlockInfo, ChainNotify}; +use common_types::{ ids::BlockId, io_message::ClientIoMessage, + chain_notify::NewBlocks, }; - -use io::IoChannel; use ethereum_types::H256; - -use std::sync::Arc; +use ethcore_io::IoChannel; +use log::{trace, warn}; +use parking_lot::Mutex; // helper trait for transforming hashes to numbers and checking if syncing. trait Oracle: Send + Sync { @@ -37,7 +37,7 @@ trait Oracle: Send + Sync { } struct StandardOracle where F: 'static + Send + Sync + Fn() -> bool { - client: Arc, + client: Arc, sync_status: F, } @@ -58,7 +58,7 @@ trait Broadcast: Send + Sync { fn take_at(&self, num: Option); } -impl Broadcast for Mutex>> { +impl Broadcast for Mutex>> { fn take_at(&self, num: Option) { let num = match num { Some(n) => n, @@ -86,17 +86,16 @@ impl Watcher { /// Create a new `Watcher` which will trigger a snapshot event /// once every `period` blocks, but only after that block is /// `history` blocks old. - pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self - where F: 'static + Send + Sync + Fn() -> bool + pub fn new(client: Arc, sync_status: F, channel: IoChannel>, period: u64, history: u64) -> Self + where + F: 'static + Send + Sync + Fn() -> bool, + C: 'static + Send + Sync, { Watcher { - oracle: Box::new(StandardOracle { - client: client, - sync_status: sync_status, - }), + oracle: Box::new(StandardOracle { client, sync_status }), broadcast: Box::new(Mutex::new(channel)), - period: period, - history: history, + period, + history, } } } @@ -123,14 +122,15 @@ impl ChainNotify for Watcher { #[cfg(test)] mod tests { - use super::{Broadcast, Oracle, Watcher}; + use std::collections::HashMap; + use std::time::Duration; - use client::{ChainNotify, NewBlocks, ChainRoute}; + use client_traits::ChainNotify; + use common_types::chain_notify::{NewBlocks, ChainRoute}; use ethereum_types::{H256, U256, BigEndianHash}; - use std::collections::HashMap; - use std::time::Duration; + use super::{Broadcast, Oracle, Watcher}; struct TestOracle(HashMap); @@ -161,8 +161,8 @@ mod tests { let watcher = Watcher { oracle: Box::new(TestOracle(map)), broadcast: Box::new(TestBroadcast(expected)), - period: period, - history: history, + period, + history, }; watcher.new_blocks(NewBlocks::new( diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index bb4855e18..000a0cd75 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -21,7 +21,7 @@ use std::sync::{Arc, Weak}; use std::time::{Instant, Duration}; use account_state::state::StateInfo; -use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert, BlockNumberKey}; +use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, TransactionAddress, ExtrasInsert, BlockNumberKey}; use bytes::Bytes; use call_contract::{CallContract, RegistryInfo}; use ethcore_miner::pool::VerifiedTransaction; @@ -41,15 +41,15 @@ use block::{LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBl use client::ancient_import::AncientVerifier; use client::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, - Call, BlockProducer, SealedBlockImporter, ChainNotify, EngineInfo, - ClientConfig, NewBlocks, ChainRoute, ChainMessageType, bad_blocks, + Call, BlockProducer, SealedBlockImporter, EngineInfo, + ClientConfig, bad_blocks, }; use client_traits::{ BlockInfo, ScheduleInfo, StateClient, BlockChainReset, Nonce, Balance, ChainInfo, TransactionInfo, ImportBlock, AccountData, BlockChain as BlockChainTrait, BlockChainClient, IoClient, BadBlocks, ProvingBlockChainClient, SnapshotClient, - DatabaseRestore, SnapshotWriter, Tick, + DatabaseRestore, SnapshotWriter, Tick, ChainNotify, StateOrBlock, }; use engine::Engine; @@ -72,7 +72,9 @@ use types::{ block::PreverifiedBlock, block_status::BlockStatus, blockchain_info::BlockChainInfo, + chain_notify::{NewBlocks, ChainRoute, ChainMessageType}, client_types::ClientReport, + import_route::ImportRoute, io_message::ClientIoMessage, encoded, engines::{ @@ -986,12 +988,14 @@ impl Client { self.importer.miner.clone() } - #[cfg(test)] + /// Access state from tests + #[cfg(any(test, feature = "test-helpers"))] pub fn state_db(&self) -> ::parking_lot::RwLockReadGuard { self.state_db.read() } - #[cfg(test)] + /// Access the BlockChain from tests + #[cfg(any(test, feature = "test-helpers"))] pub fn chain(&self) -> Arc { self.chain.read().clone() } diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 58b8b7ffa..c6b338444 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -31,7 +31,6 @@ pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockChainConfig pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; #[cfg(any(test, feature = "test-helpers"))] pub use self::test_client::{TestBlockChainClient, EachBlockWith, TestState}; -pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType}; pub use self::traits::{ ReopenBlock, PrepareOpenBlock, ImportSealedBlock, BroadcastProposalBlock, Call, EngineInfo, BlockProducer, SealedBlockImporter, @@ -40,4 +39,3 @@ pub use self::traits::{ pub use verification::VerifierType; mod traits; -mod chain_notify; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 8beaf991f..727665a6c 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -53,16 +53,13 @@ //! cargo build --release //! ``` -extern crate account_db; extern crate account_state; extern crate ansi_term; extern crate client_traits; extern crate common_types as types; -extern crate crossbeam_utils; extern crate engine; extern crate ethabi; extern crate ethcore_blockchain as blockchain; -extern crate ethcore_bloom_journal as bloom_journal; extern crate ethcore_call_contract as call_contract; extern crate ethcore_db as db; extern crate ethcore_io as io; @@ -74,13 +71,10 @@ extern crate hash_db; extern crate itertools; extern crate journaldb; extern crate keccak_hash as hash; -extern crate keccak_hasher; extern crate kvdb; extern crate machine; extern crate memory_cache; -extern crate num_cpus; extern crate parity_bytes as bytes; -extern crate parity_snappy as snappy; extern crate parking_lot; extern crate trie_db as trie; extern crate patricia_trie_ethereum as ethtrie; @@ -88,6 +82,7 @@ extern crate rand; extern crate rayon; extern crate rlp; extern crate serde; +extern crate snapshot; extern crate spec; extern crate state_db; extern crate trace; @@ -107,6 +102,8 @@ extern crate ethcore_stratum; #[cfg(any(test, feature = "stratum"))] extern crate ethash; +#[cfg(any(test, feature = "test-helpers"))] +extern crate account_db; #[cfg(any(test, feature = "test-helpers"))] extern crate ethkey; #[cfg(any(test, feature = "test-helpers"))] @@ -157,7 +154,6 @@ extern crate parity_runtime; pub mod block; pub mod client; pub mod miner; -pub mod snapshot; #[cfg(test)] mod tests; diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index a89b342de..30484bf0d 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -34,6 +34,7 @@ use parking_lot::RwLock; use rlp::{self, RlpStream}; use tempdir::TempDir; use types::{ + chain_notify::ChainMessageType, transaction::{Action, Transaction, SignedTransaction}, encoded, engines::ForkChoice, @@ -44,8 +45,8 @@ use types::{ }; use block::{OpenBlock, Drain}; -use client::{Client, ClientConfig, ChainNotify, ChainMessageType, PrepareOpenBlock}; -use client_traits::{ChainInfo, ImportBlock}; +use client::{Client, ClientConfig, PrepareOpenBlock}; +use client_traits::{ChainInfo, ChainNotify, ImportBlock}; use trie_vm_factories::Factories; use miner::Miner; use spec::{Spec, self}; diff --git a/ethcore/state-db/src/lib.rs b/ethcore/state-db/src/lib.rs index 67d70343a..7b121dfb7 100644 --- a/ethcore/state-db/src/lib.rs +++ b/ethcore/state-db/src/lib.rs @@ -207,9 +207,9 @@ impl StateDB { /// Journal all recent operations under the given era and ID. pub fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> io::Result { { - let mut bloom_lock = self.account_bloom.lock(); - Self::commit_bloom(batch, bloom_lock.drain_journal())?; - } + let mut bloom_lock = self.account_bloom.lock(); + Self::commit_bloom(batch, bloom_lock.drain_journal())?; + } let records = self.db.journal_under(batch, now, id)?; self.commit_hash = Some(id.clone()); self.commit_number = Some(now); diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 721614887..c60b2cb03 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -17,11 +17,12 @@ ethcore-io = { path = "../../util/io" } ethcore-light = { path = "../light" } ethcore-network = { path = "../../util/network" } ethcore-network-devp2p = { path = "../../util/network-devp2p" } -ethereum-types = "0.6.0" ethcore-private-tx = { path = "../private-tx" } +ethereum-types = "0.6.0" ethkey = { path = "../../accounts/ethkey" } ethstore = { path = "../../accounts/ethstore" } fastmap = { path = "../../util/fastmap" } +futures = "0.1" hash-db = "0.15.0" keccak-hash = "0.2.0" keccak-hasher = { path = "../../util/keccak-hasher" } @@ -30,20 +31,20 @@ log = "0.4" machine = { path = "../machine" } macros = { path = "../../util/macros" } parity-bytes = "0.1" -parking_lot = "0.8" +parity-runtime = { path = "../../util/runtime" } parity-util-mem = "0.2.0" +parking_lot = "0.8" rand = "0.6" rlp = "0.4.0" +snapshot = { path = "../snapshot" } trace-time = "0.1" triehash-ethereum = {version = "0.2", path = "../../util/triehash-ethereum" } -futures = "0.1" -parity-runtime = { path = "../../util/runtime" } [dev-dependencies] env_logger = "0.5" ethcore = { path = "..", features = ["test-helpers"] } ethcore-io = { path = "../../util/io", features = ["mio"] } kvdb-memorydb = "0.1" -rustc-hex = "1.0" rand_xorshift = "0.1.1" +rustc-hex = "1.0" spec = { path = "../spec" } diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index eeb6cb41d..75f9634ad 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -25,16 +25,13 @@ use network::{NetworkProtocolHandler, NetworkContext, PeerId, ProtocolId, NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, Error, ConnectionFilter}; use network::client_version::ClientVersion; - -use types::pruning_info::PruningInfo; use ethereum_types::{H256, H512, U256}; use futures::sync::mpsc as futures_mpsc; use futures::Stream; use io::{TimerToken}; use ethkey::Secret; -use ethcore::client::{ChainNotify, NewBlocks, ChainMessageType}; -use client_traits::BlockChainClient; -use ethcore::snapshot::SnapshotService; +use client_traits::{BlockChainClient, ChainNotify}; +use snapshot::SnapshotService; use ethcore_private_tx::PrivateStateDB; use types::BlockNumber; use sync_io::NetSyncIo; @@ -55,7 +52,11 @@ use parity_runtime::Executor; use std::sync::atomic::{AtomicBool, Ordering}; use network::IpFilter; use private_tx::PrivateTxHandler; -use types::transaction::UnverifiedTransaction; +use types::{ + chain_notify::{NewBlocks, ChainMessageType}, + pruning_info::PruningInfo, + transaction::UnverifiedTransaction, +}; use super::light_sync::SyncInfo; diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index e7a54fa95..8c0ab8741 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -23,10 +23,12 @@ use hash::keccak; use network::PeerId; use network::client_version::ClientVersion; use rlp::Rlp; -use snapshot::ChunkType; +use crate::{ + snapshot_sync::ChunkType, + sync_io::SyncIo, +}; use std::time::Instant; use std::{mem, cmp}; -use sync_io::SyncIo; use types::{ BlockNumber, block_status::BlockStatus, @@ -711,7 +713,7 @@ impl SyncHandler { Err(e) => { trace!(target: "privatetx", "Ignoring the message, error queueing: {}", e); } - } + } Ok(()) } @@ -739,7 +741,7 @@ impl SyncHandler { Err(e) => { trace!(target: "privatetx", "Ignoring the message, error queueing: {}", e); } - } + } Ok(()) } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 07be14f7f..ed0f86716 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -109,11 +109,13 @@ use rlp::{RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use network::client_version::ClientVersion; use client_traits::BlockChainClient; -use sync_io::SyncIo; +use crate::{ + sync_io::SyncIo, + snapshot_sync::Snapshot, +}; use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, DownloadAction}; use rand::{Rng, seq::SliceRandom}; -use snapshot::{Snapshot}; use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID, PriorityTask}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 32baddec0..54e013fd4 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -27,19 +27,20 @@ extern crate ethcore; extern crate ethcore_io as io; extern crate ethcore_network as network; extern crate ethcore_network_devp2p as devp2p; +extern crate ethcore_private_tx; extern crate ethereum_types; extern crate ethkey; extern crate ethstore; extern crate fastmap; +extern crate futures; extern crate keccak_hash as hash; extern crate parity_bytes as bytes; extern crate parity_runtime; extern crate parking_lot; -extern crate ethcore_private_tx; extern crate rand; extern crate rlp; +extern crate snapshot; extern crate triehash_ethereum; -extern crate futures; extern crate ethcore_light as light; @@ -69,7 +70,7 @@ mod blocks; mod block_sync; mod sync_io; mod private_tx; -mod snapshot; +mod snapshot_sync; mod transactions_stats; pub mod light_sync; diff --git a/ethcore/sync/src/snapshot.rs b/ethcore/sync/src/snapshot_sync.rs similarity index 99% rename from ethcore/sync/src/snapshot.rs rename to ethcore/sync/src/snapshot_sync.rs index 882ee1880..6aa007380 100644 --- a/ethcore/sync/src/snapshot.rs +++ b/ethcore/sync/src/snapshot_sync.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethereum_types::H256; use hash::keccak; use types::snapshot::ManifestData; diff --git a/ethcore/sync/src/sync_io.rs b/ethcore/sync/src/sync_io.rs index 5c974e712..0f92220af 100644 --- a/ethcore/sync/src/sync_io.rs +++ b/ethcore/sync/src/sync_io.rs @@ -23,7 +23,7 @@ use bytes::Bytes; use client_traits::BlockChainClient; use ethcore_private_tx::PrivateStateDB; use types::BlockNumber; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use parking_lot::RwLock; /// IO interface for the syncing handler. diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index 666d362eb..938569f56 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -22,11 +22,17 @@ use bytes::Bytes; use network::{self, PeerId, ProtocolId, PacketId, SessionInfo}; use network::client_version::ClientVersion; use tests::snapshot::*; -use types::io_message::ClientIoMessage; -use client_traits::BlockChainClient; -use ethcore::client::{TestBlockChainClient, Client as EthcoreClient, - ClientConfig, ChainNotify, NewBlocks, ChainMessageType}; -use ethcore::snapshot::SnapshotService; +use types::{ + chain_notify::{NewBlocks, ChainMessageType}, + io_message::ClientIoMessage, +}; +use client_traits::{BlockChainClient, ChainNotify}; +use ethcore::client::{ + TestBlockChainClient, + Client as EthcoreClient, + ClientConfig, +}; +use snapshot::SnapshotService; use spec::{self, Spec}; use ethcore_private_tx::PrivateStateDB; use ethcore::miner::Miner; diff --git a/ethcore/sync/src/tests/snapshot.rs b/ethcore/sync/src/tests/snapshot.rs index 8136dc50c..8152bc69c 100644 --- a/ethcore/sync/src/tests/snapshot.rs +++ b/ethcore/sync/src/tests/snapshot.rs @@ -20,7 +20,7 @@ use hash::keccak; use ethereum_types::H256; use parking_lot::Mutex; use bytes::Bytes; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethcore::client::EachBlockWith; use types::{ BlockNumber, diff --git a/ethcore/types/src/block.rs b/ethcore/types/src/block.rs index e110fc7dc..334ab691c 100644 --- a/ethcore/types/src/block.rs +++ b/ethcore/types/src/block.rs @@ -21,8 +21,10 @@ //! Other block types are found in `ethcore` use bytes::Bytes; +use ethereum_types::{H256, U256}; use parity_util_mem::MallocSizeOf; +use BlockNumber; use header::Header; use rlp::{Rlp, RlpStream, Decodable, DecoderError}; use transaction::{UnverifiedTransaction, SignedTransaction}; @@ -77,3 +79,40 @@ pub struct PreverifiedBlock { /// Block bytes pub bytes: Bytes, } + +/// Brief info about inserted block. +#[derive(Clone)] +pub struct BlockInfo { + /// Block hash. + pub hash: H256, + /// Block number. + pub number: BlockNumber, + /// Total block difficulty. + pub total_difficulty: U256, + /// Block location in blockchain. + pub location: BlockLocation +} + +/// Describes location of newly inserted block. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockLocation { + /// It's part of the canon chain. + CanonChain, + /// It's not a part of the canon chain. + Branch, + /// It's part of the fork which should become canon chain, + /// because its total difficulty is higher than current + /// canon chain difficulty. + BranchBecomingCanonChain(BranchBecomingCanonChainData), +} + +/// Info about heaviest fork +#[derive(Debug, Clone, PartialEq)] +pub struct BranchBecomingCanonChainData { + /// Hash of the newest common ancestor with old canon chain. + pub ancestor: H256, + /// Hashes of the blocks between ancestor and this block. + pub enacted: Vec, + /// Hashes of the blocks which were invalidated. + pub retracted: Vec, +} diff --git a/ethcore/src/client/chain_notify.rs b/ethcore/types/src/chain_notify.rs similarity index 79% rename from ethcore/src/client/chain_notify.rs rename to ethcore/types/src/chain_notify.rs index 1567f6c5c..fe41644ef 100644 --- a/ethcore/src/client/chain_notify.rs +++ b/ethcore/types/src/chain_notify.rs @@ -14,10 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . +//! Types pertaining to sending messages and finding routes through the chain. Used mostly by the +//! ChainNotify trait. + use bytes::Bytes; -use ethereum_types::{H256, U256}; -use types::transaction::UnverifiedTransaction; -use blockchain::ImportRoute; +use ethereum_types::H256; +use crate::{ + import_route::ImportRoute, +}; use std::time::Duration; use std::collections::HashMap; @@ -116,7 +120,8 @@ impl ChainRoute { } } -/// Used by `ChainNotify` `new_blocks()` +/// Used by `ChainNotify` `new_blocks()` and contains information about new blocks imported to the +/// chain. pub struct NewBlocks { /// Imported blocks pub imported: Vec, @@ -156,40 +161,3 @@ impl NewBlocks { } } } - -/// Represents what has to be handled by actor listening to chain events -pub trait ChainNotify : Send + Sync { - /// fires when chain has new blocks. - fn new_blocks(&self, _new_blocks: NewBlocks) { - // does nothing by default - } - - /// fires when chain achieves active mode - fn start(&self) { - // does nothing by default - } - - /// fires when chain achieves passive mode - fn stop(&self) { - // does nothing by default - } - - /// fires when chain broadcasts a message - fn broadcast(&self, _message_type: ChainMessageType) { - // does nothing by default - } - - /// fires when new block is about to be imported - /// implementations should be light - fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { - // does nothing by default - } - - /// fires when new transactions are received from a peer - fn transactions_received(&self, - _txs: &[UnverifiedTransaction], - _peer_id: usize, - ) { - // does nothing by default - } -} diff --git a/ethcore/types/src/client_types.rs b/ethcore/types/src/client_types.rs index a67eebcab..00fcd91be 100644 --- a/ethcore/types/src/client_types.rs +++ b/ethcore/types/src/client_types.rs @@ -17,11 +17,12 @@ //! Client related types. use std::{ + cmp, fmt::{Display, Formatter, Error as FmtError}, ops, - cmp, time::Duration, }; + use ethereum_types::U256; use crate::header::Header; @@ -88,4 +89,3 @@ impl<'a> ops::Sub<&'a ClientReport> for ClientReport { self } } - diff --git a/ethcore/blockchain/src/import_route.rs b/ethcore/types/src/import_route.rs similarity index 95% rename from ethcore/blockchain/src/import_route.rs rename to ethcore/types/src/import_route.rs index 46ebc254b..3261b418c 100644 --- a/ethcore/blockchain/src/import_route.rs +++ b/ethcore/types/src/import_route.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -//! Import route. +//! Calculate import route for newly inserted blocks. use ethereum_types::H256; -use crate::block_info::{BlockInfo, BlockLocation}; +use crate::block::{BlockInfo, BlockLocation}; /// Import route for newly inserted block. #[derive(Debug, PartialEq, Clone)] @@ -69,7 +69,7 @@ impl From for ImportRoute { #[cfg(test)] mod tests { use ethereum_types::{U256, BigEndianHash}; - use crate::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; + use crate::block::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use super::ImportRoute; #[test] diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index fb39ad6eb..3b5f1f68a 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -64,6 +64,7 @@ pub mod block; pub mod block_status; pub mod blockchain_info; pub mod call_analytics; +pub mod chain_notify; pub mod client_types; pub mod encoded; pub mod engines; @@ -72,6 +73,7 @@ pub mod filter; pub mod header; pub mod ids; pub mod io_message; +pub mod import_route; pub mod log_entry; pub mod pruning_info; pub mod receipt; diff --git a/parity/configuration.rs b/parity/configuration.rs index 2066d5908..d19e39fa4 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -31,7 +31,7 @@ use sync::{NetworkConfiguration, validate_node_url, self}; use ethkey::{Secret, Public}; use ethcore::client::{VMType}; use ethcore::miner::{stratum, MinerOptions}; -use ethcore::snapshot::SnapshotConfiguration; +use snapshot::SnapshotConfiguration; use miner::pool; use verification::queue::VerifierSettings; @@ -52,7 +52,7 @@ use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, KillBlockcha use export_hardcoded_sync::ExportHsyncCmd; use presale::ImportWallet; use account::{AccountCmd, NewAccount, ListAccounts, ImportAccounts, ImportFromGethAccounts}; -use snapshot::{self, SnapshotCommand}; +use snapshot_cmd::{self, SnapshotCommand}; use network::{IpFilter}; const DEFAULT_MAX_PEERS: u16 = 50; @@ -321,7 +321,7 @@ impl Configuration { fat_db: fat_db, compaction: compaction, file_path: self.args.arg_snapshot_file.clone(), - kind: snapshot::Kind::Take, + kind: snapshot_cmd::Kind::Take, block_at: to_block_id(&self.args.arg_snapshot_at)?, max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import, snapshot_conf: snapshot_conf, @@ -339,7 +339,7 @@ impl Configuration { fat_db: fat_db, compaction: compaction, file_path: self.args.arg_restore_file.clone(), - kind: snapshot::Kind::Restore, + kind: snapshot_cmd::Kind::Restore, block_at: to_block_id("latest")?, // unimportant. max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import, snapshot_conf: snapshot_conf, diff --git a/parity/informant.rs b/parity/informant.rs index e4511aff4..38f02d458 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -23,10 +23,11 @@ use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::time::{Instant, Duration}; use atty; -use ethcore::client::{ChainNotify, NewBlocks, Client}; -use client_traits::{BlockInfo, ChainInfo, BlockChainClient}; +use ethcore::client::Client; +use client_traits::{BlockInfo, ChainInfo, BlockChainClient, ChainNotify}; use types::{ BlockNumber, + chain_notify::NewBlocks, client_types::ClientReport, ids::BlockId, io_message::ClientIoMessage, @@ -34,8 +35,8 @@ use types::{ verification::VerificationQueueInfo as BlockQueueInfo, snapshot::RestorationStatus, }; -use ethcore::snapshot::SnapshotService as SS; -use ethcore::snapshot::service::Service as SnapshotService; +use snapshot::SnapshotService as SS; +use snapshot::service::Service as SnapshotService; use sync::{LightSyncProvider, LightSync, SyncProvider, ManageNetwork}; use io::{TimerToken, IoContext, IoHandler}; use light::Cache as LightDataCache; @@ -224,7 +225,7 @@ pub struct Informant { last_tick: RwLock, with_color: bool, target: T, - snapshot: Option>, + snapshot: Option>>, rpc_stats: Option>, last_import: Mutex, skipped: AtomicUsize, @@ -237,16 +238,16 @@ impl Informant { /// Make a new instance potentially `with_color` output. pub fn new( target: T, - snapshot: Option>, + snapshot: Option>>, rpc_stats: Option>, with_color: bool, ) -> Self { Informant { last_tick: RwLock::new(Instant::now()), - with_color: with_color, - target: target, - snapshot: snapshot, - rpc_stats: rpc_stats, + with_color, + target, + snapshot, + rpc_stats, last_import: Mutex::new(Instant::now()), skipped: AtomicUsize::new(0), skipped_txs: AtomicUsize::new(0), diff --git a/parity/lib.rs b/parity/lib.rs index 2ea24c0bf..3ecf1e84c 100644 --- a/parity/lib.rs +++ b/parity/lib.rs @@ -73,6 +73,7 @@ extern crate parity_runtime; extern crate parity_updater as updater; extern crate parity_version; extern crate registrar; +extern crate snapshot; extern crate spec; extern crate verification; @@ -112,7 +113,7 @@ mod rpc_apis; mod run; mod secretstore; mod signer; -mod snapshot; +mod snapshot_cmd; mod upgrade; mod user_defaults; mod db; @@ -213,7 +214,7 @@ fn execute( Cmd::SignerSign { id, pwfile, port, authfile } => cli_signer::signer_sign(id, pwfile, port, authfile).map(|s| ExecutionAction::Instant(Some(s))), Cmd::SignerList { port, authfile } => cli_signer::signer_list(port, authfile).map(|s| ExecutionAction::Instant(Some(s))), Cmd::SignerReject { id, port, authfile } => cli_signer::signer_reject(id, port, authfile).map(|s| ExecutionAction::Instant(Some(s))), - Cmd::Snapshot(snapshot_cmd) => snapshot::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))), + Cmd::Snapshot(snapshot_cmd) => snapshot_cmd::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))), Cmd::ExportHardcodedSync(export_hs_cmd) => export_hardcoded_sync::execute(export_hs_cmd).map(|s| ExecutionAction::Instant(Some(s))), } } diff --git a/parity/modules.rs b/parity/modules.rs index 783d6fcb1..28ba66316 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -16,15 +16,14 @@ use std::sync::{Arc, mpsc}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use sync::{self, SyncConfig, NetworkConfiguration, Params, ConnectionFilter}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use ethcore_private_tx::PrivateStateDB; use light::Provider; use parity_runtime::Executor; pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler}; -pub use ethcore::client::ChainNotify; use ethcore_logger::Config as LogConfig; pub type SyncModules = ( diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index 35e65c8ad..a3f1836f1 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -24,7 +24,7 @@ pub use parity_rpc::signer::SignerService; use account_utils::{self, AccountProvider}; use ethcore::client::Client; use ethcore::miner::Miner; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use client_traits::BlockChainClient; use sync::SyncState; use ethcore_logger::RotatingLogger; diff --git a/parity/run.rs b/parity/run.rs index 7b8874737..400541336 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -25,7 +25,7 @@ use call_contract::CallContract; use client_traits::{BlockInfo, BlockChainClient}; use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; use ethcore::miner::{self, stratum, Miner, MinerService, MinerOptions}; -use ethcore::snapshot::{self, SnapshotConfiguration}; +use snapshot::{self, SnapshotConfiguration}; use spec::SpecParams; use verification::queue::VerifierSettings; use ethcore_logger::{Config as LogConfig, RotatingLogger}; diff --git a/parity/snapshot.rs b/parity/snapshot_cmd.rs similarity index 96% rename from parity/snapshot.rs rename to parity/snapshot_cmd.rs index c9657b2d6..8c4a1cb10 100644 --- a/parity/snapshot.rs +++ b/parity/snapshot_cmd.rs @@ -22,10 +22,10 @@ use std::sync::Arc; use client_traits::SnapshotClient; use hash::keccak; -use ethcore::snapshot::{SnapshotConfiguration, SnapshotService as SS}; -use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; -use ethcore::snapshot::service::Service as SnapshotService; -use ethcore::client::{DatabaseCompactionProfile, VMType}; +use snapshot::{SnapshotConfiguration, SnapshotService as SS}; +use snapshot::io::{SnapshotReader, PackedReader, PackedWriter}; +use snapshot::service::Service as SnapshotService; +use ethcore::client::{Client, DatabaseCompactionProfile, VMType}; use ethcore::miner::Miner; use ethcore_service::ClientService; use types::{ @@ -73,7 +73,7 @@ pub struct SnapshotCommand { // helper for reading chunks from arbitrary reader and feeding them into the // service. -fn restore_using(snapshot: Arc, reader: &R, recover: bool) -> Result<(), String> { +fn restore_using(snapshot: Arc>, reader: &R, recover: bool) -> Result<(), String> { let manifest = reader.manifest(); info!("Restoring to block #{} (0x{:?})", manifest.block_number, manifest.block_hash); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 27b2b6fda..aa555d75f 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -64,6 +64,7 @@ parity-updater = { path = "../updater" } parity-version = { path = "../util/version" } rlp = "0.4.0" account-state = { path = "../ethcore/account-state" } +snapshot = { path = "../ethcore/snapshot" } stats = { path = "../util/stats" } trace = { path = "../ethcore/trace" } vm = { path = "../ethcore/vm" } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 5cb96fa1f..955947339 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -87,6 +87,7 @@ extern crate rlp; extern crate account_state; extern crate stats; +extern crate snapshot; extern crate tempdir; extern crate trace; extern crate vm; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 5c572b8ee..1e6b08847 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -29,7 +29,7 @@ use client_traits::{BlockChainClient, StateClient, ProvingBlockChainClient, Stat use ethash::{self, SeedHashCompute}; use ethcore::client::{Call, EngineInfo}; use ethcore::miner::{self, MinerService}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use hash::keccak; use miner::external::ExternalMinerService; use sync::SyncProvider; diff --git a/rpc/src/v1/impls/eth_pubsub.rs b/rpc/src/v1/impls/eth_pubsub.rs index 7841827e9..57f7d7e1a 100644 --- a/rpc/src/v1/impls/eth_pubsub.rs +++ b/rpc/src/v1/impls/eth_pubsub.rs @@ -31,8 +31,7 @@ use v1::traits::EthPubSub; use v1::types::{pubsub, RichHeader, Log}; use sync::{SyncState, Notification}; -use ethcore::client::{ChainNotify, NewBlocks, ChainRouteType}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::H256; use light::cache::Cache; use light::client::{LightChainClient, LightChainNotify}; @@ -43,6 +42,7 @@ use parking_lot::{RwLock, Mutex}; use sync::{LightSyncProvider, LightNetworkDispatcher, ManageNetwork}; use types::{ + chain_notify::{NewBlocks, ChainRouteType}, ids::BlockId, encoded, filter::Filter as EthFilter, diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 3503af586..2905324d6 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -23,7 +23,7 @@ use ethereum_types::{H64, H160, H256, H512, U64, U256}; use ethcore::client::Call; use client_traits::{BlockChainClient, StateClient}; use ethcore::miner::{self, MinerService, FilterOptions}; -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use account_state::state::StateInfo; use ethcore_logger::RotatingLogger; use ethkey::{crypto::ecies, Brain, Generator}; diff --git a/rpc/src/v1/tests/helpers/snapshot_service.rs b/rpc/src/v1/tests/helpers/snapshot_service.rs index aee9a6681..7f049ab20 100644 --- a/rpc/src/v1/tests/helpers/snapshot_service.rs +++ b/rpc/src/v1/tests/helpers/snapshot_service.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use ethcore::snapshot::SnapshotService; +use snapshot::SnapshotService; use bytes::Bytes; use ethereum_types::H256; diff --git a/rpc/src/v1/tests/mocked/eth_pubsub.rs b/rpc/src/v1/tests/mocked/eth_pubsub.rs index a2bfa118d..c3563475a 100644 --- a/rpc/src/v1/tests/mocked/eth_pubsub.rs +++ b/rpc/src/v1/tests/mocked/eth_pubsub.rs @@ -23,11 +23,12 @@ use jsonrpc_pubsub::Session; use std::time::Duration; use v1::{EthPubSub, EthPubSubClient, Metadata}; -use ethcore::client::{TestBlockChainClient, EachBlockWith, ChainNotify, NewBlocks, ChainRoute, ChainRouteType}; +use ethcore::client::{TestBlockChainClient, EachBlockWith}; use parity_runtime::Runtime; use ethereum_types::{Address, H256}; -use client_traits::BlockInfo; +use client_traits::{BlockInfo, ChainNotify}; use types::{ + chain_notify::{NewBlocks, ChainRoute, ChainRouteType}, log_entry::{LocalizedLogEntry, LogEntry}, ids::BlockId, }; diff --git a/secret-store/src/acl_storage.rs b/secret-store/src/acl_storage.rs index 110ce3d5c..f8f295ecc 100644 --- a/secret-store/src/acl_storage.rs +++ b/secret-store/src/acl_storage.rs @@ -16,10 +16,13 @@ use std::sync::Arc; use std::collections::{HashMap, HashSet}; -use common_types::ids::BlockId; +use common_types::{ + chain_notify::NewBlocks, + ids::BlockId +}; use parking_lot::{Mutex, RwLock}; use call_contract::CallContract; -use ethcore::client::{ChainNotify, NewBlocks}; +use client_traits::ChainNotify; use ethereum_types::Address; use ethabi::FunctionOutputDecoder; use trusted_client::TrustedClient; diff --git a/secret-store/src/key_server_set.rs b/secret-store/src/key_server_set.rs index 5ffaa7fb7..c69be8ce7 100644 --- a/secret-store/src/key_server_set.rs +++ b/secret-store/src/key_server_set.rs @@ -20,9 +20,12 @@ use std::collections::{BTreeMap, HashSet}; use parking_lot::Mutex; use call_contract::CallContract; use ethabi::FunctionOutputDecoder; -use ethcore::client::{Client, ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; -use common_types::ids::BlockId; +use ethcore::client::Client; +use client_traits::{BlockChainClient, ChainNotify}; +use common_types::{ + chain_notify::NewBlocks, + ids::BlockId, +}; use ethereum_types::{H256, Address}; use ethkey::public_to_address; use bytes::Bytes; diff --git a/secret-store/src/listener/service_contract_listener.rs b/secret-store/src/listener/service_contract_listener.rs index 1b6a6afdf..c0b306050 100644 --- a/secret-store/src/listener/service_contract_listener.rs +++ b/secret-store/src/listener/service_contract_listener.rs @@ -18,10 +18,10 @@ use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread; -use parking_lot::Mutex; -use ethcore::client::{ChainNotify, NewBlocks}; -use ethkey::{Public, public_to_address}; +use client_traits::ChainNotify; +use common_types::chain_notify::NewBlocks; use bytes::Bytes; +use ethkey::{Public, public_to_address}; use ethereum_types::{H256, U256, Address, BigEndianHash as _}; use key_server_set::KeyServerSet; use key_server_cluster::{NodeId, ClusterClient, ClusterSessionsListener, ClusterSession}; @@ -32,6 +32,7 @@ use key_server_cluster::decryption_session::SessionImpl as DecryptionSession; use key_server_cluster::key_version_negotiation_session::{SessionImpl as KeyVersionNegotiationSession, IsolatedSessionTransport as KeyVersionNegotiationTransport, FailedContinueAction}; use key_storage::KeyStorage; +use parking_lot::Mutex; use acl_storage::AclStorage; use listener::service_contract::ServiceContract; use listener::tasks_queue::TasksQueue; diff --git a/updater/src/updater.rs b/updater/src/updater.rs index c675fef94..561f2c335 100644 --- a/updater/src/updater.rs +++ b/updater/src/updater.rs @@ -29,9 +29,9 @@ use common_types::{ BlockNumber, ids::BlockId, filter::Filter, + chain_notify::NewBlocks, }; -use ethcore::client::{ChainNotify, NewBlocks}; -use client_traits::BlockChainClient; +use client_traits::{BlockChainClient, ChainNotify}; use ethereum_types::{H256, H160}; use hash_fetch::{self as fetch, HashFetch}; use parity_path::restrict_permissions_owner;