diff --git a/.gitignore b/.gitignore index 58b1895c6..3226ea5a2 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ # jetbrains ide stuff .idea *.iml + +# Build artifacts +out/ diff --git a/Cargo.lock b/Cargo.lock index 622ac2e54..9cfc4de62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,7 @@ dependencies = [ "ethsync 0.9.99", "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -43,6 +44,18 @@ dependencies = [ "syntex_syntax 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bigint" +version = "0.1.0" +dependencies = [ + "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "0.3.3" @@ -151,15 +164,15 @@ dependencies = [ [[package]] name = "eth-secp256k1" version = "0.5.4" -source = "git+https://github.com/arkpar/rust-secp256k1.git#45503e1de68d909b1862e3f2bdb9e1cdfdff3f1e" +source = "git+https://github.com/ethcore/rust-secp256k1#283a0677d8327536be58a87e0494d7e0e7b1d1d8" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -205,12 +218,12 @@ dependencies = [ "ethcore 0.9.99", "ethcore-util 0.9.99", "ethsync 0.9.99", - "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-http-server 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-http-server 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_codegen 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -220,11 +233,12 @@ name = "ethcore-util" version = "0.9.99" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "bigint 0.1.0", "clippy 0.0.44 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "eth-secp256k1 0.5.4 (git+https://github.com/arkpar/rust-secp256k1.git)", + "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)", "ethcore-devtools 0.9.99", "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -239,7 +253,7 @@ dependencies = [ "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.1.0", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -256,6 +270,7 @@ dependencies = [ "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", "ethcore-util 0.9.99", + "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -372,22 +387,22 @@ dependencies = [ [[package]] name = "jsonrpc-core" -version = "1.1.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_codegen 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "jsonrpc-http-server" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -542,6 +557,14 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "odds" version = "0.2.12" @@ -690,9 +713,14 @@ dependencies = [ "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde_codegen" -version = "0.6.14" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aster 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -704,11 +732,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2de097ad9..9b8ec6405 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } daemonize = "0.2" ethcore-devtools = { path = "devtools" } +number_prefix = "0.2" [features] default = ["rpc"] diff --git a/README.md b/README.md index f8b24f088..4fd2a53cc 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ First (if you don't already have it) get multirust: - Linux: ```bash -curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes +curl -sf https://raw.githubusercontent.com/brson/multirust/master/quick-install.sh | sudo sh -s -- --yes ``` - OSX with Homebrew: diff --git a/ethcore/res/ethereum/frontier.json b/ethcore/res/ethereum/frontier.json index 6e31a2fce..cd66ad3f1 100644 --- a/ethcore/res/ethereum/frontier.json +++ b/ethcore/res/ethereum/frontier.json @@ -1,9 +1,9 @@ { - "name": "Frontier", + "name": "Frontier/Homestead", "engineName": "Ethash", "params": { "accountStartNonce": "0x00", - "frontierCompatibilityModeLimit": "0x10c8e0", + "frontierCompatibilityModeLimit": "0x118c30", "maximumExtraDataSize": "0x20", "tieBreakingGas": false, "minGasLimit": "0x1388", diff --git a/ethcore/res/ethereum/frontier_like_test.json b/ethcore/res/ethereum/frontier_like_test.json index 553bb8018..3e4108566 100644 --- a/ethcore/res/ethereum/frontier_like_test.json +++ b/ethcore/res/ethereum/frontier_like_test.json @@ -3,7 +3,7 @@ "engineName": "Ethash", "params": { "accountStartNonce": "0x00", - "frontierCompatibilityModeLimit": "0x10c8e0", + "frontierCompatibilityModeLimit": "0x118c30", "maximumExtraDataSize": "0x20", "tieBreakingGas": false, "minGasLimit": "0x1388", diff --git a/ethcore/res/ethereum/morden.json b/ethcore/res/ethereum/morden.json index 2e45c2edd..e9f8e0e99 100644 --- a/ethcore/res/ethereum/morden.json +++ b/ethcore/res/ethereum/morden.json @@ -3,7 +3,7 @@ "engineName": "Ethash", "params": { "accountStartNonce": "0x0100000", - "frontierCompatibilityModeLimit": "0x10c8e0", + "frontierCompatibilityModeLimit": "0x789b0", "maximumExtraDataSize": "0x20", "tieBreakingGas": false, "minGasLimit": "0x1388", diff --git a/ethcore/src/account.rs b/ethcore/src/account.rs index c36c35232..6901996bc 100644 --- a/ethcore/src/account.rs +++ b/ethcore/src/account.rs @@ -92,10 +92,10 @@ impl Account { /// Create a new contract account. /// NOTE: make sure you use `init_code` on this before `commit`ing. - pub fn new_contract(balance: U256) -> Account { + pub fn new_contract(balance: U256, nonce: U256) -> Account { Account { balance: balance, - nonce: U256::from(0u8), + nonce: nonce, storage_root: SHA3_NULL_RLP, storage_overlay: RefCell::new(HashMap::new()), code_hash: None, @@ -261,7 +261,7 @@ mod tests { let mut db = MemoryDB::new(); let mut db = AccountDBMut::new(&mut db, &Address::new()); let rlp = { - let mut a = Account::new_contract(U256::from(69u8)); + let mut a = Account::new_contract(x!(69), x!(0)); a.set_storage(H256::from(&U256::from(0x00u64)), H256::from(&U256::from(0x1234u64))); a.commit_storage(&mut db); a.init_code(vec![]); @@ -281,7 +281,7 @@ mod tests { let mut db = AccountDBMut::new(&mut db, &Address::new()); let rlp = { - let mut a = Account::new_contract(U256::from(69u8)); + let mut a = Account::new_contract(x!(69), x!(0)); a.init_code(vec![0x55, 0x44, 0xffu8]); a.commit_code(&mut db); a.rlp() @@ -296,7 +296,7 @@ mod tests { #[test] fn commit_storage() { - let mut a = Account::new_contract(U256::from(69u8)); + let mut a = Account::new_contract(x!(69), x!(0)); let mut db = MemoryDB::new(); let mut db = AccountDBMut::new(&mut db, &Address::new()); a.set_storage(x!(0), x!(0x1234)); @@ -307,7 +307,7 @@ mod tests { #[test] fn commit_remove_commit_storage() { - let mut a = Account::new_contract(U256::from(69u8)); + let mut a = Account::new_contract(x!(69), x!(0)); let mut db = MemoryDB::new(); let mut db = AccountDBMut::new(&mut db, &Address::new()); a.set_storage(x!(0), x!(0x1234)); @@ -321,7 +321,7 @@ mod tests { #[test] fn commit_code() { - let mut a = Account::new_contract(U256::from(69u8)); + let mut a = Account::new_contract(x!(69), x!(0)); let mut db = MemoryDB::new(); let mut db = AccountDBMut::new(&mut db, &Address::new()); a.init_code(vec![0x55, 0x44, 0xffu8]); diff --git a/ethcore/src/action_params.rs b/ethcore/src/action_params.rs index 9e2d72c73..fa40d30a0 100644 --- a/ethcore/src/action_params.rs +++ b/ethcore/src/action_params.rs @@ -15,9 +15,7 @@ // along with Parity. If not, see . //! Evm input params. -use util::hash::*; -use util::uint::*; -use util::bytes::*; +use common::*; /// Transaction value #[derive(Clone, Debug)] diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 62763386f..668c004e5 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -28,6 +28,31 @@ use service::*; use client::BlockStatus; use util::panics::*; +known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); + +const MIN_MEM_LIMIT: usize = 16384; +const MIN_QUEUE_LIMIT: usize = 512; + +/// Block queue configuration +#[derive(Debug)] +pub struct BlockQueueConfig { + /// Maximum number of blocks to keep in unverified queue. + /// When the limit is reached, is_full returns true. + pub max_queue_size: usize, + /// Maximum heap memory to use. + /// When the limit is reached, is_full returns true. + pub max_mem_use: usize, +} + +impl Default for BlockQueueConfig { + fn default() -> Self { + BlockQueueConfig { + max_queue_size: 30000, + max_mem_use: 50 * 1024 * 1024, + } + } +} + /// Block queue status #[derive(Debug)] pub struct BlockQueueInfo { @@ -37,6 +62,12 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, /// Number of blocks being verified pub verifying_queue_size: usize, + /// Configured maximum number of blocks in the queue + pub max_queue_size: usize, + /// Configured maximum number of bytes to use + pub max_mem_use: usize, + /// Heap memory used in bytes + pub mem_used: usize, } impl BlockQueueInfo { @@ -48,7 +79,8 @@ impl BlockQueueInfo { /// Indicates that queue is full pub fn is_full(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > MAX_UNVERIFIED_QUEUE_SIZE + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || + self.mem_used > self.max_mem_use } /// Indicates that queue is empty @@ -68,7 +100,9 @@ pub struct BlockQueue { deleting: Arc, ready_signal: Arc, empty: Arc, - processing: RwLock> + processing: RwLock>, + max_queue_size: usize, + max_mem_use: usize, } struct UnVerifiedBlock { @@ -106,11 +140,9 @@ struct Verification { bad: HashSet, } -const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; - impl BlockQueue { /// Creates a new queue instance. - pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { + pub fn new(config: BlockQueueConfig, engine: Arc>, message_channel: IoChannel) -> BlockQueue { let verification = Arc::new(Mutex::new(Verification::default())); let more_to_verify = Arc::new(Condvar::new()); let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); @@ -133,7 +165,7 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) + BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) .expect("Error starting block verification thread") @@ -149,6 +181,8 @@ impl BlockQueue { deleting: deleting.clone(), processing: RwLock::new(HashSet::new()), empty: empty.clone(), + max_queue_size: max(config.max_queue_size, MIN_QUEUE_LIMIT), + max_mem_use: max(config.max_mem_use, MIN_MEM_LIMIT), } } @@ -340,8 +374,26 @@ impl BlockQueue { verified_queue_size: verification.verified.len(), unverified_queue_size: verification.unverified.len(), verifying_queue_size: verification.verifying.len(), + max_queue_size: self.max_queue_size, + max_mem_use: self.max_mem_use, + mem_used: + verification.unverified.heap_size_of_children() + + verification.verifying.heap_size_of_children() + + verification.verified.heap_size_of_children(), + // TODO: https://github.com/servo/heapsize/pull/50 + //+ self.processing.read().unwrap().heap_size_of_children(), } } + + pub fn collect_garbage(&self) { + { + let mut verification = self.verification.lock().unwrap(); + verification.unverified.shrink_to_fit(); + verification.verifying.shrink_to_fit(); + verification.verified.shrink_to_fit(); + } + self.processing.write().unwrap().shrink_to_fit(); + } } impl MayPanic for BlockQueue { @@ -373,7 +425,7 @@ mod tests { fn get_test_queue() -> BlockQueue { let spec = get_test_spec(); let engine = spec.to_engine().unwrap(); - BlockQueue::new(Arc::new(engine), IoChannel::disconnected()) + BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()) } #[test] @@ -381,7 +433,7 @@ mod tests { // TODO better test let spec = Spec::new_test(); let engine = spec.to_engine().unwrap(); - let _ = BlockQueue::new(Arc::new(engine), IoChannel::disconnected()); + let _ = BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()); } #[test] @@ -437,4 +489,19 @@ mod tests { assert!(queue.queue_info().is_empty()); } + + #[test] + fn test_mem_limit() { + let spec = get_test_spec(); + let engine = spec.to_engine().unwrap(); + let mut config = BlockQueueConfig::default(); + config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 + let mut queue = BlockQueue::new(config, Arc::new(engine), IoChannel::disconnected()); + assert!(!queue.queue_info().is_full()); + let mut blocks = get_good_dummy_block_seq(50); + for b in blocks.drain(..) { + queue.import_block(b).unwrap(); + } + assert!(queue.queue_info().is_full()); + } } diff --git a/ethcore/src/blockchain/best_block.rs b/ethcore/src/blockchain/best_block.rs new file mode 100644 index 000000000..00c092713 --- /dev/null +++ b/ethcore/src/blockchain/best_block.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::numbers::{U256,H256}; +use header::BlockNumber; + +/// Best block info. +#[derive(Default)] +pub struct BestBlock { + /// Best block hash. + pub hash: H256, + /// Best block number. + pub number: BlockNumber, + /// Best block total difficulty. + pub total_difficulty: U256 +} diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs new file mode 100644 index 000000000..ce639bfed --- /dev/null +++ b/ethcore/src/blockchain/block_info.rs @@ -0,0 +1,47 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::numbers::{U256,H256}; +use header::BlockNumber; + +/// Brief info about inserted block. +pub struct BlockInfo { + /// Block hash. + pub hash: H256, + /// Block number. + pub number: BlockNumber, + /// Total block difficulty. + pub total_difficulty: U256, + /// Block location in blockchain. + pub location: BlockLocation +} + +/// Describes location of newly inserted block. +pub enum BlockLocation { + /// It's part of the canon chain. + CanonChain, + /// It's not a part of the canon chain. + Branch, + /// It's part of the fork which should become canon chain, + /// because it's total difficulty is higher than current + /// canon chain difficulty. + BranchBecomingCanonChain { + /// Hash of the newest common ancestor with old canon chain. + ancestor: H256, + /// Hashes of the blocks between ancestor and this block. + route: Vec + } +} diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain/blockchain.rs similarity index 68% rename from ethcore/src/blockchain.rs rename to ethcore/src/blockchain/blockchain.rs index 7e878b81e..53dcfb62c 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -23,109 +23,30 @@ use transaction::*; use views::*; use receipt::Receipt; use chainfilter::{ChainFilter, BloomIndex, FilterDataSource}; +use blockchain::block_info::{BlockInfo, BlockLocation}; +use blockchain::best_block::BestBlock; +use blockchain::bloom_indexer::BloomIndexer; +use blockchain::tree_route::TreeRoute; +use blockchain::update::ExtrasUpdate; +use blockchain::CacheSize; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; -/// Represents a tree route between `from` block and `to` block: -pub struct TreeRoute { - /// A vector of hashes of all blocks, ordered from `from` to `to`. - pub blocks: Vec, - /// Best common ancestor of these blocks. - pub ancestor: H256, - /// An index where best common ancestor would be. - pub index: usize, -} - -/// Represents blockchain's in-memory cache size in bytes. +/// Blockchain configuration. #[derive(Debug)] -pub struct CacheSize { - /// Blocks cache size. - pub blocks: usize, - /// BlockDetails cache size. - pub block_details: usize, - /// Transaction addresses cache size. - pub transaction_addresses: usize, - /// Logs cache size. - pub block_logs: usize, - /// Blooms cache size. - pub blocks_blooms: usize, - /// Block receipts size. - pub block_receipts: usize, +pub struct BlockChainConfig { + /// Preferred cache size in bytes. + pub pref_cache_size: usize, + /// Maximum cache size in bytes. + pub max_cache_size: usize, } -struct BloomIndexer { - index_size: usize, - levels: u8, -} - -impl BloomIndexer { - fn new(index_size: usize, levels: u8) -> Self { - BloomIndexer { - index_size: index_size, - levels: levels - } - } - - /// Calculates bloom's position in database. - fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { - use std::{mem, ptr}; - - let hash = unsafe { - let mut hash: H256 = mem::zeroed(); - ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); - hash[8] = bloom_index.level; - hash.reverse(); - hash - }; - - BlocksBloomLocation { - hash: hash, - index: bloom_index.index % self.index_size - } - } - - fn index_size(&self) -> usize { - self.index_size - } - - fn levels(&self) -> u8 { - self.levels - } -} - -/// Blockchain update info. -struct ExtrasUpdate { - /// Block hash. - hash: H256, - /// DB update batch. - batch: DBTransaction, - /// Inserted block familial details. - details: BlockDetails, - /// New best block (if it has changed). - new_best: Option, - /// Changed blocks bloom location hashes. - bloom_hashes: HashSet, -} - -impl CacheSize { - /// Total amount used by the cache. - fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } -} - -/// Information about best block gathered together -struct BestBlock { - pub hash: H256, - pub number: BlockNumber, - pub total_difficulty: U256 -} - -impl BestBlock { - fn new() -> BestBlock { - BestBlock { - hash: H256::new(), - number: 0, - total_difficulty: U256::from(0) +impl Default for BlockChainConfig { + fn default() -> Self { + BlockChainConfig { + pref_cache_size: 1 << 14, + max_cache_size: 1 << 20, } } } @@ -308,33 +229,7 @@ const COLLECTION_QUEUE_SIZE: usize = 8; impl BlockChain { /// Create new instance of blockchain from given Genesis - /// - /// ```rust - /// extern crate ethcore_util as util; - /// extern crate ethcore; - /// use std::env; - /// use std::str::FromStr; - /// use ethcore::spec::*; - /// use ethcore::blockchain::*; - /// use ethcore::ethereum; - /// use util::hash::*; - /// use util::uint::*; - /// - /// fn main() { - /// let spec = ethereum::new_frontier(); - /// - /// let mut dir = env::temp_dir(); - /// dir.push(H32::random().hex()); - /// - /// let bc = BlockChain::new(&spec.genesis_block(), &dir); - /// - /// let genesis_hash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"; - /// assert_eq!(bc.genesis_hash(), H256::from_str(genesis_hash).unwrap()); - /// assert!(bc.is_known(&bc.genesis_hash())); - /// assert_eq!(bc.genesis_hash(), bc.block_hash(0).unwrap()); - /// } - /// ``` - pub fn new(genesis: &[u8], path: &Path) -> BlockChain { + pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { // open extras db let mut extras_path = path.to_path_buf(); extras_path.push("extras"); @@ -349,9 +244,9 @@ impl BlockChain { (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: 1 << 14, - max_cache_size: 1 << 20, - best_block: RwLock::new(BestBlock::new()), + pref_cache_size: config.pref_cache_size, + max_cache_size: config.max_cache_size, + best_block: RwLock::new(BestBlock::default()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), block_hashes: RwLock::new(HashMap::new()), @@ -452,40 +347,26 @@ impl BlockChain { /// ```json /// { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 } /// ``` - pub fn tree_route(&self, from: H256, to: H256) -> Option { - let from_details = match self.block_details(&from) { - Some(h) => h, - None => return None, - }; - let to_details = match self.block_details(&to) { - Some(h) => h, - None => return None, - }; - Some(self.tree_route_aux((&from_details, &from), (&to_details, &to))) - } - - /// Similar to `tree_route` function, but can be used to return a route - /// between blocks which may not be in database yet. - fn tree_route_aux(&self, from: (&BlockDetails, &H256), to: (&BlockDetails, &H256)) -> TreeRoute { + pub fn tree_route(&self, from: H256, to: H256) -> TreeRoute { let mut from_branch = vec![]; let mut to_branch = vec![]; - let mut from_details = from.0.clone(); - let mut to_details = to.0.clone(); - let mut current_from = from.1.clone(); - let mut current_to = to.1.clone(); + let mut from_details = self.block_details(&from).expect(&format!("0. Expected to find details for block {:?}", from)); + let mut to_details = self.block_details(&to).expect(&format!("1. Expected to find details for block {:?}", to)); + let mut current_from = from; + let mut current_to = to; // reset from && to to the same level while from_details.number > to_details.number { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("1. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("2. Expected to find details for block {:?}", from_details.parent)); } while to_details.number > from_details.number { to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("2. Expected to find details for block {:?}", to_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("3. Expected to find details for block {:?}", to_details.parent)); } assert_eq!(from_details.number, to_details.number); @@ -494,11 +375,11 @@ impl BlockChain { while current_from != current_to { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("3. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("5. Expected to find details for block {:?}", from_details.parent)); } let index = from_branch.len(); @@ -527,165 +408,237 @@ impl BlockChain { // store block in db self.blocks_db.put(&hash, &bytes).unwrap(); - let update = self.block_to_extras_update(bytes, receipts); - self.apply_update(update); + + let info = self.block_info(bytes); + + self.apply_update(ExtrasUpdate { + block_hashes: self.prepare_block_hashes_update(bytes, &info), + block_details: self.prepare_block_details_update(bytes, &info), + block_receipts: self.prepare_block_receipts_update(receipts, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), + blocks_blooms: self.prepare_block_blooms_update(bytes, &info), + info: info + }); } /// Applies extras update. fn apply_update(&self, update: ExtrasUpdate) { + let batch = DBTransaction::new(); + batch.put(b"best", &update.info.hash).unwrap(); + // update best block let mut best_block = self.best_block.write().unwrap(); - if let Some(b) = update.new_best { - *best_block = b; + match update.info.location { + BlockLocation::Branch => (), + _ => { + *best_block = BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty + }; + } } - // update details cache - let mut write_details = self.block_details.write().unwrap(); - write_details.remove(&update.details.parent); - write_details.insert(update.hash.clone(), update.details); - self.note_used(CacheID::Block(update.hash)); + let mut write_hashes = self.block_hashes.write().unwrap(); + for (number, hash) in &update.block_hashes { + batch.put_extras(number, hash); + write_hashes.remove(number); + } + + let mut write_details = self.block_details.write().unwrap(); + for (hash, details) in update.block_details.into_iter() { + batch.put_extras(&hash, &details); + write_details.insert(hash, details); + } + + let mut write_receipts = self.block_receipts.write().unwrap(); + for (hash, receipt) in &update.block_receipts { + batch.put_extras(hash, receipt); + write_receipts.remove(hash); + } + + let mut write_txs = self.transaction_addresses.write().unwrap(); + for (hash, tx_address) in &update.transactions_addresses { + batch.put_extras(hash, tx_address); + write_txs.remove(hash); + } - // update blocks blooms cache let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for bloom_hash in &update.bloom_hashes { + for (bloom_hash, blocks_bloom) in &update.blocks_blooms { + batch.put_extras(bloom_hash, blocks_bloom); write_blocks_blooms.remove(bloom_hash); } // update extras database - self.extras_db.write(update.batch).unwrap(); + self.extras_db.write(batch).unwrap(); } - /// Transforms block into WriteBatch that may be written into database - /// Additionally, if it's new best block it returns new best block object. - fn block_to_extras_update(&self, bytes: &[u8], receipts: Vec) -> ExtrasUpdate { - // create views onto rlp - let block = BlockView::new(bytes); + /// Get inserted block info which is critical to preapre extras updates. + fn block_info(&self, block_bytes: &[u8]) -> BlockInfo { + let block = BlockView::new(block_bytes); let header = block.header_view(); - - // prepare variables let hash = block.sha3(); - let mut parent_details = self.block_details(&header.parent_hash()).expect(format!("Invalid parent hash: {:?}", header.parent_hash()).as_ref()); + let number = header.number(); + let parent_hash = header.parent_hash(); + let parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); let total_difficulty = parent_details.total_difficulty + header.difficulty(); let is_new_best = total_difficulty > self.best_block_total_difficulty(); + + BlockInfo { + hash: hash, + number: number, + total_difficulty: total_difficulty, + location: if is_new_best { + // on new best block we need to make sure that all ancestors + // are moved to "canon chain" + // find the route between old best block and the new one + let best_hash = self.best_block_hash(); + let route = self.tree_route(best_hash, parent_hash); + + assert_eq!(number, parent_details.number + 1); + + match route.blocks.len() { + 0 => BlockLocation::CanonChain, + _ => BlockLocation::BranchBecomingCanonChain { + ancestor: route.ancestor, + route: route.blocks.into_iter().skip(route.index).collect() + } + } + } else { + BlockLocation::Branch + } + } + } + + /// This function returns modified block hashes. + fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let mut block_hashes = HashMap::new(); + let block = BlockView::new(block_bytes); + let header = block.header_view(); + let number = header.number(); + + match info.location { + BlockLocation::Branch => (), + BlockLocation::CanonChain => { + block_hashes.insert(number, info.hash.clone()); + }, + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); + let start_number = ancestor_number + 1; + + for (index, hash) in route.iter().cloned().enumerate() { + block_hashes.insert(start_number + index as BlockNumber, hash); + } + + block_hashes.insert(number, info.hash.clone()); + } + } + + block_hashes + } + + /// This function returns modified block details. + fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let header = block.header_view(); let parent_hash = header.parent_hash(); + // update parent + let mut parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); + parent_details.children.push(info.hash.clone()); + // create current block details let details = BlockDetails { number: header.number(), - total_difficulty: total_difficulty, + total_difficulty: info.total_difficulty, parent: parent_hash.clone(), children: vec![] }; - // prepare the batch - let batch = DBTransaction::new(); + // write to batch + let mut block_details = HashMap::new(); + block_details.insert(parent_hash, parent_details); + block_details.insert(info.hash.clone(), details); + block_details + } - // insert new block details - batch.put_extras(&hash, &details); + /// This function returns modified block receipts. + fn prepare_block_receipts_update(&self, receipts: Vec, info: &BlockInfo) -> HashMap { + let mut block_receipts = HashMap::new(); + block_receipts.insert(info.hash.clone(), BlockReceipts::new(receipts)); + block_receipts + } - // update parent details - parent_details.children.push(hash.clone()); - batch.put_extras(&parent_hash, &parent_details); + /// This function returns modified transaction addresses. + fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let transaction_hashes = block.transaction_hashes(); - // update transaction addresses - for (i, tx_hash) in block.transaction_hashes().iter().enumerate() { - batch.put_extras(tx_hash, &TransactionAddress { - block_hash: hash.clone(), - index: i - }); - } + transaction_hashes.into_iter() + .enumerate() + .fold(HashMap::new(), |mut acc, (i ,tx_hash)| { + acc.insert(tx_hash, TransactionAddress { + block_hash: info.hash.clone(), + index: i + }); + acc + }) + } - // update block receipts - batch.put_extras(&hash, &BlockReceipts::new(receipts)); + /// This functions returns modified blocks blooms. + /// + /// To accelerate blooms lookups, blomms are stored in multiple + /// layers (BLOOM_LEVELS, currently 3). + /// ChainFilter is responsible for building and rebuilding these layers. + /// It returns them in HashMap, where values are Blooms and + /// keys are BloomIndexes. BloomIndex represents bloom location on one + /// of these layers. + /// + /// To reduce number of queries to databse, block blooms are stored + /// in BlocksBlooms structure which contains info about several + /// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms. + /// + /// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex) + /// to bloom location in database (BlocksBloomLocation). + /// + fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let block = BlockView::new(block_bytes); + let header = block.header_view(); - // if it's not new best block, just return - if !is_new_best { - return ExtrasUpdate { - hash: hash.clone(), - batch: batch, - details: details, - new_best: None, - bloom_hashes: HashSet::new() - }; - } - - // if its new best block we need to make sure that all ancestors - // are moved to "canon chain" - // find the route between old best block and the new one - let best_hash = self.best_block_hash(); - let best_details = self.block_details(&best_hash).expect("best block hash is invalid!"); - let route = self.tree_route_aux((&best_details, &best_hash), (&details, &hash)); - - let modified_blooms; - - match route.blocks.len() { - // its our parent - 1 => { - batch.put_extras(&header.number(), &hash); - - // update block blooms - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .add_bloom(&header.log_bloom(), header.number() as usize); + let modified_blooms = match info.location { + BlockLocation::Branch => HashMap::new(), + BlockLocation::CanonChain => { + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .add_bloom(&header.log_bloom(), header.number() as usize) }, - // it is a fork - i if i > 1 => { - let ancestor_number = self.block_number(&route.ancestor).unwrap(); + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - for (index, hash) in route.blocks.iter().skip(route.index).enumerate() { - batch.put_extras(&(start_number + index as BlockNumber), hash); - } - // get all blocks that are not part of canon chain (TODO: optimize it to one query) - let blooms: Vec = route.blocks.iter() - .skip(route.index) + let mut blooms: Vec = route.iter() .map(|hash| self.block(hash).unwrap()) .map(|bytes| BlockView::new(&bytes).header_view().log_bloom()) .collect(); - // reset blooms chain head - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize); - }, - // route.blocks.len() could be 0 only if inserted block is best block, - // and this is not possible at this stage - _ => { unreachable!(); } + blooms.push(header.log_bloom()); + + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize) + } }; - let bloom_hashes = modified_blooms.iter() - .map(|(bloom_index, _)| self.bloom_indexer.location(&bloom_index).hash) - .collect(); - - for (bloom_hash, blocks_blooms) in modified_blooms.into_iter() + modified_blooms.into_iter() .fold(HashMap::new(), | mut acc, (bloom_index, bloom) | { { let location = self.bloom_indexer.location(&bloom_index); let mut blocks_blooms = acc .entry(location.hash.clone()) .or_insert_with(|| self.blocks_blooms(&location.hash).unwrap_or_else(BlocksBlooms::new)); - assert_eq!(self.bloom_indexer.index_size, blocks_blooms.blooms.len()); + assert_eq!(self.bloom_indexer.index_size(), blocks_blooms.blooms.len()); blocks_blooms.blooms[location.index] = bloom; } acc - }) { - batch.put_extras(&bloom_hash, &blocks_blooms); - } - - // this is new best block - batch.put(b"best", &hash).unwrap(); - - let best_block = BestBlock { - hash: hash.clone(), - number: header.number(), - total_difficulty: total_difficulty - }; - - ExtrasUpdate { - hash: hash, - batch: batch, - new_best: Some(best_block), - details: details, - bloom_hashes: bloom_hashes - } + }) } /// Get best block hash. @@ -813,18 +766,23 @@ mod tests { use std::str::FromStr; use rustc_serialize::hex::FromHex; use util::hash::*; - use blockchain::{BlockProvider, BlockChain}; + use util::sha3::Hashable; + use blockchain::{BlockProvider, BlockChain, BlockChainConfig}; use tests::helpers::*; use devtools::*; + use blockchain::helpers::generators::{ChainGenerator, ChainIterator}; + use views::BlockView; #[test] - fn valid_tests_extra32() { - let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0925002c3260b44e44c3edebad1cc442142b03020209df1ab8bb86752edbd2cd7a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a0363659b251bf8b819179874c8cce7b9b983d7f3704cbb58a3b334431f7032871889032d09c281e1236c0c0".from_hex().unwrap(); + fn basic_blockchain_insert() { + let mut canon_chain = ChainGenerator::default(); + let genesis = canon_chain.next().unwrap().rlp(); + let first = canon_chain.next().unwrap().rlp(); + let genesis_hash = BlockView::new(&genesis).header_view().sha3(); + let first_hash = BlockView::new(&first).header_view().sha3(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); - - let genesis_hash = H256::from_str("3caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942").unwrap(); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.genesis_hash(), genesis_hash.clone()); assert_eq!(bc.best_block_number(), 0); @@ -832,13 +790,9 @@ mod tests { assert_eq!(bc.block_hash(0), Some(genesis_hash.clone())); assert_eq!(bc.block_hash(1), None); assert_eq!(bc.block_details(&genesis_hash).unwrap().children, vec![]); - - let first = "f90285f90219a03caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0bac6177a79e910c98d86ec31a09ae37ac2de15b754fd7bed1ba52362c49416bfa0d45893a296c1490a978e0bd321b5f2635d8280365c1fe9f693d65f233e791344a0c7778a7376099ee2e5c455791c1885b5c361b95713fddcbe32d97fd01334d296b90100000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000400000000000000000000000000000000000000000000000000000008302000001832fefd882560b845627cb99a00102030405060708091011121314151617181920212223242526272829303132a08ccb2837fb2923bd97e8f2d08ea32012d6e34be018c73e49a0f98843e8f47d5d88e53be49fec01012ef866f864800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d8785012a05f200801ba0cb088b8d2ff76a7b2c6616c9d02fb6b7a501afbf8b69d7180b09928a1b80b5e4a06448fe7476c606582039bb72a9f6f4b4fad18507b8dfbd00eebbe151cc573cd2c0".from_hex().unwrap(); bc.insert_block(&first, vec![]); - let first_hash = H256::from_str("a940e5af7d146b3b917c953a82e1966b906dace3a4e355b5b0a4560190357ea1").unwrap(); - assert_eq!(bc.block_hash(0), Some(genesis_hash.clone())); assert_eq!(bc.best_block_number(), 1); assert_eq!(bc.best_block_hash(), first_hash.clone()); @@ -851,23 +805,27 @@ mod tests { #[test] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { - let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); - let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap(); - let b2 = "f902ccf901f9a0437e51676ff10756fcfee5edd9159fa41dbcb1b2c592850450371cbecd54ee4fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004002832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - let b3a = "f90261f901f9a036fde1253128666fcb95a5956da14a73489e988bb72738717ec1d31e1cee781aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a05fb2b4bfdef7b314451cb138a534d225c922fc0e5fbe25e451142732c3e25c25a09dc4b1357c0b7b8108f8a098f4f9a1a274957bc9ebc22a9ae67ae81739e5b19ca007c6fdfa8eea7e86b81f5b0fc0f78f90cc19f4aa60d323151e0cac660199e9a1b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefd882524d84562791eb80a074861666bd346c025889745c793b91ab9cd1e2ca19b5cf3c50d04d135b0a4d2b8809fe9587ea4cdc04f862f86002018304cb2f94ec0e71ad0a90ffe1909d27dac207f7680abba42d01801ba06fd84874d36d5de9e8e48978c03619b53a96b7ae0a4cd1ac118f103098b44801a00572596974dd7df4f9f69bd7456585618c568d8434ef6453391b89281ce12ae1c0".from_hex().unwrap(); - let b3b = "f90265f901f9a036fde1253128666fcb95a5956da14a73489e988bb72738717ec1d31e1cee781aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ab87dc338bfd6f662b1cd90bc0c9e40a1b2146a095312393c9e13ce3a5008b09a0e609b7a7d4b8a2403ec1268627ecd98783627246e8f1b26addb3ff504f76a054a0592fabf92476512952db3a69a2481a42912e668a1ee28c4c322e703bb665f8beb90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefd882a1f084562791ee80a0fe7098fa7e4ac5d637eea81fb23f8f78346826dbab430068dd9a249d0afa99818853e1a6b201ae3545f866f86402018304cb2f94ec0e71ad0a90ffe1909d27dac207f7680abba42d0284c04062261ca06edc9ce8e7da4cc34067beb325dcad59e5655a164a5100a50bc3eb681b12c716a0abf9053d5de65b1be81fe50d327b84de685efbeecea34e7b747180a6c6023e44c0".from_hex().unwrap(); + let mut canon_chain = ChainGenerator::default(); + let genesis = canon_chain.next().unwrap().rlp(); + let blocks = canon_chain.clone().take(3).map(|block| block.rlp()).collect::>(); + let fork = canon_chain.skip(2).fork(1).take(1).next().unwrap().rlp(); - let genesis_hash = H256::from_str("5716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2").unwrap(); - let b1_hash = H256::from_str("437e51676ff10756fcfee5edd9159fa41dbcb1b2c592850450371cbecd54ee4f").unwrap(); - let b2_hash = H256::from_str("36fde1253128666fcb95a5956da14a73489e988bb72738717ec1d31e1cee781a").unwrap(); - let b3a_hash = H256::from_str("c208f88c9f5bf7e00840439742c12e5226d9752981f3ec0521bdcb6dd08af277").unwrap(); - let b3b_hash = H256::from_str("bf72270ae0d95c9ea39a6adab994793fddb8c10fba7391e26279474124605d54").unwrap(); + let b1 = blocks[0].clone(); + let b2 = blocks[1].clone(); + let b3a = blocks[2].clone(); + let b3b = fork; + + let genesis_hash = BlockView::new(&genesis).header_view().sha3(); + let b1_hash= BlockView::new(&b1).header_view().sha3(); + let b2_hash= BlockView::new(&b2).header_view().sha3(); + let b3a_hash= BlockView::new(&b3a).header_view().sha3(); + let b3b_hash= BlockView::new(&b3b).header_view().sha3(); // b3a is a part of canon chain, whereas b3b is part of sidechain - let best_block_hash = H256::from_str("c208f88c9f5bf7e00840439742c12e5226d9752981f3ec0521bdcb6dd08af277").unwrap(); + let best_block_hash = b3a_hash.clone(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1, vec![]); bc.insert_block(&b2, vec![]); bc.insert_block(&b3a, vec![]); @@ -886,52 +844,52 @@ mod tests { assert_eq!(bc.block_hash(3).unwrap(), b3a_hash); // test trie route - let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()).unwrap(); + let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()); assert_eq!(r0_1.ancestor, genesis_hash); assert_eq!(r0_1.blocks, [b1_hash.clone()]); assert_eq!(r0_1.index, 0); - let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()).unwrap(); + let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); assert_eq!(r0_2.ancestor, genesis_hash); assert_eq!(r0_2.blocks, [b1_hash.clone(), b2_hash.clone()]); assert_eq!(r0_2.index, 0); - let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()).unwrap(); + let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); assert_eq!(r1_3a.ancestor, b1_hash); assert_eq!(r1_3a.blocks, [b2_hash.clone(), b3a_hash.clone()]); assert_eq!(r1_3a.index, 0); - let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()).unwrap(); + let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); assert_eq!(r1_3b.ancestor, b1_hash); assert_eq!(r1_3b.blocks, [b2_hash.clone(), b3b_hash.clone()]); assert_eq!(r1_3b.index, 0); - let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()).unwrap(); + let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); assert_eq!(r3a_3b.ancestor, b2_hash); assert_eq!(r3a_3b.blocks, [b3a_hash.clone(), b3b_hash.clone()]); assert_eq!(r3a_3b.index, 1); - let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()).unwrap(); + let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()); assert_eq!(r1_0.ancestor, genesis_hash); assert_eq!(r1_0.blocks, [b1_hash.clone()]); assert_eq!(r1_0.index, 1); - let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()).unwrap(); + let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()); assert_eq!(r2_0.ancestor, genesis_hash); assert_eq!(r2_0.blocks, [b2_hash.clone(), b1_hash.clone()]); assert_eq!(r2_0.index, 2); - let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()).unwrap(); + let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()); assert_eq!(r3a_1.ancestor, b1_hash); assert_eq!(r3a_1.blocks, [b3a_hash.clone(), b2_hash.clone()]); assert_eq!(r3a_1.index, 2); - let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()).unwrap(); + let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()); assert_eq!(r3b_1.ancestor, b1_hash); assert_eq!(r3b_1.blocks, [b3b_hash.clone(), b2_hash.clone()]); assert_eq!(r3b_1.index, 2); - let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()).unwrap(); + let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()); assert_eq!(r3b_3a.ancestor, b2_hash); assert_eq!(r3b_3a.blocks, [b3b_hash.clone(), b3a_hash.clone()]); assert_eq!(r3b_3a.index, 1); @@ -946,14 +904,14 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), genesis_hash); bc.insert_block(&b1, vec![]); assert_eq!(bc.best_block_hash(), b1_hash); } { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), b1_hash); } } @@ -1006,9 +964,9 @@ mod tests { let b1_hash = H256::from_str("f53f268d23a71e85c7d6d83a9504298712b84c1a2ba220441c86eeda0bf0b6e3").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1, vec![]); - + let transactions = bc.transactions(&b1_hash).unwrap(); assert_eq!(transactions.len(), 7); for t in transactions { @@ -1028,7 +986,7 @@ mod tests { // prepare for fork (b1a, child of genesis) let b1a = "f902ccf901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004001832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - + // fork (b2a, child of b1a, with higher total difficulty) let b2a = "f902ccf901f9a0626b0774a7cbdad7bdce07b87d74b6fa91c1c359d725076215d76348f8399f56a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004002832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); @@ -1042,13 +1000,13 @@ mod tests { let bloom_ba = H2048::from_str("00000000000000000000000000000000000000000000020000000800000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); assert_eq!(blocks_b1, vec![]); assert_eq!(blocks_b2, vec![]); - + bc.insert_block(&b1, vec![]); let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); @@ -1088,31 +1046,4 @@ mod tests { assert_eq!(blocks_b2, vec![2]); assert_eq!(blocks_ba, vec![3]); } - - #[test] - fn test_bloom_indexer() { - use chainfilter::BloomIndex; - use blockchain::BloomIndexer; - use extras::BlocksBloomLocation; - - let bi = BloomIndexer::new(16, 3); - - let index = BloomIndex::new(0, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::new(), - index: 0 - }); - - let index = BloomIndex::new(1, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), - index: 0 - }); - - let index = BloomIndex::new(0, 299_999); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), - index: 15 - }); - } } diff --git a/ethcore/src/blockchain/bloom_indexer.rs b/ethcore/src/blockchain/bloom_indexer.rs new file mode 100644 index 000000000..a672a5445 --- /dev/null +++ b/ethcore/src/blockchain/bloom_indexer.rs @@ -0,0 +1,102 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::numbers::H256; +use chainfilter::BloomIndex; + +/// Represents location of block bloom in extras database. +#[derive(Debug, PartialEq)] +pub struct BlocksBloomLocation { + /// Unique hash of BlocksBloom + pub hash: H256, + /// Index within BlocksBloom + pub index: usize, +} + +/// Should be used to localize blocks blooms in extras database. +pub struct BloomIndexer { + index_size: usize, + levels: u8, +} + +impl BloomIndexer { + /// Plain constructor. + pub fn new(index_size: usize, levels: u8) -> Self { + BloomIndexer { + index_size: index_size, + levels: levels + } + } + + /// Calculates bloom's position in database. + pub fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { + use std::{mem, ptr}; + + let hash = unsafe { + let mut hash: H256 = mem::zeroed(); + ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); + hash[8] = bloom_index.level; + hash.reverse(); + hash + }; + + BlocksBloomLocation { + hash: hash, + index: bloom_index.index % self.index_size + } + } + + /// Returns index size. + pub fn index_size(&self) -> usize { + self.index_size + } + + /// Returns number of cache levels. + pub fn levels(&self) -> u8 { + self.levels + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use util::hash::{H256, FixedHash}; + use chainfilter::BloomIndex; + use blockchain::bloom_indexer::{BloomIndexer, BlocksBloomLocation}; + + #[test] + fn test_bloom_indexer() { + let bi = BloomIndexer::new(16, 3); + + let index = BloomIndex::new(0, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::new(), + index: 0 + }); + + let index = BloomIndex::new(1, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), + index: 0 + }); + + let index = BloomIndex::new(0, 299_999); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), + index: 15 + }); + } +} diff --git a/ethcore/src/blockchain/cache.rs b/ethcore/src/blockchain/cache.rs new file mode 100644 index 000000000..722f83c16 --- /dev/null +++ b/ethcore/src/blockchain/cache.rs @@ -0,0 +1,37 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +/// Represents blockchain's in-memory cache size in bytes. +#[derive(Debug)] +pub struct CacheSize { + /// Blocks cache size. + pub blocks: usize, + /// BlockDetails cache size. + pub block_details: usize, + /// Transaction addresses cache size. + pub transaction_addresses: usize, + /// Logs cache size. + pub block_logs: usize, + /// Blooms cache size. + pub blocks_blooms: usize, + /// Block receipts size. + pub block_receipts: usize, +} + +impl CacheSize { + /// Total amount used by the cache. + pub fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } +} diff --git a/ethcore/src/blockchain/helpers/generators.rs b/ethcore/src/blockchain/helpers/generators.rs new file mode 100644 index 000000000..a20b0cf79 --- /dev/null +++ b/ethcore/src/blockchain/helpers/generators.rs @@ -0,0 +1,203 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::rlp::*; +use util::hash::{H256, H2048}; +use util::uint::{U256}; +use util::bytes::Bytes; +use header::{BlockNumber, Header}; +use transaction::SignedTransaction; + +pub trait Forkable { + fn fork(self, fork_number: usize) -> Self where Self: Sized; +} + +pub struct Fork { + iter: I, + fork_number: usize, +} + +impl Iterator for Fork where I: Iterator, ::Item: Forkable { + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|item| item.fork(self.fork_number)) + } +} + +pub trait WithBloom { + fn with_bloom(self, bloom: H2048) -> Self where Self: Sized; +} + +pub struct Bloom { + iter: I, + bloom: H2048, +} + +impl Iterator for Bloom where I: Iterator, ::Item: WithBloom { + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|item| item.with_bloom(self.bloom.clone())) + } +} + +/// Chain iterator interface. +pub trait ChainIterator: Iterator { + /// Should be called to create a fork of current iterator. + /// Blocks generated by fork will have lower difficulty than current chain. + fn fork(&mut self, fork_number: usize) -> Fork where Self: Sized; + /// Should be called to make every consecutive block have given bloom. + fn with_bloom(&mut self, bloom: H2048) -> Bloom where Self: Sized; +} + +impl ChainIterator for I where I: Iterator + Sized + Clone { + fn fork(&mut self, fork_number: usize) -> Fork { + Fork { + iter: self.clone(), + fork_number: fork_number + } + } + + fn with_bloom(&mut self, bloom: H2048) -> Bloom { + Bloom { + iter: self.clone(), + bloom: bloom + } + } +} + +/// Helper structure, used for encoding blocks. +#[derive(Default)] +pub struct Block { + header: Header, + transactions: Vec, + uncles: Vec
+} + +impl Block { + pub fn rlp(&self) -> Bytes { + encode(self).to_vec() + } +} + +impl Encodable for Block { + fn rlp_append(&self, s: &mut RlpStream) { + s.begin_list(3); + s.append(&self.header); + s.append(&self.transactions); + s.append(&self.uncles); + } +} + +impl Forkable for Block { + fn fork(mut self, fork_number: usize) -> Self where Self: Sized { + self.header.difficulty = self.header.difficulty - U256::from(fork_number); + self + } +} + +impl WithBloom for Block { + fn with_bloom(mut self, bloom: H2048) -> Self where Self: Sized { + self.header.log_bloom = bloom; + self + } +} + +/// Blockchain generator. +#[derive(Clone)] +pub struct ChainGenerator { + /// Next block number. + number: BlockNumber, + /// Next block parent hash. + parent_hash: H256, + /// Next block difficulty. + difficulty: U256, +} + +impl ChainGenerator { + fn prepare_block(&self) -> Block { + let mut block = Block::default(); + block.header.parent_hash = self.parent_hash.clone(); + block.header.number = self.number; + block.header.difficulty = self.difficulty; + block + } +} + +impl Default for ChainGenerator { + fn default() -> Self { + ChainGenerator { + number: 0, + parent_hash: H256::default(), + difficulty: U256::from(1000), + } + } +} + +impl Iterator for ChainGenerator { + type Item = Block; + + fn next(&mut self) -> Option { + let block = self.prepare_block(); + self.number += 1; + self.parent_hash = block.header.hash(); + Some(block) + } +} + + +#[cfg(test)] +mod tests { + use util::hash::H256; + use util::sha3::Hashable; + use views::BlockView; + use super::{ChainIterator, ChainGenerator}; + + #[test] + fn canon_chain_generator() { + let mut canon_chain = ChainGenerator::default(); + + let genesis_rlp = canon_chain.next().unwrap().rlp(); + let genesis = BlockView::new(&genesis_rlp); + + assert_eq!(genesis.header_view().parent_hash(), H256::default()); + assert_eq!(genesis.header_view().number(), 0); + + let b1_rlp = canon_chain.next().unwrap().rlp(); + let b1 = BlockView::new(&b1_rlp); + + assert_eq!(b1.header_view().parent_hash(), genesis.header_view().sha3()); + assert_eq!(b1.header_view().number(), 1); + + let mut fork_chain = canon_chain.fork(1); + + let b2_rlp_fork = fork_chain.next().unwrap().rlp(); + let b2_fork = BlockView::new(&b2_rlp_fork); + + assert_eq!(b2_fork.header_view().parent_hash(), b1.header_view().sha3()); + assert_eq!(b2_fork.header_view().number(), 2); + + let b2_rlp = canon_chain.next().unwrap().rlp(); + let b2 = BlockView::new(&b2_rlp); + + assert_eq!(b2.header_view().parent_hash(), b1.header_view().sha3()); + assert_eq!(b2.header_view().number(), 2); + assert!(b2.header_view().difficulty() > b2_fork.header_view().difficulty()); + } +} diff --git a/ethcore/src/blockchain/helpers/mod.rs b/ethcore/src/blockchain/helpers/mod.rs new file mode 100644 index 000000000..233f8f1f8 --- /dev/null +++ b/ethcore/src/blockchain/helpers/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +pub mod generators; diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs new file mode 100644 index 000000000..60a1aeb33 --- /dev/null +++ b/ethcore/src/blockchain/mod.rs @@ -0,0 +1,31 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Blockchain database. + +pub mod blockchain; +mod best_block; +mod block_info; +mod bloom_indexer; +mod cache; +mod tree_route; +mod update; +#[cfg(test)] +mod helpers; + +pub use self::blockchain::{BlockProvider, BlockChain, BlockChainConfig}; +pub use self::cache::CacheSize; +pub use self::tree_route::TreeRoute; diff --git a/ethcore/src/blockchain/tree_route.rs b/ethcore/src/blockchain/tree_route.rs new file mode 100644 index 000000000..3c4906449 --- /dev/null +++ b/ethcore/src/blockchain/tree_route.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::numbers::H256; + +/// Represents a tree route between `from` block and `to` block: +#[derive(Debug)] +pub struct TreeRoute { + /// A vector of hashes of all blocks, ordered from `from` to `to`. + pub blocks: Vec, + /// Best common ancestor of these blocks. + pub ancestor: H256, + /// An index where best common ancestor would be. + pub index: usize, +} + diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs new file mode 100644 index 000000000..6be2647d3 --- /dev/null +++ b/ethcore/src/blockchain/update.rs @@ -0,0 +1,21 @@ +use std::collections::HashMap; +use util::numbers::H256; +use header::BlockNumber; +use blockchain::block_info::BlockInfo; +use extras::{BlockDetails, BlockReceipts, TransactionAddress, BlocksBlooms}; + +/// Block extras update info. +pub struct ExtrasUpdate { + /// Block info. + pub info: BlockInfo, + /// Modified block hashes. + pub block_hashes: HashMap, + /// Modified block details. + pub block_details: HashMap, + /// Modified block receipts. + pub block_receipts: HashMap, + /// Modified transaction addresses. + pub transactions_addresses: HashMap, + /// Modified blocks blooms. + pub blocks_blooms: HashMap, +} diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index ef6917b3a..5aabafec1 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -18,7 +18,7 @@ use util::*; use util::panics::*; -use blockchain::{BlockChain, BlockProvider, CacheSize}; +use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; use header::{BlockNumber, Header}; @@ -26,7 +26,7 @@ use state::State; use spec::Spec; use engine::Engine; use views::HeaderView; -use block_queue::{BlockQueue, BlockQueueInfo}; +use block_queue::BlockQueue; use service::{NetSyncMessage, SyncMessage}; use env_info::LastHashes; use verification::*; @@ -35,7 +35,8 @@ use transaction::LocalizedTransaction; use extras::TransactionAddress; use filter::Filter; use log_entry::LocalizedLogEntry; -pub use blockchain::TreeRoute; +pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; +pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; /// Uniquely identifies block. #[derive(Debug, PartialEq, Clone)] @@ -74,7 +75,16 @@ pub enum BlockStatus { Unknown, } -/// Information about the blockchain gthered together. +/// Client configuration. Includes configs for all sub-systems. +#[derive(Debug, Default)] +pub struct ClientConfig { + /// Block queue configuration. + pub queue: BlockQueueConfig, + /// Blockchain configuration. + pub blockchain: BlockChainConfig, +} + +/// Information about the blockchain gathered together. #[derive(Debug)] pub struct BlockChainInfo { /// Blockchain difficulty. @@ -193,14 +203,14 @@ const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { /// Create a new client with given spec and DB path. - pub fn new(spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); let path = dir.as_path(); let gb = spec.genesis_block(); - let chain = Arc::new(RwLock::new(BlockChain::new(&gb, path))); + let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); let mut state_path = path.to_path_buf(); state_path.push("state"); @@ -210,7 +220,7 @@ impl Client { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } - let block_queue = BlockQueue::new(engine.clone(), message_channel); + let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel); let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); @@ -359,7 +369,7 @@ impl Client { } /// Get info on the cache. - pub fn cache_info(&self) -> CacheSize { + pub fn blockchain_cache_info(&self) -> BlockChainCacheSize { self.chain.read().unwrap().cache_size() } @@ -371,6 +381,7 @@ impl Client { /// Tick the client. pub fn tick(&self) { self.chain.read().unwrap().collect_garbage(); + self.block_queue.read().unwrap().collect_garbage(); } /// Set up the cache behaviour. @@ -458,7 +469,11 @@ impl BlockChainClient for Client { } fn tree_route(&self, from: &H256, to: &H256) -> Option { - self.chain.read().unwrap().tree_route(from.clone(), to.clone()) + let chain = self.chain.read().unwrap(); + match chain.is_known(from) && chain.is_known(to) { + true => Some(chain.tree_route(from.clone(), to.clone())), + false => None + } } fn state_data(&self, _hash: &H256) -> Option { diff --git a/ethcore/src/evm/ext.rs b/ethcore/src/evm/ext.rs index ae4cff3be..f4172f10a 100644 --- a/ethcore/src/evm/ext.rs +++ b/ethcore/src/evm/ext.rs @@ -16,9 +16,7 @@ //! Interface for Evm externalities. -use common::Bytes; -use util::hash::*; -use util::uint::*; +use util::common::*; use evm::{Schedule, Error}; use env_info::*; @@ -60,22 +58,22 @@ pub trait Ext { fn blockhash(&self, number: &U256) -> H256; /// Creates new contract. - /// + /// /// Returns gas_left and contract address if contract creation was succesfull. fn create(&mut self, gas: &U256, value: &U256, code: &[u8]) -> ContractCreateResult; /// Message call. - /// + /// /// Returns Err, if we run out of gas. - /// Otherwise returns call_result which contains gas left + /// Otherwise returns call_result which contains gas left /// and true if subcall was successfull. - fn call(&mut self, - gas: &U256, - sender_address: &Address, - receive_address: &Address, + fn call(&mut self, + gas: &U256, + sender_address: &Address, + receive_address: &Address, value: Option, - data: &[u8], - code_address: &Address, + data: &[u8], + code_address: &Address, output: &mut [u8]) -> MessageCallResult; /// Returns code at given address @@ -99,7 +97,7 @@ pub trait Ext { fn env_info(&self) -> &EnvInfo; /// Returns current depth of execution. - /// + /// /// If contract A calls contract B, and contract B calls C, /// then A depth is 0, B is 1, C is 2 and so on. fn depth(&self) -> usize; diff --git a/ethcore/src/extras.rs b/ethcore/src/extras.rs index 128357576..f4759b040 100644 --- a/ethcore/src/extras.rs +++ b/ethcore/src/extras.rs @@ -262,15 +262,6 @@ impl Encodable for BlocksBlooms { } } -/// Represents location of bloom in database. -#[derive(Debug, PartialEq)] -pub struct BlocksBloomLocation { - /// Unique hash of BlocksBloom - pub hash: H256, - /// Index within BlocksBloom - pub index: usize, -} - /// Represents address of certain transaction within block #[derive(Clone)] pub struct TransactionAddress { diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index a386e2854..89bd5da2b 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use super::test_common::*; -use client::{BlockChainClient,Client}; +use client::{BlockChainClient, Client, ClientConfig}; use pod_state::*; use block::Block; use ethereum; @@ -53,7 +53,7 @@ pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec { let temp = RandomTempPath::new(); { - let client = Client::new(spec, temp.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), spec, temp.as_path(), IoChannel::disconnected()).unwrap(); assert_eq!(client.chain_info().best_block_hash, genesis_hash); for (b, is_valid) in blocks.into_iter() { if Block::is_good(&b) { diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 5eb3beeb2..938da02a0 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -75,7 +75,7 @@ #[macro_use] extern crate ethcore_util as util; #[macro_use] extern crate lazy_static; extern crate rustc_serialize; -extern crate heapsize; +#[macro_use] extern crate heapsize; extern crate crypto; extern crate time; extern crate env_logger; @@ -86,8 +86,6 @@ extern crate crossbeam; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; -pub mod blockchain; -pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -121,6 +119,8 @@ mod substate; mod executive; mod externalities; mod verification; +mod block_queue; +mod blockchain; #[cfg(test)] mod tests; diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index cdf3425e8..756d02407 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -20,7 +20,7 @@ use util::*; use util::panics::*; use spec::Spec; use error::*; -use client::Client; +use client::{Client, ClientConfig}; /// Message type for external and internal events #[derive(Clone)] @@ -48,14 +48,14 @@ pub struct ClientService { impl ClientService { /// Start the service in a separate thread. - pub fn start(spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { + pub fn start(config: ClientConfig, spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { let panic_handler = PanicHandler::new_in_arc(); let mut net_service = try!(NetworkService::start(net_config)); panic_handler.forward_from(&net_service); info!("Starting {}", net_service.host_info()); info!("Configured for {} using {} engine", spec.name, spec.engine_name); - let client = try!(Client::new(spec, db_path, net_service.io().channel())); + let client = try!(Client::new(config, spec, db_path, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { client: client.clone() @@ -135,12 +135,13 @@ mod tests { use tests::helpers::*; use util::network::*; use devtools::*; + use client::ClientConfig; #[test] fn it_can_be_started() { let spec = get_test_spec(); let temp_path = RandomTempPath::new(); - let service = ClientService::start(spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); + let service = ClientService::start(ClientConfig::default(), spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); assert!(service.is_ok()); } } diff --git a/ethcore/src/spec.rs b/ethcore/src/spec.rs index 5714ca734..38a0dda53 100644 --- a/ethcore/src/spec.rs +++ b/ethcore/src/spec.rs @@ -58,6 +58,8 @@ pub struct Spec { /// Known nodes on the network in enode format. pub nodes: Vec, + /// Network ID + pub network_id: U256, /// Parameters concerning operation of the specific engine we're using. /// Maps the parameter name to an RLP-encoded value. @@ -120,6 +122,9 @@ impl Spec { /// Get the known knodes of the network in enode format. pub fn nodes(&self) -> &Vec { &self.nodes } + /// Get the configured Network ID. + pub fn network_id(&self) -> U256 { self.network_id } + /// Get the header of the genesis block. pub fn genesis_header(&self) -> Header { Header { @@ -250,6 +255,7 @@ impl FromJson for Spec { engine_name: json["engineName"].as_string().unwrap().to_owned(), engine_params: json_to_rlp_map(&json["params"]), nodes: nodes, + network_id: U256::from_str(&json["params"]["networkID"].as_string().unwrap()[2..]).unwrap(), builtins: builtins, parent_hash: H256::from_str(&genesis["parentHash"].as_string().unwrap()[2..]).unwrap(), author: Address::from_str(&genesis["author"].as_string().unwrap()[2..]).unwrap(), diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 8bbf317c1..c13678c38 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -138,7 +138,7 @@ impl State { /// Create a new contract at address `contract`. If there is already an account at the address /// it will have its code reset, ready for `init_code()`. pub fn new_contract(&mut self, contract: &Address, balance: U256) { - self.insert_cache(&contract, Some(Account::new_contract(balance))); + self.insert_cache(&contract, Some(Account::new_contract(balance, self.account_start_nonce))); } /// Remove an existing account. @@ -204,7 +204,7 @@ impl State { /// Initialise the code of account `a` so that it is `value` for `key`. /// NOTE: Account should have been created with `new_contract`. pub fn init_code(&mut self, a: &Address, code: Bytes) { - self.require_or_from(a, true, || Account::new_contract(U256::from(0u8)), |_|{}).init_code(code); + self.require_or_from(a, true, || Account::new_contract(x!(0), self.account_start_nonce), |_|{}).init_code(code); } /// Execute a given transaction. @@ -335,10 +335,9 @@ impl fmt::Debug for State { mod tests { use super::*; -use util::hash::*; +use util::common::*; use util::trie::*; use util::rlp::*; -use util::uint::*; use account::*; use tests::helpers::*; use devtools::*; @@ -349,7 +348,7 @@ fn code_from_database() { let temp = RandomTempPath::new(); let (root, db) = { let mut state = get_temp_state_in(temp.as_path()); - state.require_or_from(&a, false, ||Account::new_contract(U256::from(42u32)), |_|{}); + state.require_or_from(&a, false, ||Account::new_contract(x!(42), x!(0)), |_|{}); state.init_code(&a, vec![1, 2, 3]); assert_eq!(state.code(&a), Some([1u8, 2, 3].to_vec())); state.commit(); diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index af25d1b72..83df81fa2 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client, BlockId}; +use client::{BlockChainClient, Client, ClientConfig, BlockId}; use tests::helpers::*; use common::*; use devtools::*; @@ -22,14 +22,14 @@ use devtools::*; #[test] fn created() { let dir = RandomTempPath::new(); - let client_result = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()); + let client_result = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()); assert!(client_result.is_ok()); } #[test] fn imports_from_empty() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); client.import_verified_blocks(&IoChannel::disconnected()); client.flush_queue(); } @@ -37,7 +37,7 @@ fn imports_from_empty() { #[test] fn imports_good_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let good_block = get_good_dummy_block(); if let Err(_) = client.import_block(good_block) { panic!("error importing block being good by definition"); @@ -52,7 +52,7 @@ fn imports_good_block() { #[test] fn query_none_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let non_existant = client.block_header(BlockId::Number(188)); assert!(non_existant.is_none()); @@ -104,5 +104,5 @@ fn can_collect_garbage() { let client_result = generate_dummy_client(100); let client = client_result.reference(); client.tick(); - assert!(client.cache_info().blocks < 100 * 1024); + assert!(client.blockchain_cache_info().blocks < 100 * 1024); } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index f1a4e6c03..44ad667b9 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client}; +use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; -use blockchain::{BlockChain}; +use blockchain::{BlockChain, BlockChainConfig}; use state::*; use evm::{Schedule, Factory}; use engine::*; @@ -134,7 +134,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[&SignedTrans pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let test_spec = get_test_spec(); let test_engine = test_spec.to_engine().unwrap(); let state_root = test_engine.spec().genesis_header().state_root; @@ -172,7 +172,7 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); for block in &blocks { if let Err(_) = client.import_block(block.clone()) { panic!("panic importing block which is well-formed"); @@ -189,7 +189,7 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block(block_order, bc.best_block_hash()), vec![]); } @@ -202,7 +202,7 @@ pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None), vec![]); } @@ -215,7 +215,7 @@ pub fn generate_dummy_blockchain_with_extra(block_number: u32) -> GuardedTempRes pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); GuardedTempResult:: { _temp: temp, @@ -250,6 +250,25 @@ pub fn get_temp_state_in(path: &Path) -> State { State::new(journal_db, U256::from(0u8)) } +pub fn get_good_dummy_block_seq(count: usize) -> Vec { + let test_spec = get_test_spec(); + let test_engine = test_spec.to_engine().unwrap(); + let mut parent = test_engine.spec().genesis_header().hash(); + let mut r = Vec::new(); + for i in 1 .. count + 1 { + let mut block_header = Header::new(); + block_header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap()); + block_header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap()); + block_header.timestamp = i as u64; + block_header.number = i as u64; + block_header.parent_hash = parent; + block_header.state_root = test_engine.spec().genesis_header().state_root; + parent = block_header.hash(); + r.push(create_test_block(&block_header)); + } + r +} + pub fn get_good_dummy_block() -> Bytes { let mut block_header = Header::new(); let test_spec = get_test_spec(); diff --git a/ethcore/src/transaction.rs b/ethcore/src/transaction.rs index fc9886a4d..a51824494 100644 --- a/ethcore/src/transaction.rs +++ b/ethcore/src/transaction.rs @@ -100,10 +100,10 @@ impl FromJson for SignedTransaction { v: match json.find("v") { Some(ref j) => u16::from_json(j) as u8, None => 0 }, r: match json.find("r") { Some(j) => xjson!(j), None => x!(0) }, s: match json.find("s") { Some(j) => xjson!(j), None => x!(0) }, - hash: RefCell::new(None), + hash: Cell::new(None), sender: match json.find("sender") { - Some(&Json::String(ref sender)) => RefCell::new(Some(address_from_hex(clean(sender)))), - _ => RefCell::new(None), + Some(&Json::String(ref sender)) => Cell::new(Some(address_from_hex(clean(sender)))), + _ => Cell::new(None), } } } @@ -127,8 +127,8 @@ impl Transaction { r: r, s: s, v: v + 27, - hash: RefCell::new(None), - sender: RefCell::new(None) + hash: Cell::new(None), + sender: Cell::new(None), } } @@ -140,8 +140,8 @@ impl Transaction { r: U256::zero(), s: U256::zero(), v: 0, - hash: RefCell::new(None), - sender: RefCell::new(None) + hash: Cell::new(None), + sender: Cell::new(None), } } @@ -171,9 +171,9 @@ pub struct SignedTransaction { /// The S field of the signature; helps describe the point on the curve. s: U256, /// Cached hash. - hash: RefCell>, + hash: Cell>, /// Cached sender. - sender: RefCell> + sender: Cell>, } impl PartialEq for SignedTransaction { @@ -208,8 +208,8 @@ impl Decodable for SignedTransaction { v: try!(d.val_at(6)), r: try!(d.val_at(7)), s: try!(d.val_at(8)), - hash: RefCell::new(None), - sender: RefCell::new(None), + hash: Cell::new(None), + sender: Cell::new(None), }) } } @@ -238,13 +238,14 @@ impl SignedTransaction { /// Get the hash of this header (sha3 of the RLP). pub fn hash(&self) -> H256 { - let mut hash = self.hash.borrow_mut(); - match &mut *hash { - &mut Some(ref h) => h.clone(), - hash @ &mut None => { - *hash = Some(self.rlp_sha3()); - hash.as_ref().unwrap().clone() - } + let hash = self.hash.get(); + match hash { + Some(h) => h, + None => { + let h = self.rlp_sha3(); + self.hash.set(Some(h)); + h + } } } @@ -265,13 +266,14 @@ impl SignedTransaction { /// Returns transaction sender. pub fn sender(&self) -> Result { - let mut sender = self.sender.borrow_mut(); - match &mut *sender { - &mut Some(ref h) => Ok(h.clone()), - sender @ &mut None => { - *sender = Some(From::from(try!(ec::recover(&self.signature(), &self.unsigned.hash())).sha3())); - Ok(sender.as_ref().unwrap().clone()) - } + let sender = self.sender.get(); + match sender { + Some(s) => Ok(s), + None => { + let s = Address::from(try!(ec::recover(&self.signature(), &self.unsigned.hash())).sha3()); + self.sender.set(Some(s)); + Ok(s) + } } } diff --git a/parity/main.rs b/parity/main.rs index c0c260371..3d0d183a3 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -31,6 +31,7 @@ extern crate ctrlc; extern crate fdlimit; extern crate daemonize; extern crate time; +extern crate number_prefix; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; @@ -47,10 +48,10 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethcore::blockchain::CacheSize; -use ethsync::EthSync; +use ethsync::{EthSync, SyncConfig}; use docopt::Docopt; use daemonize::Daemonize; +use number_prefix::{binary_prefix, Standalone, Prefixed}; const USAGE: &'static str = r#" Parity. Ethereum Client. @@ -78,6 +79,7 @@ Options: --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in bytes [default: 262144]. + --queue-max-size BYTES Specify the maximum size of memory to use for block queue [default: 52428800]. -j --jsonrpc Enable the JSON-RPC API sever. --jsonrpc-url URL Specify URL for JSON-RPC API server [default: 127.0.0.1:8545]. @@ -106,6 +108,7 @@ struct Args { flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, + flag_queue_max_size: usize, flag_jsonrpc: bool, flag_jsonrpc_url: String, flag_jsonrpc_cors: String, @@ -145,9 +148,9 @@ fn setup_rpc_server(client: Arc, sync: Arc, url: &str, cors_dom let mut server = rpc::HttpServer::new(1); server.add_delegate(Web3Client::new().to_delegate()); - server.add_delegate(EthClient::new(client.clone(), sync.clone()).to_delegate()); - server.add_delegate(EthFilterClient::new(client).to_delegate()); - server.add_delegate(NetClient::new(sync).to_delegate()); + server.add_delegate(EthClient::new(&client, &sync).to_delegate()); + server.add_delegate(EthFilterClient::new(&client).to_delegate()); + server.add_delegate(NetClient::new(&sync).to_delegate()); server.start_async(url, cors_domain); } @@ -283,14 +286,19 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); + let mut sync_config = SyncConfig::default(); + sync_config.network_id = spec.network_id(); // Build client - let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); + let mut client_config = ClientConfig::default(); + client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; + client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.queue.max_mem_use = self.args.flag_queue_max_size; + let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); - client.configure_cache(self.args.flag_cache_pref_size, self.args.flag_cache_max_size); // Sync - let sync = EthSync::register(service.network(), client); + let sync = EthSync::register(service.network(), sync_config, client); // Setup rpc if self.args.flag_jsonrpc { @@ -331,7 +339,7 @@ fn main() { struct Informant { chain_info: RwLock>, - cache_info: RwLock>, + cache_info: RwLock>, report: RwLock>, } @@ -346,18 +354,26 @@ impl Default for Informant { } impl Informant { + + fn format_bytes(b: usize) -> String { + match binary_prefix(b as f64) { + Standalone(bytes) => format!("{} bytes", bytes), + Prefixed(prefix, n) => format!("{:.0} {}B", n, prefix), + } + } + pub fn tick(&self, client: &Client, sync: &EthSync) { // 5 seconds betwen calls. TODO: calculate this properly. let dur = 5usize; let chain_info = client.chain_info(); let queue_info = client.queue_info(); - let cache_info = client.cache_info(); + let cache_info = client.blockchain_cache_info(); let report = client.report(); let sync_info = sync.status(); - if let (_, &Some(ref last_cache_info), &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { - println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// {} ({}) bl {} ({}) ex ]", + if let (_, _, &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { + println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// mem: {} chain, {} queue, {} sync ]", chain_info.best_block_number, chain_info.best_block_hash, (report.blocks_imported - last_report.blocks_imported) / dur, @@ -370,10 +386,9 @@ impl Informant { queue_info.unverified_queue_size, queue_info.verified_queue_size, - cache_info.blocks, - cache_info.blocks as isize - last_cache_info.blocks as isize, - cache_info.block_details, - cache_info.block_details as isize - last_cache_info.block_details as isize + Informant::format_bytes(cache_info.total()), + Informant::format_bytes(queue_info.mem_used), + Informant::format_bytes(sync_info.mem_used), ); } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 3c9851ed3..74b722cd4 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -9,20 +9,20 @@ build = "build.rs" [lib] [dependencies] -serde = "0.6.7" -serde_json = "0.6.0" -jsonrpc-core = "1.1" -jsonrpc-http-server = "2.0" +serde = "0.7.0" +serde_json = "0.7.0" +jsonrpc-core = "1.2" +jsonrpc-http-server = "2.1" ethcore-util = { path = "../util" } ethcore = { path = "../ethcore" } ethsync = { path = "../sync" } clippy = { version = "0.0.44", optional = true } rustc-serialize = "0.3" -serde_macros = { version = "0.6.13", optional = true } transient-hashmap = "0.1" +serde_macros = { version = "0.7.0", optional = true } [build-dependencies] -serde_codegen = { version = "0.6.13", optional = true } +serde_codegen = { version = "0.7.0", optional = true } syntex = "0.29.0" [features] diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 70d6fae82..5b3545653 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -16,8 +16,8 @@ //! Ethcore rpc. #![warn(missing_docs)] -#![cfg_attr(nightly, feature(custom_derive, custom_attribute, plugin))] -#![cfg_attr(nightly, plugin(serde_macros, clippy))] +#![cfg_attr(feature="nightly", feature(custom_derive, custom_attribute, plugin))] +#![cfg_attr(feature="nightly", plugin(serde_macros, clippy))] extern crate rustc_serialize; extern crate serde; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index e2cc1a5c3..32c143a84 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -15,11 +15,10 @@ // along with Parity. If not, see . //! Eth rpc implementation. -use std::sync::{Mutex, Arc}; +use std::sync::{Arc, Weak, Mutex}; use ethsync::{EthSync, SyncState}; use jsonrpc_core::*; -use util::hash::*; -use util::uint::*; +use util::numbers::*; use util::sha3::*; use ethcore::client::*; use ethcore::views::*; @@ -30,21 +29,22 @@ use v1::helpers::{PollFilter, PollIndexer}; /// Eth rpc implementation. pub struct EthClient { - client: Arc, - sync: Arc + client: Weak, + sync: Weak } impl EthClient { /// Creates new EthClient. - pub fn new(client: Arc, sync: Arc) -> Self { + pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { - client: client, - sync: sync + client: Arc::downgrade(client), + sync: Arc::downgrade(sync) } } fn block(&self, id: BlockId, include_txs: bool) -> Result { - match (self.client.block(id.clone()), self.client.block_total_difficulty(id)) { + let client = take_weak!(self.client); + match (client.block(id.clone()), client.block_total_difficulty(id)) { (Some(bytes), Some(total_difficulty)) => { let block_view = BlockView::new(&bytes); let view = block_view.header_view(); @@ -79,9 +79,9 @@ impl EthClient { _ => Ok(Value::Null) } } - + fn transaction(&self, id: TransactionId) -> Result { - match self.client.transaction(id) { + match take_weak!(self.client).transaction(id) { Some(t) => to_value(&Transaction::from(t)), None => Ok(Value::Null) } @@ -91,7 +91,7 @@ impl EthClient { impl Eth for EthClient { fn protocol_version(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.sync.status().protocol_version)), + Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), _ => Err(Error::invalid_params()) } } @@ -99,12 +99,12 @@ impl Eth for EthClient { fn syncing(&self, params: Params) -> Result { match params { Params::None => { - let status = self.sync.status(); + let status = take_weak!(self.sync).status(); let res = match status.state { SyncState::NotSynced | SyncState::Idle => SyncStatus::None, SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks => SyncStatus::Info(SyncInfo { starting_block: U256::from(status.start_block_number), - current_block: U256::from(self.client.chain_info().best_block_number), + current_block: U256::from(take_weak!(self.client).chain_info().best_block_number), highest_block: U256::from(status.highest_block_number.unwrap_or(status.start_block_number)) }) }; @@ -147,14 +147,14 @@ impl Eth for EthClient { fn block_number(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.client.chain_info().best_block_number)), + Params::None => to_value(&U256::from(take_weak!(self.client).chain_info().best_block_number)), _ => Err(Error::invalid_params()) } } fn block_transaction_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).transactions_count()), None => Ok(Value::Null) }) @@ -162,7 +162,7 @@ impl Eth for EthClient { fn block_uncles_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).uncles_count()), None => Ok(Value::Null) }) @@ -171,7 +171,7 @@ impl Eth for EthClient { // TODO: do not ignore block number param fn code_at(&self, params: Params) -> Result { from_params::<(Address, BlockNumber)>(params) - .and_then(|(address, _block_number)| to_value(&self.client.code(&address).map_or_else(Bytes::default, Bytes::new))) + .and_then(|(address, _block_number)| to_value(&take_weak!(self.client).code(&address).map_or_else(Bytes::default, Bytes::new))) } fn block_by_hash(&self, params: Params) -> Result { @@ -202,7 +202,7 @@ impl Eth for EthClient { fn logs(&self, params: Params) -> Result { from_params::<(Filter,)>(params) .and_then(|(filter,)| { - let logs = self.client.logs(filter.into()) + let logs = take_weak!(self.client).logs(filter.into()) .into_iter() .map(From::from) .collect::>(); @@ -213,15 +213,15 @@ impl Eth for EthClient { /// Eth filter rpc implementation. pub struct EthFilterClient { - client: Arc, - polls: Mutex> + client: Weak, + polls: Mutex>, } impl EthFilterClient { /// Creates new Eth filter client. - pub fn new(client: Arc) -> Self { + pub fn new(client: &Arc) -> Self { EthFilterClient { - client: client, + client: Arc::downgrade(client), polls: Mutex::new(PollIndexer::new()) } } @@ -232,7 +232,7 @@ impl EthFilter for EthFilterClient { from_params::<(Filter,)>(params) .and_then(|(filter,)| { let mut polls = self.polls.lock().unwrap(); - let id = polls.create_poll(PollFilter::Logs(filter.into()), self.client.chain_info().best_block_number); + let id = polls.create_poll(PollFilter::Logs(filter.into()), take_weak!(self.client).chain_info().best_block_number); to_value(&U256::from(id)) }) } @@ -241,7 +241,7 @@ impl EthFilter for EthFilterClient { match params { Params::None => { let mut polls = self.polls.lock().unwrap(); - let id = polls.create_poll(PollFilter::Block, self.client.chain_info().best_block_number); + let id = polls.create_poll(PollFilter::Block, take_weak!(self.client).chain_info().best_block_number); to_value(&U256::from(id)) }, _ => Err(Error::invalid_params()) @@ -252,7 +252,7 @@ impl EthFilter for EthFilterClient { match params { Params::None => { let mut polls = self.polls.lock().unwrap(); - let id = polls.create_poll(PollFilter::PendingTransaction, self.client.chain_info().best_block_number); + let id = polls.create_poll(PollFilter::PendingTransaction, take_weak!(self.client).chain_info().best_block_number); to_value(&U256::from(id)) }, _ => Err(Error::invalid_params()) @@ -260,6 +260,7 @@ impl EthFilter for EthFilterClient { } fn filter_changes(&self, params: Params) -> Result { + let client = take_weak!(self.client); from_params::<(Index,)>(params) .and_then(|(index,)| { let info = self.polls.lock().unwrap().get_poll(&index.value()).cloned(); @@ -267,10 +268,10 @@ impl EthFilter for EthFilterClient { None => Ok(Value::Array(vec![] as Vec)), Some(info) => match info.filter { PollFilter::Block => { - let current_number = self.client.chain_info().best_block_number; + let current_number = client.chain_info().best_block_number; let hashes = (info.block_number..current_number).into_iter() .map(BlockId::Number) - .filter_map(|id| self.client.block_hash(id)) + .filter_map(|id| client.block_hash(id)) .collect::>(); self.polls.lock().unwrap().update_poll(&index.value(), current_number); @@ -279,17 +280,17 @@ impl EthFilter for EthFilterClient { }, PollFilter::PendingTransaction => { // TODO: fix implementation - to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + to_value(&client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) }, PollFilter::Logs(mut filter) => { filter.from_block = BlockId::Number(info.block_number); filter.to_block = BlockId::Latest; - let logs = self.client.logs(filter) + let logs = client.logs(filter) .into_iter() .map(From::from) .collect::>(); - - let current_number = self.client.chain_info().best_block_number; + + let current_number = client.chain_info().best_block_number; self.polls.lock().unwrap().update_poll(&index.value(), current_number); to_value(&logs) diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs index bc8b436fd..d9102b1db 100644 --- a/rpc/src/v1/impls/mod.rs +++ b/rpc/src/v1/impls/mod.rs @@ -15,6 +15,16 @@ // along with Parity. If not, see . //! Ethereum rpc interface implementation. + +macro_rules! take_weak { + ($weak: expr) => { + match $weak.upgrade() { + Some(arc) => arc, + None => return Err(Error::internal_error()) + } + } +} + mod web3; mod eth; mod net; diff --git a/rpc/src/v1/impls/net.rs b/rpc/src/v1/impls/net.rs index b36042dba..9e24caad2 100644 --- a/rpc/src/v1/impls/net.rs +++ b/rpc/src/v1/impls/net.rs @@ -15,31 +15,31 @@ // along with Parity. If not, see . //! Net rpc implementation. -use std::sync::Arc; +use std::sync::{Arc, Weak}; use jsonrpc_core::*; use ethsync::EthSync; use v1::traits::Net; /// Net rpc implementation. pub struct NetClient { - sync: Arc + sync: Weak } impl NetClient { /// Creates new NetClient. - pub fn new(sync: Arc) -> Self { + pub fn new(sync: &Arc) -> Self { NetClient { - sync: sync + sync: Arc::downgrade(sync) } } } impl Net for NetClient { fn version(&self, _: Params) -> Result { - Ok(Value::U64(self.sync.status().protocol_version as u64)) + Ok(Value::U64(take_weak!(self.sync).status().protocol_version as u64)) } fn peer_count(&self, _params: Params) -> Result { - Ok(Value::U64(self.sync.status().num_peers as u64)) + Ok(Value::U64(take_weak!(self.sync).status().num_peers as u64)) } } diff --git a/rpc/src/v1/types/block.rs b/rpc/src/v1/types/block.rs index b92111bcb..2457efcf8 100644 --- a/rpc/src/v1/types/block.rs +++ b/rpc/src/v1/types/block.rs @@ -15,8 +15,7 @@ // along with Parity. If not, see . use serde::{Serialize, Serializer}; -use util::hash::*; -use util::uint::*; +use util::numbers::*; use v1::types::{Bytes, Transaction, OptionalValue}; #[derive(Debug)] @@ -71,8 +70,7 @@ pub struct Block { #[cfg(test)] mod tests { use serde_json; - use util::hash::*; - use util::uint::*; + use util::numbers::*; use v1::types::{Transaction, Bytes, OptionalValue}; use super::*; diff --git a/rpc/src/v1/types/block_number.rs b/rpc/src/v1/types/block_number.rs index bb563de99..c955ec895 100644 --- a/rpc/src/v1/types/block_number.rs +++ b/rpc/src/v1/types/block_number.rs @@ -30,7 +30,7 @@ pub enum BlockNumber { impl Deserialize for BlockNumber { fn deserialize(deserializer: &mut D) -> Result where D: Deserializer { - deserializer.visit(BlockNumberVisitor) + deserializer.deserialize(BlockNumberVisitor) } } @@ -44,8 +44,8 @@ impl Visitor for BlockNumberVisitor { "latest" => Ok(BlockNumber::Latest), "earliest" => Ok(BlockNumber::Earliest), "pending" => Ok(BlockNumber::Pending), - _ if value.starts_with("0x") => u64::from_str_radix(&value[2..], 16).map(BlockNumber::Num).map_err(|_| Error::syntax("invalid block number")), - _ => value.parse::().map(BlockNumber::Num).map_err(|_| Error::syntax("invalid block number")) + _ if value.starts_with("0x") => u64::from_str_radix(&value[2..], 16).map(BlockNumber::Num).map_err(|_| Error::custom("invalid block number")), + _ => value.parse::().map(BlockNumber::Num).map_err(|_| Error::custom("invalid block number")) } } diff --git a/rpc/src/v1/types/bytes.rs b/rpc/src/v1/types/bytes.rs index d6a648d7c..f09f24e4d 100644 --- a/rpc/src/v1/types/bytes.rs +++ b/rpc/src/v1/types/bytes.rs @@ -40,7 +40,7 @@ impl Serialize for Bytes { where S: Serializer { let mut serialized = "0x".to_owned(); serialized.push_str(self.0.to_hex().as_ref()); - serializer.visit_str(serialized.as_ref()) + serializer.serialize_str(serialized.as_ref()) } } diff --git a/rpc/src/v1/types/filter.rs b/rpc/src/v1/types/filter.rs index c5c3604dd..16a08764b 100644 --- a/rpc/src/v1/types/filter.rs +++ b/rpc/src/v1/types/filter.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Deserializer, Error}; use serde_json::value; use jsonrpc_core::Value; -use util::hash::*; +use util::numbers::*; use v1::types::BlockNumber; use ethcore::filter::Filter as EthFilter; use ethcore::client::BlockId; @@ -40,7 +40,7 @@ impl Deserialize for VariadicValue where T: Deserialize { Deserialize::deserialize(&mut value::Deserializer::new(v.clone())).map(VariadicValue::Single) .or_else(|_| Deserialize::deserialize(&mut value::Deserializer::new(v.clone())).map(VariadicValue::Multiple)) - .map_err(|_| Error::syntax("")) // unreachable, but types must match + .map_err(|_| Error::custom("")) // unreachable, but types must match } } @@ -48,6 +48,7 @@ pub type FilterAddress = VariadicValue
; pub type Topic = VariadicValue; #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Filter { #[serde(rename="fromBlock")] pub from_block: Option, diff --git a/rpc/src/v1/types/index.rs b/rpc/src/v1/types/index.rs index a77096fbf..e7cbbd255 100644 --- a/rpc/src/v1/types/index.rs +++ b/rpc/src/v1/types/index.rs @@ -30,7 +30,7 @@ impl Index { impl Deserialize for Index { fn deserialize(deserializer: &mut D) -> Result where D: Deserializer { - deserializer.visit(IndexVisitor) + deserializer.deserialize(IndexVisitor) } } @@ -41,8 +41,8 @@ impl Visitor for IndexVisitor { fn visit_str(&mut self, value: &str) -> Result where E: Error { match value { - _ if value.starts_with("0x") => usize::from_str_radix(&value[2..], 16).map(Index).map_err(|_| Error::syntax("invalid index")), - _ => value.parse::().map(Index).map_err(|_| Error::syntax("invalid index")) + _ if value.starts_with("0x") => usize::from_str_radix(&value[2..], 16).map(Index).map_err(|_| Error::custom("invalid index")), + _ => value.parse::().map(Index).map_err(|_| Error::custom("invalid index")) } } diff --git a/rpc/src/v1/types/log.rs b/rpc/src/v1/types/log.rs index 0629c5534..0f5fdee15 100644 --- a/rpc/src/v1/types/log.rs +++ b/rpc/src/v1/types/log.rs @@ -14,8 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use util::hash::*; -use util::uint::*; +use util::numbers::*; use ethcore::log_entry::LocalizedLogEntry; use v1::types::Bytes; @@ -55,8 +54,7 @@ impl From for Log { mod tests { use serde_json; use std::str::FromStr; - use util::hash::*; - use util::uint::*; + use util::numbers::*; use v1::types::{Bytes, Log}; #[test] @@ -66,7 +64,7 @@ mod tests { let log = Log { address: Address::from_str("33990122638b9132ca29c723bdf037f1a891a70c").unwrap(), topics: vec![ - H256::from_str("a6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc").unwrap(), + H256::from_str("a6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc").unwrap(), H256::from_str("4861736852656700000000000000000000000000000000000000000000000000").unwrap() ], data: Bytes::new(vec![]), diff --git a/rpc/src/v1/types/sync.rs b/rpc/src/v1/types/sync.rs index b5568acda..c0e480140 100644 --- a/rpc/src/v1/types/sync.rs +++ b/rpc/src/v1/types/sync.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use serde::{Serialize, Serializer}; -use util::uint::*; +use util::numbers::*; #[derive(Default, Debug, Serialize, PartialEq)] pub struct SyncInfo { diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index 0e9256ada..232cf0bf3 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -14,8 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use util::hash::*; -use util::uint::*; +use util::numbers::*; use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 7a81c7d97..26a7d463c 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -15,6 +15,7 @@ log = "0.3" env_logger = "0.3" time = "0.1.34" rand = "0.3.13" +heapsize = "0.3" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 930518007..85380f3cd 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -39,7 +39,9 @@ use ethcore::error::*; use ethcore::block::Block; use io::SyncIo; use time; -use std::option::Option; +use super::SyncConfig; + +known_heap_size!(0, PeerInfo, Header, HeaderId); impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { @@ -80,9 +82,7 @@ const NODE_DATA_PACKET: u8 = 0x0e; const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; -const NETWORK_ID: U256 = ONE_U256; //TODO: get this from parent - -const CONNECTION_TIMEOUT_SEC: f64 = 10f64; +const CONNECTION_TIMEOUT_SEC: f64 = 5f64; struct Header { /// Header data @@ -135,6 +135,8 @@ pub struct SyncStatus { pub num_peers: usize, /// Total number of active peers pub num_active_peers: usize, + /// Heap memory used in bytes + pub mem_used: usize, } #[derive(PartialEq, Eq, Debug, Clone)] @@ -203,13 +205,17 @@ pub struct ChainSync { have_common_block: bool, /// Last propagated block number last_send_block_number: BlockNumber, + /// Max blocks to download ahead + max_download_ahead_blocks: usize, + /// Network ID + network_id: U256, } type RlpResponseResult = Result, PacketDecodeError>; impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new() -> ChainSync { + pub fn new(config: SyncConfig) -> ChainSync { ChainSync { state: SyncState::NotSynced, starting_block: 0, @@ -226,6 +232,8 @@ impl ChainSync { syncing_difficulty: U256::from(0u64), have_common_block: false, last_send_block_number: 0, + max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), + network_id: config.network_id, } } @@ -237,10 +245,19 @@ impl ChainSync { start_block_number: self.starting_block, last_imported_block_number: self.last_imported_block, highest_block_number: self.highest_block, - blocks_received: match self.last_imported_block { None => 0, Some(x) => x - self.starting_block }, - blocks_total: match self.highest_block { None => 0, Some(x) => x - self.starting_block }, + blocks_received: match self.last_imported_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, + blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), + mem_used: + // TODO: https://github.com/servo/heapsize/pull/50 + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + + self.header_ids.heap_size_of_children(), } } @@ -275,7 +292,6 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; - io.chain().clear_queue(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -307,7 +323,7 @@ impl ChainSync { trace!(target: "sync", "Peer {} genesis hash not matched", peer_id); return Ok(()); } - if peer.network_id != NETWORK_ID { + if peer.network_id != self.network_id { io.disable_peer(peer_id); trace!(target: "sync", "Peer {} network id not matched", peer_id); return Ok(()); @@ -436,7 +452,7 @@ impl ChainSync { trace!(target: "sync", "Got body {}", n); } None => { - debug!(target: "sync", "Ignored unknown block body"); + trace!(target: "sync", "Ignored unknown/stale block body"); } } } @@ -611,7 +627,7 @@ impl ChainSync { self.request_headers_by_hash(io, peer_id, &peer_latest, 1, 0, false); } else if self.state == SyncState::Blocks && io.chain().block_status(BlockId::Hash(peer_latest)) == BlockStatus::Unknown { - self.request_blocks(io, peer_id); + self.request_blocks(io, peer_id, false); } } @@ -620,7 +636,7 @@ impl ChainSync { } /// Find some headers or blocks to download for a peer. - fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId) { + fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, ignore_others: bool) { self.clear_peer_download(peer_id); if io.chain().queue_info().is_full() { @@ -640,28 +656,34 @@ impl ChainSync { let mut index: BlockNumber = 0; while index != items.len() as BlockNumber && needed_bodies.len() < MAX_BODIES_TO_REQUEST { let block = start + index; - if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { + if ignore_others || (!self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block)) { needed_bodies.push(items[index as usize].hash.clone()); needed_numbers.push(block); - self.downloading_bodies.insert(block); } index += 1; } } } if !needed_bodies.is_empty() { + let (head, _) = self.headers.range_iter().next().unwrap(); + if needed_numbers.first().unwrap() - head > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading block bodies", peer_id, needed_numbers.first().unwrap(), head); + self.request_blocks(io, peer_id, true); + return; + } + self.downloading_bodies.extend(needed_numbers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, needed_numbers); self.request_bodies(io, peer_id, needed_bodies); } else { // check if need to download headers - let mut start = 0usize; + let mut start = 0; if !self.have_common_block { // download backwards until common block is found 1 header at a time let chain_info = io.chain().chain_info(); - start = chain_info.best_block_number as usize; + start = chain_info.best_block_number; if !self.headers.is_empty() { - start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); + start = min(start, self.headers.range_iter().next().unwrap().0 - 1); } if start == 0 { self.have_common_block = true; //reached genesis @@ -672,6 +694,7 @@ impl ChainSync { if self.have_common_block { let mut headers: Vec = Vec::new(); let mut prev = self.current_base_block() + 1; + let head = self.headers.range_iter().next().map(|(h, _)| h); for (next, ref items) in self.headers.range_iter() { if !headers.is_empty() { break; @@ -682,9 +705,8 @@ impl ChainSync { } let mut block = prev; while block < next && headers.len() < MAX_HEADERS_TO_REQUEST { - if !self.downloading_headers.contains(&(block as BlockNumber)) { + if ignore_others || !self.downloading_headers.contains(&(block as BlockNumber)) { headers.push(block as BlockNumber); - self.downloading_headers.insert(block as BlockNumber); } block += 1; } @@ -692,17 +714,23 @@ impl ChainSync { } if !headers.is_empty() { - start = headers[0] as usize; + start = headers[0]; + if head.is_some() && start > head.unwrap() && start - head.unwrap() > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading headers", peer_id, start, head.unwrap()); + self.request_blocks(io, peer_id, true); + return; + } let count = headers.len(); + self.downloading_headers.extend(headers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, headers); - assert!(!self.headers.have_item(&(start as BlockNumber))); - self.request_headers_by_number(io, peer_id, start as BlockNumber, count, 0, false); + assert!(!self.headers.have_item(&start)); + self.request_headers_by_number(io, peer_id, start, count, 0, false); } } else { // continue search for common block - self.downloading_headers.insert(start as BlockNumber); - self.request_headers_by_number(io, peer_id, start as BlockNumber, 1, 0, false); + self.downloading_headers.insert(start); + self.request_headers_by_number(io, peer_id, start, 1, 0, false); } } } @@ -894,7 +922,7 @@ impl ChainSync { let mut packet = RlpStream::new_list(5); let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); - packet.append(&NETWORK_ID); //TODO: network id + packet.append(&self.network_id); packet.append(&chain.total_difficulty); packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); @@ -1143,8 +1171,8 @@ impl ChainSync { .collect::>() } - /// propagades latest block to lagging peers - fn propagade_blocks(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { + /// propagates latest block to lagging peers + fn propagate_blocks(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { let updated_peers = { let lagging_peers = self.get_lagging_peers(io); @@ -1170,8 +1198,8 @@ impl ChainSync { sent } - /// propagades new known hashes to all peers - fn propagade_new_hashes(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { + /// propagates new known hashes to all peers + fn propagate_new_hashes(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { let updated_peers = self.get_lagging_peers(io); let mut sent = 0; let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(local_best.clone())).unwrap()).parent_hash(); @@ -1206,8 +1234,8 @@ impl ChainSync { pub fn chain_blocks_verified(&mut self, io: &mut SyncIo) { let chain = io.chain().chain_info(); if (((chain.best_block_number as i64) - (self.last_send_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let blocks = self.propagade_blocks(&chain.best_block_hash, chain.best_block_number, io); - let hashes = self.propagade_new_hashes(&chain.best_block_hash, chain.best_block_number, io); + let blocks = self.propagate_blocks(&chain.best_block_hash, chain.best_block_number, io); + let hashes = self.propagate_new_hashes(&chain.best_block_hash, chain.best_block_number, io); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); } @@ -1220,6 +1248,7 @@ impl ChainSync { mod tests { use tests::helpers::*; use super::*; + use ::SyncConfig; use util::*; use super::{PeerInfo, PeerAsking}; use ethcore::header::*; @@ -1333,7 +1362,7 @@ mod tests { } fn dummy_sync_with_peer(peer_latest_hash: H256) -> ChainSync { - let mut sync = ChainSync::new(); + let mut sync = ChainSync::new(SyncConfig::default()); sync.peers.insert(0, PeerInfo { protocol_version: 0, @@ -1390,7 +1419,7 @@ mod tests { let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagade_new_hashes(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_new_hashes(&best_hash, best_number, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1410,7 +1439,7 @@ mod tests { let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagade_blocks(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_blocks(&best_hash, best_number, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1516,7 +1545,7 @@ mod tests { let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagade_new_hashes(&best_hash, best_number, &mut io); + sync.propagate_new_hashes(&best_hash, best_number, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_hashes(&mut io, 0, &UntrustedRlp::new(&data)); @@ -1535,7 +1564,7 @@ mod tests { let best_number = client.chain_info().best_block_number; let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagade_blocks(&best_hash, best_number, &mut io); + sync.propagate_blocks(&best_hash, best_number, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &UntrustedRlp::new(&data)); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index fd586409a..6f28fc320 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -34,15 +34,15 @@ //! use std::env; //! use std::sync::Arc; //! use util::network::{NetworkService, NetworkConfiguration}; -//! use ethcore::client::Client; -//! use ethsync::EthSync; +//! use ethcore::client::{Client, ClientConfig}; +//! use ethsync::{EthSync, SyncConfig}; //! use ethcore::ethereum; //! //! fn main() { //! let mut service = NetworkService::start(NetworkConfiguration::new()).unwrap(); //! let dir = env::temp_dir(); -//! let client = Client::new(ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); -//! EthSync::register(&mut service, client); +//! let client = Client::new(ClientConfig::default(), ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); +//! EthSync::register(&mut service, SyncConfig::default(), client); //! } //! ``` @@ -54,12 +54,15 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; +#[macro_use] +extern crate heapsize; use std::ops::*; use std::sync::*; use ethcore::client::Client; use util::network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId}; use util::TimerToken; +use util::{U256, ONE_U256}; use chain::ChainSync; use ethcore::service::SyncMessage; use io::NetSyncIo; @@ -71,6 +74,23 @@ mod range_collection; #[cfg(test)] mod tests; +/// Sync configuration +pub struct SyncConfig { + /// Max blocks to download ahead + pub max_download_ahead_blocks: usize, + /// Network ID + pub network_id: U256, +} + +impl Default for SyncConfig { + fn default() -> SyncConfig { + SyncConfig { + max_download_ahead_blocks: 20000, + network_id: ONE_U256, + } + } +} + /// Ethereum network protocol handler pub struct EthSync { /// Shared blockchain client. TODO: this should evetually become an IPC endpoint @@ -83,10 +103,10 @@ pub use self::chain::{SyncStatus, SyncState}; impl EthSync { /// Creates and register protocol with the network service - pub fn register(service: &mut NetworkService, chain: Arc) -> Arc { + pub fn register(service: &mut NetworkService, config: SyncConfig, chain: Arc) -> Arc { let sync = Arc::new(EthSync { chain: chain, - sync: RwLock::new(ChainSync::new()), + sync: RwLock::new(ChainSync::new(config)), }); service.register_protocol(sync.clone(), "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); sync diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index 1dd9a1e78..b01c894a0 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -121,7 +121,7 @@ fn status_packet() { } #[test] -fn propagade_hashes() { +fn propagate_hashes() { let mut net = TestNet::new(6); net.peer_mut(1).chain.add_blocks(10, false); net.sync(); @@ -147,7 +147,7 @@ fn propagade_hashes() { } #[test] -fn propagade_blocks() { +fn propagate_blocks() { let mut net = TestNet::new(2); net.peer_mut(1).chain.add_blocks(10, false); net.sync(); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 1d442f908..e9c5f0edc 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -15,12 +15,12 @@ // along with Parity. If not, see . use util::*; -use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId}; -use ethcore::block_queue::BlockQueueInfo; +use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId, BlockQueueInfo}; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; -use chain::{ChainSync}; +use chain::ChainSync; +use ::SyncConfig; use ethcore::receipt::Receipt; use ethcore::transaction::LocalizedTransaction; use ethcore::filter::Filter; @@ -121,7 +121,7 @@ impl BlockChainClient for TestBlockChainClient { unimplemented!(); } - fn logs(&self, filter: Filter) -> Vec { + fn logs(&self, _filter: Filter) -> Vec { unimplemented!(); } @@ -255,6 +255,9 @@ impl BlockChainClient for TestBlockChainClient { verified_queue_size: 0, unverified_queue_size: 0, verifying_queue_size: 0, + max_queue_size: 0, + max_mem_use: 0, + mem_used: 0, } } @@ -344,7 +347,7 @@ impl TestNet { for _ in 0..n { net.peers.push(TestPeer { chain: TestBlockChainClient::new(), - sync: ChainSync::new(), + sync: ChainSync::new(SyncConfig::default()), queue: VecDeque::new(), }); } diff --git a/util/Cargo.toml b/util/Cargo.toml index 3c742715e..0c7df3f40 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -18,7 +18,7 @@ time = "0.1.34" tiny-keccak = "1.0" rocksdb = { git = "https://github.com/arkpar/rust-rocksdb.git" } lazy_static = "0.1" -eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } +eth-secp256k1 = { git = "https://github.com/ethcore/rust-secp256k1" } rust-crypto = "0.2.34" elastic-array = "0.4" heapsize = "0.3" @@ -26,7 +26,7 @@ itertools = "0.4" crossbeam = "0.2" slab = "0.1" sha3 = { path = "sha3" } -serde = "0.6.7" +serde = "0.7.0" clippy = { version = "0.0.44", optional = true } json-tests = { path = "json-tests" } rustc_version = "0.1.0" @@ -35,11 +35,12 @@ ethcore-devtools = { path = "../devtools" } libc = "0.2.7" vergen = "0.1" target_info = "0.1" +bigint = { path = "bigint" } [features] default = [] dev = ["clippy"] -x64asm = [] [build-dependencies] vergen = "*" +rustc_version = "0.1" diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index da82084b8..fc41ab628 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -74,6 +74,19 @@ fn u256_mul(b: &mut Bencher) { } +#[bench] +fn u256_full_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), + |old, new| { + let U512(ref u512words) = old.full_mul(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()])); + U256([u512words[0], u512words[2], u512words[2], u512words[3]]) + }) + }); +} + + #[bench] fn u128_mul(b: &mut Bencher) { b.iter(|| { diff --git a/util/bigint/Cargo.toml b/util/bigint/Cargo.toml new file mode 100644 index 000000000..377391eeb --- /dev/null +++ b/util/bigint/Cargo.toml @@ -0,0 +1,23 @@ +[package] +description = "Rust-assembler implementation of big integers arithmetic" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "bigint" +version = "0.1.0" +authors = ["Ethcore "] +build = "build.rs" + +[build-dependencies] +rustc_version = "0.1" + +[dependencies] +rustc-serialize = "0.3" +arrayvec = "0.3" +rand = "0.3.12" +serde = "0.7.0" +clippy = { version = "0.0.44", optional = true } +heapsize = "0.3" + +[features] +x64asm_arithmetic=[] +rust_arithmetic=[] diff --git a/util/bigint/build.rs b/util/bigint/build.rs new file mode 100644 index 000000000..248823229 --- /dev/null +++ b/util/bigint/build.rs @@ -0,0 +1,25 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +extern crate rustc_version; + +use rustc_version::{version_meta, Channel}; + +fn main() { + if let Channel::Nightly = version_meta().channel { + println!("cargo:rustc-cfg=asm_available"); + } +} diff --git a/util/bigint/src/lib.rs b/util/bigint/src/lib.rs new file mode 100644 index 000000000..149878538 --- /dev/null +++ b/util/bigint/src/lib.rs @@ -0,0 +1,23 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![cfg_attr(asm_available, feature(asm))] + +extern crate rustc_serialize; +extern crate serde; +#[macro_use] extern crate heapsize; + +pub mod uint; diff --git a/util/src/uint.rs b/util/bigint/src/uint.rs similarity index 70% rename from util/src/uint.rs rename to util/bigint/src/uint.rs index 5a2730126..4cc0dfc0e 100644 --- a/util/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -36,10 +36,26 @@ //! The functions here are designed to be fast. //! -use standard::*; -use from_json::*; -use rustc_serialize::hex::ToHex; +use std::fmt; +use std::cmp; + +use std::mem; +use std::ops; +use std::slice; +use std::result; +use std::option; +use std::str::{FromStr}; +use std::convert::From; +use std::hash::{Hash, Hasher}; +use std::ops::*; +use std::cmp::*; +use std::collections::*; + use serde; +use rustc_serialize::json::Json; +use rustc_serialize::base64::FromBase64; +use rustc_serialize::hex::{FromHex, FromHexError, ToHex}; + macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { @@ -51,7 +67,7 @@ macro_rules! impl_map_from { } } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(asm_available, target_arch="x86_64")))] macro_rules! uint_overflowing_add { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) @@ -88,8 +104,7 @@ macro_rules! uint_overflowing_add_reg { }) } - -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(asm_available, target_arch="x86_64"))] macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -165,7 +180,7 @@ macro_rules! uint_overflowing_add { ) } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(asm_available, target_arch="x86_64")))] macro_rules! uint_overflowing_sub { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let res = overflowing!((!$other).overflowing_add(From::from(1u64))); @@ -174,7 +189,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(asm_available, target_arch="x86_64"))] macro_rules! uint_overflowing_sub { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -250,7 +265,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(asm_available, target_arch="x86_64"))] macro_rules! uint_overflowing_mul { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -265,53 +280,60 @@ macro_rules! uint_overflowing_mul { mov %rax, $0 mov %rdx, $1 - mov $6, %rax - mulq $9 + mov $5, %rax + mulq $10 add %rax, $1 + adc $$0, %rdx mov %rdx, $2 mov $5, %rax - mulq $10 - add %rax, $1 - adc %rdx, $2 - - mov $6, %rax - mulq $10 + mulq $11 add %rax, $2 + adc $$0, %rdx mov %rdx, $3 - mov $7, %rax - mulq $9 - add %rax, $2 - adc %rdx, $3 - - mov $5, %rax - mulq $11 - add %rax, $2 - adc %rdx, $3 - - mov $8, %rax - mulq $9 - adc %rax, $3 - adc $$0, %rdx - mov %rdx, %rcx - - mov $7, %rax - mulq $10 - add %rax, $3 - adc $$0, %rdx - or %rdx, %rcx - - mov $6, %rax - mulq $11 - add %rax, $3 - adc $$0, %rdx - or %rdx, %rcx - mov $5, %rax mulq $12 add %rax, $3 adc $$0, %rdx + mov %rdx, %rcx + + mov $6, %rax + mulq $9 + add %rax, $1 + adc %rdx, $2 + adc $$0, $3 + adc $$0, %rcx + + mov $6, %rax + mulq $10 + add %rax, $2 + adc %rdx, $3 + adc $$0, %rcx + adc $$0, $3 + adc $$0, %rcx + + mov $6, %rax + mulq $11 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $7, %rax + mulq $9 + add %rax, $2 + adc %rdx, $3 + adc $$0, %rcx + + mov $7, %rax + mulq $10 + add %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + + mov $8, %rax + mulq $9 + add %rax, $3 or %rdx, %rcx cmpq $$0, %rcx @@ -351,7 +373,7 @@ macro_rules! uint_overflowing_mul { : /* $5 */ "m"(self_t[0]), /* $6 */ "m"(self_t[1]), /* $7 */ "m"(self_t[2]), /* $8 */ "m"(self_t[3]), /* $9 */ "m"(other_t[0]), /* $10 */ "m"(other_t[1]), /* $11 */ "m"(other_t[2]), /* $12 */ "m"(other_t[3]) - : "rax", "rdx", "rbx" + : "rax", "rdx" : ); @@ -363,7 +385,7 @@ macro_rules! uint_overflowing_mul { ) } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(asm_available, target_arch="x86_64")))] macro_rules! uint_overflowing_mul { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) @@ -374,7 +396,6 @@ macro_rules! uint_overflowing_mul_reg { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut res = $name::from(0u64); let mut overflow = false; - // TODO: be more efficient about this for i in 0..(2 * $n_words) { let v = overflowing!($self_expr.overflowing_mul_u32(($other >> (32 * i)).low_u32()), overflow); let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); @@ -409,7 +430,7 @@ macro_rules! panic_on_overflow { } /// Large, fixed-length unsigned integer type. -pub trait Uint: Sized + Default + FromStr + From + FromJson + fmt::Debug + fmt::Display + PartialOrd + Ord + PartialEq + Eq + Hash { +pub trait Uint: Sized + Default + FromStr + From + fmt::Debug + fmt::Display + PartialOrd + Ord + PartialEq + Eq + Hash { /// Returns new instance equalling zero. fn zero() -> Self; @@ -760,7 +781,7 @@ macro_rules! construct_uint { self.to_bytes(&mut bytes); let len = cmp::max((self.bits() + 7) / 8, 1); hex.push_str(bytes[bytes.len() - len..].to_hex().as_ref()); - serializer.visit_str(hex.as_ref()) + serializer.serialize_str(hex.as_ref()) } } @@ -772,22 +793,6 @@ macro_rules! construct_uint { } } - impl FromJson for $name { - fn from_json(json: &Json) -> Self { - match *json { - Json::String(ref s) => { - if s.len() >= 2 && &s[0..2] == "0x" { - FromStr::from_str(&s[2..]).unwrap_or_else(|_| Default::default()) - } else { - Uint::from_dec_str(s).unwrap_or_else(|_| Default::default()) - } - }, - Json::U64(u) => From::from(u), - Json::I64(i) => From::from(i as u64), - _ => Uint::zero(), - } - } - } impl_map_from!($name, u8, u64); impl_map_from!($name, u16, u64); @@ -1090,6 +1095,157 @@ construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); +impl U256 { + /// Multiplies two 256-bit integers to produce full 512-bit integer + /// No overflow possible + #[cfg(all(asm_available, target_arch="x86_64"))] + pub fn full_mul(self, other: U256) -> U512 { + let self_t: &[u64; 4] = unsafe { &mem::transmute(self) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute(other) }; + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + unsafe { + asm!(" + mov $8, %rax + mulq $12 + mov %rax, $0 + mov %rdx, $1 + + mov $8, %rax + mulq $13 + add %rax, $1 + adc $$0, %rdx + mov %rdx, $2 + + mov $8, %rax + mulq $14 + add %rax, $2 + adc $$0, %rdx + mov %rdx, $3 + + mov $8, %rax + mulq $15 + add %rax, $3 + adc $$0, %rdx + mov %rdx, $4 + + mov $9, %rax + mulq $12 + add %rax, $1 + adc %rdx, $2 + adc $$0, $3 + adc $$0, $4 + xor $5, $5 + adc $$0, $5 + xor $6, $6 + adc $$0, $6 + xor $7, $7 + adc $$0, $7 + + mov $9, %rax + mulq $13 + add %rax, $2 + adc %rdx, $3 + adc $$0, $4 + adc $$0, $5 + adc $$0, $6 + adc $$0, $7 + + mov $9, %rax + mulq $14 + add %rax, $3 + adc %rdx, $4 + adc $$0, $5 + adc $$0, $6 + adc $$0, $7 + + mov $9, %rax + mulq $15 + add %rax, $4 + adc %rdx, $5 + adc $$0, $6 + adc $$0, $7 + + mov $10, %rax + mulq $12 + add %rax, $2 + adc %rdx, $3 + adc $$0, $4 + adc $$0, $5 + adc $$0, $6 + adc $$0, $7 + + mov $10, %rax + mulq $13 + add %rax, $3 + adc %rdx, $4 + adc $$0, $5 + adc $$0, $6 + adc $$0, $7 + + mov $10, %rax + mulq $14 + add %rax, $4 + adc %rdx, $5 + adc $$0, $6 + adc $$0, $7 + + mov $10, %rax + mulq $15 + add %rax, $5 + adc %rdx, $6 + adc $$0, $7 + + mov $11, %rax + mulq $12 + add %rax, $3 + adc %rdx, $4 + adc $$0, $5 + adc $$0, $6 + adc $$0, $7 + + mov $11, %rax + mulq $13 + add %rax, $4 + adc %rdx, $5 + adc $$0, $6 + adc $$0, $7 + + mov $11, %rax + mulq $14 + add %rax, $5 + adc %rdx, $6 + adc $$0, $7 + + mov $11, %rax + mulq $15 + add %rax, $6 + adc %rdx, $7 + " + : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), + /* $3 */ "={r11}"(result[3]), /* $4 */ "={r12}"(result[4]), /* $5 */ "={r13}"(result[5]), + /* $6 */ "={r14}"(result[6]), /* $7 */ "={r15}"(result[7]) + + : /* $8 */ "m"(self_t[0]), /* $9 */ "m"(self_t[1]), /* $10 */ "m"(self_t[2]), + /* $11 */ "m"(self_t[3]), /* $12 */ "m"(other_t[0]), /* $13 */ "m"(other_t[1]), + /* $14 */ "m"(other_t[2]), /* $15 */ "m"(other_t[3]) + : "rax", "rdx" + : + ); + } + U512(result) + } + + /// Multiplies two 256-bit integers to produce full 512-bit integer + /// No overflow possible + #[cfg(not(all(asm_available, target_arch="x86_64")))] + pub fn full_mul(self, other: U256) -> U512 { + let self_512 = U512::from(self); + let other_512 = U512::from(other); + let (result, _) = self_512.overflowing_mul(other_512); + result + } +} + impl From for U512 { fn from(value: U256) -> U512 { let U256(ref arr) = value; @@ -1180,6 +1336,9 @@ pub const ZERO_U256: U256 = U256([0x00u64; 4]); /// Constant value of `U256::one()` that can be used for a reference saving an additional instance creation. pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); + +known_heap_size!(0, U128, U256); + #[cfg(test)] mod tests { use uint::{Uint, U128, U256, U512}; @@ -1629,31 +1788,30 @@ mod tests { } - #[test] - fn u256_multi_subs() { - let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); - assert_eq!(result, U256([0, 0, 0, 0])); + #[test] + fn u256_multi_subs() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); - let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); - assert_eq!(result, U256([0, 0, 0, 0])); + let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 0])); - let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); - assert!(overflow); + let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); + assert!(overflow); - let (result, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) - .overflowing_sub(U256([::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2])); - assert!(!overflow); - assert_eq!(U256([::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1]), result); + let (result, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_sub(U256([::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1]), result); - let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); - assert!(!overflow); - assert_eq!(U256([0, 0, ::std::u64::MAX, 0]), result); - - let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); - assert!(!overflow); - assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); - } + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); + assert!(!overflow); + assert_eq!(U256([0, 0, ::std::u64::MAX, 0]), result); + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + } #[test] fn u512_multi_subs() { @@ -1671,76 +1829,262 @@ mod tests { } #[test] - fn u256_multi_muls() { - let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); + fn u256_multi_carry_all() { + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX-1, 0, 0]), result); - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([1, 0, 0, 0]), result); + let (result, _) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([0, 1, ::std::u64::MAX-1, 0]), result); - let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); - assert_eq!(U256([25, 0, 0, 0]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX-1, 0]), result); - let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 25, 0]), result); + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX-1, 0]), result); - let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 1]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX-1, ::std::u64::MAX]), result); - let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); - assert_eq!(U256([0, 0, 0, 10]), result); + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1]), result); - let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 0, 0, 5]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1]), result); - let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); - assert_eq!(U256([0, 0, 0, 0]), result); + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul( + U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]), result); - let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); - assert_eq!(U256([0, 10, 0, 0]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]), result); - let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); - assert_eq!(U256([1, ::std::u64::MAX-1, 0, 0]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX-1]), result); - let (result, _) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); - assert_eq!(U256([0, 0, 0, 0]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX-1]), result); - let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); - assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX]), result); - let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]) .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); - assert_eq!(U256([1, 0, 0, 0]), result); + assert_eq!(U256([1, 0, ::std::u64::MAX, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX-1]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U256([1, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, 0]), result); + } + + #[test] + fn u256_multi_muls() { + use hash::*; + + let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([1, 0, 0, 0]), result); + + let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); + assert_eq!(U256([25, 0, 0, 0]), result); + + let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 25, 0]), result); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 1]), result); + + let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 10]), result); + + let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 0, 5]), result); + + let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 10, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let x1 = U256::from_str("0000000000000000000000000000000000000000000000000000012365124623").unwrap(); + let x2sqr_right = U256::from_str("000000000000000000000000000000000000000000014baeef72e0378e2328c9").unwrap(); + let x1sqr = x1 * x1; + assert_eq!(H256::from(x2sqr_right), H256::from(x1sqr)); + let x1cube = x1sqr * x1; + let x1cube_right = U256::from_str("0000000000000000000000000000000001798acde139361466f712813717897b").unwrap(); + assert_eq!(H256::from(x1cube_right), H256::from(x1cube)); + let x1quad = x1cube * x1; + let x1quad_right = U256::from_str("000000000000000000000001adbdd6bd6ff027485484b97f8a6a4c7129756dd1").unwrap(); + assert_eq!(H256::from(x1quad_right), H256::from(x1quad)); + let x1penta = x1quad * x1; + let x1penta_right = U256::from_str("00000000000001e92875ac24be246e1c57e0507e8c46cc8d233b77f6f4c72993").unwrap(); + assert_eq!(H256::from(x1penta_right), H256::from(x1penta)); + let x1septima = x1penta * x1; + let x1septima_right = U256::from_str("00022cca1da3f6e5722b7d3cc5bbfb486465ebc5a708dd293042f932d7eee119").unwrap(); + assert_eq!(H256::from(x1septima_right), H256::from(x1septima)); } #[test] fn u256_multi_muls_overflow() { - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); - assert!(!overflow); + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert!(!overflow); - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); - assert!(!overflow); + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(!overflow); - let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); - assert!(overflow); + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); - let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); - assert!(!overflow); + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); + assert!(!overflow); - let (_, overflow) = U256([0, 1, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 1, 0, ::std::u64::MAX])); - assert!(overflow); + let (_, overflow) = U256([0, 1, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 1, 0, ::std::u64::MAX])); + assert!(overflow); - let (_, overflow) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([0, ::std::u64::MAX, 0, 0])); - assert!(!overflow); + let (_, overflow) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([0, ::std::u64::MAX, 0, 0])); + assert!(!overflow); - let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); - assert!(!overflow); + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); + assert!(!overflow); - let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX / 2])); - assert!(!overflow); + let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX / 2])); + assert!(!overflow); - let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); - assert!(overflow); + let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert!(overflow); } + + + #[test] + fn u256_multi_full_mul() { + let result = U256([0, 0, 0, 0]).full_mul(U256([0, 0, 0, 0])); + assert_eq!(U512([0, 0, 0, 0, 0, 0, 0, 0]), result); + + let result = U256([1, 0, 0, 0]).full_mul(U256([1, 0, 0, 0])); + assert_eq!(U512([1, 0, 0, 0, 0, 0, 0, 0]), result); + + let result = U256([5, 0, 0, 0]).full_mul(U256([5, 0, 0, 0])); + assert_eq!(U512([25, 0, 0, 0, 0, 0, 0, 0]), result); + + let result = U256([0, 5, 0, 0]).full_mul(U256([0, 5, 0, 0])); + assert_eq!(U512([0, 0, 25, 0, 0, 0, 0, 0]), result); + + let result = U256([0, 0, 0, 4]).full_mul(U256([4, 0, 0, 0])); + assert_eq!(U512([0, 0, 0, 16, 0, 0, 0, 0]), result); + + let result = U256([0, 0, 0, 5]).full_mul(U256([2, 0, 0, 0])); + assert_eq!(U512([0, 0, 0, 10, 0, 0, 0, 0]), result); + + let result = U256([0, 0, 2, 0]).full_mul(U256([0, 5, 0, 0])); + assert_eq!(U512([0, 0, 0, 10, 0, 0, 0, 0]), result); + + let result = U256([0, 3, 0, 0]).full_mul(U256([0, 0, 3, 0])); + assert_eq!(U512([0, 0, 0, 9, 0, 0, 0, 0]), result); + + let result = U256([0, 0, 8, 0]).full_mul(U256([0, 0, 6, 0])); + assert_eq!(U512([0, 0, 0, 0, 48, 0, 0, 0]), result); + + let result = U256([9, 0, 0, 0]).full_mul(U256([0, 3, 0, 0])); + assert_eq!(U512([0, 27, 0, 0, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, 0, 0, 0]).full_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U512([1, ::std::u64::MAX-1, 0, 0, 0, 0, 0, 0]), result); + + let result = U256([0, ::std::u64::MAX, 0, 0]).full_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U512([0, 1, ::std::u64::MAX-1, 0, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).full_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, 0, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U512([1, 0, ::std::u64::MAX-1, ::std::u64::MAX, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, 0, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).full_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, 0, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]).full_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U512([1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U512([1, 0, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U512([1, 0, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, 0, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, 0, 0])); + assert_eq!(U512([1, 0, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, 0, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U512([1, 0, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U512([1, 0, 0, ::std::u64::MAX-1, ::std::u64::MAX, ::std::u64::MAX, 0, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U512([1, 0, 0, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0])); + assert_eq!(U512([1, 0, 0, ::std::u64::MAX, ::std::u64::MAX-1, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + + let result = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]).full_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U512([1, 0, 0, 0, ::std::u64::MAX-1, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]), result); + + let result = U256([0, 0, 0, ::std::u64::MAX]).full_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U512([0, 0, 0, 0, 0, 0, 1, ::std::u64::MAX-1]), result); + + let result = U256([1, 0, 0, 0]).full_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U512([0, 0, 0, ::std::u64::MAX, 0, 0, 0, 0]), result); + + let result = U256([1, 2, 3, 4]).full_mul(U256([5, 0, 0, 0])); + assert_eq!(U512([5, 10, 15, 20, 0, 0, 0, 0]), result); + + let result = U256([1, 2, 3, 4]).full_mul(U256([0, 6, 0, 0])); + assert_eq!(U512([0, 6, 12, 18, 24, 0, 0, 0]), result); + + let result = U256([1, 2, 3, 4]).full_mul(U256([0, 0, 7, 0])); + assert_eq!(U512([0, 0, 7, 14, 21, 28, 0, 0]), result); + + let result = U256([1, 2, 3, 4]).full_mul(U256([0, 0, 0, 8])); + assert_eq!(U512([0, 0, 0, 8, 16, 24, 32, 0]), result); + + let result = U256([1, 2, 3, 4]).full_mul(U256([5, 6, 7, 8])); + assert_eq!(U512([5, 16, 34, 60, 61, 52, 32, 0]), result); + } } diff --git a/util/build.rs b/util/build.rs index 32ee30472..eed080e29 100644 --- a/util/build.rs +++ b/util/build.rs @@ -1,6 +1,7 @@ extern crate vergen; + use vergen::*; fn main() { vergen(OutputFns::all()).unwrap(); -} \ No newline at end of file +} diff --git a/util/src/common.rs b/util/src/common.rs index 0816b72e4..b2a06c4b9 100644 --- a/util/src/common.rs +++ b/util/src/common.rs @@ -19,10 +19,9 @@ pub use standard::*; pub use from_json::*; pub use error::*; -pub use hash::*; -pub use uint::*; pub use bytes::*; pub use vector::*; +pub use numbers::*; pub use sha3::*; #[macro_export] diff --git a/util/src/crypto.rs b/util/src/crypto.rs index 596265cd0..66e9f2edb 100644 --- a/util/src/crypto.rs +++ b/util/src/crypto.rs @@ -16,9 +16,8 @@ //! Ethcore crypto. -use hash::*; +use numbers::*; use bytes::*; -use uint::*; use secp256k1::{key, Secp256k1}; use rand::os::OsRng; @@ -151,8 +150,7 @@ impl KeyPair { /// EC functions pub mod ec { - use hash::*; - use uint::*; + use numbers::*; use standard::*; use crypto::*; use crypto::{self}; @@ -254,7 +252,7 @@ pub mod ecies { use crypto::*; /// Encrypt a message with a public key - pub fn encrypt(public: &Public, plain: &[u8]) -> Result { + pub fn encrypt(public: &Public, shared_mac: &[u8], plain: &[u8]) -> Result { use ::rcrypto::digest::Digest; use ::rcrypto::sha2::Sha256; use ::rcrypto::hmac::Hmac; @@ -284,13 +282,14 @@ pub mod ecies { let cipher_iv = &msgd[64..(64 + 16 + plain.len())]; hmac.input(cipher_iv); } + hmac.input(shared_mac); hmac.raw_result(&mut msgd[(64 + 16 + plain.len())..]); } Ok(msg) } /// Decrypt a message with a secret key - pub fn decrypt(secret: &Secret, encrypted: &[u8]) -> Result { + pub fn decrypt(secret: &Secret, shared_mac: &[u8], encrypted: &[u8]) -> Result { use ::rcrypto::digest::Digest; use ::rcrypto::sha2::Sha256; use ::rcrypto::hmac::Hmac; @@ -322,6 +321,7 @@ pub mod ecies { // Verify tag let mut hmac = Hmac::new(Sha256::new(), &mkey); hmac.input(cipher_with_iv); + hmac.input(shared_mac); let mut mac = H256::new(); hmac.raw_result(&mut mac); if &mac[..] != msg_mac { @@ -405,4 +405,20 @@ mod tests { let pair = KeyPair::from_secret(h256_from_hex("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2")).unwrap(); assert_eq!(pair.public().hex(), "101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c"); } + + #[test] + fn ecies_shared() { + let kp = KeyPair::create().unwrap(); + let message = b"So many books, so little time"; + + let shared = b"shared"; + let wrong_shared = b"incorrect"; + let encrypted = ecies::encrypt(kp.public(), shared, message).unwrap(); + assert!(encrypted[..] != message[..]); + assert_eq!(encrypted[0], 0x04); + + assert!(ecies::decrypt(kp.secret(), wrong_shared, &encrypted).is_err()); + let decrypted = ecies::decrypt(kp.secret(), shared, &encrypted).unwrap(); + assert_eq!(decrypted[..message.len()], message[..]); + } } diff --git a/util/src/from_json.rs b/util/src/from_json.rs index 7d977afc9..a598ed961 100644 --- a/util/src/from_json.rs +++ b/util/src/from_json.rs @@ -17,6 +17,7 @@ //! Coversion from json. use standard::*; +use bigint::uint::*; #[macro_export] macro_rules! xjson { @@ -30,3 +31,20 @@ pub trait FromJson { /// Convert a JSON value to an instance of this type. fn from_json(json: &Json) -> Self; } + +impl FromJson for U256 { + fn from_json(json: &Json) -> Self { + match *json { + Json::String(ref s) => { + if s.len() >= 2 && &s[0..2] == "0x" { + FromStr::from_str(&s[2..]).unwrap_or_else(|_| Default::default()) + } else { + Uint::from_dec_str(s).unwrap_or_else(|_| Default::default()) + } + }, + Json::U64(u) => From::from(u), + Json::I64(i) => From::from(i as u64), + _ => Uint::zero(), + } + } +} diff --git a/util/src/hash.rs b/util/src/hash.rs index a6e8f7950..73fa33b47 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -23,7 +23,7 @@ use rand::Rng; use rand::os::OsRng; use bytes::{BytesConvertable,Populatable}; use from_json::*; -use uint::{Uint, U256}; +use bigint::uint::{Uint, U256}; use rustc_serialize::hex::ToHex; use serde; @@ -239,7 +239,7 @@ macro_rules! impl_hash { where S: serde::Serializer { let mut hex = "0x".to_owned(); hex.push_str(self.to_hex().as_ref()); - serializer.visit_str(hex.as_ref()) + serializer.serialize_str(hex.as_ref()) } } @@ -254,10 +254,10 @@ macro_rules! impl_hash { fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { // 0x + len if value.len() != 2 + $size * 2 { - return Err(serde::Error::syntax("Invalid length.")); + return Err(serde::Error::custom("Invalid length.")); } - value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::syntax("Invalid valid hex.")) + value[2..].from_hex().map(|ref v| $from::from_slice(v)).map_err(|_| serde::Error::custom("Invalid valid hex.")) } fn visit_string(&mut self, value: String) -> Result where E: serde::Error { @@ -265,7 +265,7 @@ macro_rules! impl_hash { } } - deserializer.visit(HashVisitor) + deserializer.deserialize(HashVisitor) } } @@ -304,6 +304,8 @@ macro_rules! impl_hash { } } + impl Copy for $from {} + #[cfg_attr(feature="dev", allow(expl_impl_clone_on_copy))] impl Clone for $from { fn clone(&self) -> $from { unsafe { @@ -595,7 +597,7 @@ pub fn h256_from_hex(s: &str) -> H256 { /// Convert `n` to an `H256`, setting the rightmost 8 bytes. pub fn h256_from_u64(n: u64) -> H256 { - use uint::U256; + use bigint::uint::U256; H256::from(&U256::from(n)) } @@ -631,7 +633,7 @@ pub static ZERO_H256: H256 = H256([0x00; 32]); #[cfg(test)] mod tests { use hash::*; - use uint::*; + use bigint::uint::*; use std::str::FromStr; #[test] diff --git a/util/src/heapsizeof.rs b/util/src/heapsizeof.rs index d7c8124cd..feb679a0b 100644 --- a/util/src/heapsizeof.rs +++ b/util/src/heapsizeof.rs @@ -16,8 +16,7 @@ //! Calculates heapsize of util types. -use uint::*; use hash::*; known_heap_size!(0, H32, H64, H128, Address, H256, H264, H512, H520, H1024, H2048); -known_heap_size!(0, U128, U256); + diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs index 2ee3c987c..cdbd9b213 100644 --- a/util/src/keys/geth_import.rs +++ b/util/src/keys/geth_import.rs @@ -46,14 +46,14 @@ pub fn enumerate_geth_keys(path: &Path) -> Result, io::Er #[derive(Debug)] pub enum ImportError { /// Io error reading geth file - IoError(io::Error), + Io(io::Error), /// format error - FormatError, + Format, } impl From for ImportError { fn from (err: io::Error) -> ImportError { - ImportError::IoError(err) + ImportError::Io(err) } } @@ -65,15 +65,15 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) let mut json_result = Json::from_str(&buf); let mut json = match json_result { - Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::FormatError)), - Err(_) => { return Err(ImportError::FormatError); } + Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::Format)), + Err(_) => { return Err(ImportError::Format); } }; - let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone(); + let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::Format)).clone(); json.insert("crypto".to_owned(), Json::Object(crypto_object)); json.remove("Crypto"); match KeyFileContent::load(&Json::Object(json.clone())) { Ok(key_file) => try!(secret_store.import_key(key_file)), - Err(_) => { return Err(ImportError::FormatError); } + Err(_) => { return Err(ImportError::Format); } }; Ok(()) } @@ -82,7 +82,7 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { use std::path::PathBuf; let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); - for &(ref address, ref file_path) in geth_files.iter() { + for &(ref address, ref file_path) in &geth_files { let mut path = PathBuf::new(); path.push(geth_keyfiles_directory); path.push(file_path); diff --git a/util/src/lib.rs b/util/src/lib.rs index be1e788c9..8594e6f40 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,7 +16,6 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] -#![cfg_attr(feature="x64asm", feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings @@ -111,15 +110,16 @@ extern crate libc; extern crate rustc_version; extern crate target_info; extern crate vergen; +extern crate bigint; pub mod standard; #[macro_use] pub mod from_json; #[macro_use] pub mod common; +pub mod numbers; pub mod error; pub mod hash; -pub mod uint; pub mod bytes; pub mod rlp; pub mod misc; @@ -144,6 +144,7 @@ pub mod network; pub mod log; pub mod panics; pub mod keys; +pub mod table; pub use common::*; pub use misc::*; diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index a7396ff33..55e688c91 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -279,7 +279,7 @@ impl EncryptedConnection { /// Create an encrypted connection out of the handshake. Consumes a handshake object. pub fn new(handshake: &mut Handshake) -> Result { - let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); + let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_ephemeral)); let mut nonce_material = H512::new(); if handshake.originated { handshake.remote_nonce.copy_to(&mut nonce_material[0..32]); diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index e2fd1c269..e52d5d25f 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -85,7 +85,8 @@ pub struct Discovery { discovery_id: NodeId, discovery_nodes: HashSet, node_buckets: Vec, - send_queue: VecDeque + send_queue: VecDeque, + check_timestamps: bool, } pub struct TableUpdates { @@ -107,6 +108,7 @@ impl Discovery { node_buckets: (0..NODE_BINS).map(|_| NodeBucket::new()).collect(), udp_socket: socket, send_queue: VecDeque::new(), + check_timestamps: true, } } @@ -344,20 +346,20 @@ impl Discovery { } } + fn check_timestamp(&self, timestamp: u64) -> Result<(), NetworkError> { + if self.check_timestamps && timestamp < time::get_time().sec as u64{ + debug!(target: "discovery", "Expired packet"); + return Err(NetworkError::Expired); + } + Ok(()) + } + fn on_ping(&mut self, rlp: &UntrustedRlp, node: &NodeId, from: &SocketAddr) -> Result, NetworkError> { trace!(target: "discovery", "Got Ping from {:?}", &from); - let version: u32 = try!(rlp.val_at(0)); - if version != PROTOCOL_VERSION { - debug!(target: "discovery", "Unexpected protocol version: {}", version); - return Err(NetworkError::BadProtocol); - } let source = try!(NodeEndpoint::from_rlp(&try!(rlp.at(1)))); let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(2)))); let timestamp: u64 = try!(rlp.val_at(3)); - if timestamp < time::get_time().sec as u64{ - debug!(target: "discovery", "Expired ping"); - return Err(NetworkError::Expired); - } + try!(self.check_timestamp(timestamp)); let mut added_map = HashMap::new(); let entry = NodeEntry { id: node.clone(), endpoint: source.clone() }; if !entry.endpoint.is_valid() || !entry.endpoint.is_global() { @@ -381,9 +383,7 @@ impl Discovery { // TODO: validate pong packet let dest = try!(NodeEndpoint::from_rlp(&try!(rlp.at(0)))); let timestamp: u64 = try!(rlp.val_at(2)); - if timestamp < time::get_time().sec as u64 { - return Err(NetworkError::Expired); - } + try!(self.check_timestamp(timestamp)); let mut entry = NodeEntry { id: node.clone(), endpoint: dest }; if !entry.endpoint.is_valid() { debug!(target: "discovery", "Bad address: {:?}", entry); @@ -399,10 +399,7 @@ impl Discovery { trace!(target: "discovery", "Got FindNode from {:?}", &from); let target: NodeId = try!(rlp.val_at(0)); let timestamp: u64 = try!(rlp.val_at(1)); - if timestamp < time::get_time().sec as u64 { - return Err(NetworkError::Expired); - } - + try!(self.check_timestamp(timestamp)); let limit = (MAX_DATAGRAM_SIZE - 109) / 90; let nearest = Discovery::nearest_node_entries(&target, &self.node_buckets); if nearest.is_empty() { @@ -501,6 +498,7 @@ mod tests { use network::node_table::*; use crypto::KeyPair; use std::str::FromStr; + use rustc_serialize::hex::FromHex; #[test] fn discovery() { @@ -540,7 +538,7 @@ mod tests { #[test] fn removes_expired() { let key = KeyPair::create().unwrap(); - let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40444 }; + let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40446").unwrap(), udp_port: 40447 }; let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0); for _ in 0..1200 { discovery.add_node(NodeEntry { id: NodeId::random(), endpoint: ep.clone() }); @@ -549,4 +547,70 @@ mod tests { let removed = discovery.check_expired(true).len(); assert!(removed > 0); } + + #[test] + fn packets() { + let key = KeyPair::create().unwrap(); + let ep = NodeEndpoint { address: SocketAddr::from_str("127.0.0.1:40447").unwrap(), udp_port: 40447 }; + let mut discovery = Discovery::new(&key, ep.address.clone(), ep.clone(), 0); + discovery.check_timestamps = false; + let from = SocketAddr::from_str("99.99.99.99:40445").unwrap(); + + let packet = "\ + e9614ccfd9fc3e74360018522d30e1419a143407ffcce748de3e22116b7e8dc92ff74788c0b6663a\ + aa3d67d641936511c8f8d6ad8698b820a7cf9e1be7155e9a241f556658c55428ec0563514365799a\ + 4be2be5a685a80971ddcfa80cb422cdd0101ec04cb847f000001820cfa8215a8d790000000000000\ + 000000000000000000018208ae820d058443b9a3550102\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + 577be4349c4dd26768081f58de4c6f375a7a22f3f7adda654d1428637412c3d7fe917cadc56d4e5e\ + 7ffae1dbe3efffb9849feb71b262de37977e7c7a44e677295680e9e38ab26bee2fcbae207fba3ff3\ + d74069a50b902a82c9903ed37cc993c50001f83e82022bd79020010db83c4d001500000000abcdef\ + 12820cfa8215a8d79020010db885a308d313198a2e037073488208ae82823a8443b9a355c5010203\ + 040531b9019afde696e582a78fa8d95ea13ce3297d4afb8ba6433e4154caa5ac6431af1b80ba7602\ + 3fa4090c408f6b4bc3701562c031041d4702971d102c9ab7fa5eed4cd6bab8f7af956f7d565ee191\ + 7084a95398b6a21eac920fe3dd1345ec0a7ef39367ee69ddf092cbfe5b93e5e568ebc491983c09c7\ + 6d922dc3\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + 09b2428d83348d27cdf7064ad9024f526cebc19e4958f0fdad87c15eb598dd61d08423e0bf66b206\ + 9869e1724125f820d851c136684082774f870e614d95a2855d000f05d1648b2d5945470bc187c2d2\ + 216fbe870f43ed0909009882e176a46b0102f846d79020010db885a308d313198a2e037073488208\ + ae82823aa0fbc914b16819237dcd8801d7e53f69e9719adecb3cc0e790c57e91ca4461c9548443b9\ + a355c6010203c2040506a0c969a58f6f9095004c0177a6b47f451530cab38966a25cca5cb58f0555 + 42124e\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + c7c44041b9f7c7e41934417ebac9a8e1a4c6298f74553f2fcfdcae6ed6fe53163eb3d2b52e39fe91\ + 831b8a927bf4fc222c3902202027e5e9eb812195f95d20061ef5cd31d502e47ecb61183f74a504fe\ + 04c51e73df81f25c4d506b26db4517490103f84eb840ca634cae0d49acb401d8a4c6b6fe8c55b70d\ + 115bf400769cc1400f3258cd31387574077f301b421bc84df7266c44e9e6d569fc56be0081290476\ + 7bf5ccd1fc7f8443b9a35582999983999999280dc62cc8255c73471e0a61da0c89acdc0e035e260a\ + dd7fc0c04ad9ebf3919644c91cb247affc82b69bd2ca235c71eab8e49737c937a2c396\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + + let packet = "\ + c679fc8fe0b8b12f06577f2e802d34f6fa257e6137a995f6f4cbfc9ee50ed3710faf6e66f932c4c8\ + d81d64343f429651328758b47d3dbc02c4042f0fff6946a50f4a49037a72bb550f3a7872363a83e1\ + b9ee6469856c24eb4ef80b7535bcf99c0004f9015bf90150f84d846321163782115c82115db84031\ + 55e1427f85f10a5c9a7755877748041af1bcd8d474ec065eb33df57a97babf54bfd2103575fa8291\ + 15d224c523596b401065a97f74010610fce76382c0bf32f84984010203040101b840312c55512422\ + cf9b8a4097e9a6ad79402e87a15ae909a4bfefa22398f03d20951933beea1e4dfa6f968212385e82\ + 9f04c2d314fc2d4e255e0d3bc08792b069dbf8599020010db83c4d001500000000abcdef12820d05\ + 820d05b84038643200b172dcfef857492156971f0e6aa2c538d8b74010f8e140811d53b98c765dd2\ + d96126051913f44582e8c199ad7c6d6819e9a56483f637feaac9448aacf8599020010db885a308d3\ + 13198a2e037073488203e78203e8b8408dcab8618c3253b558d459da53bd8fa68935a719aff8b811\ + 197101a4b2b47dd2d47295286fc00cc081bb542d760717d1bdd6bec2c37cd72eca367d6dd3b9df73\ + 8443b9a355010203b525a138aa34383fec3d2719a0\ + ".from_hex().unwrap(); + assert!(discovery.on_packet(&packet, from.clone()).is_ok()); + } + } diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index 087de63b3..cca133ba9 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -15,9 +15,11 @@ // along with Parity. If not, see . use std::sync::Arc; +use rand::random; use mio::*; use mio::tcp::*; use hash::*; +use rlp::*; use sha3::Hashable; use bytes::Bytes; use crypto::*; @@ -36,8 +38,12 @@ enum HandshakeState { New, /// Waiting for auth packet ReadingAuth, + /// Waiting for extended auth packet + ReadingAuthEip8, /// Waiting for ack packet ReadingAck, + /// Waiting for extended ack packet + ReadingAckEip8, /// Ready to start a session StartSession, } @@ -57,9 +63,11 @@ pub struct Handshake { /// Connection nonce pub nonce: H256, /// Handshake public key - pub remote_public: Public, + pub remote_ephemeral: Public, /// Remote connection nonce. pub remote_nonce: H256, + /// Remote RLPx protocol version. + pub remote_version: u64, /// A copy of received encryped auth packet pub auth_cipher: Bytes, /// A copy of received encryped ack packet @@ -68,9 +76,12 @@ pub struct Handshake { pub expired: bool, } -const AUTH_PACKET_SIZE: usize = 307; -const ACK_PACKET_SIZE: usize = 210; +const V4_AUTH_PACKET_SIZE: usize = 307; +const V4_ACK_PACKET_SIZE: usize = 210; const HANDSHAKE_TIMEOUT: u64 = 5000; +const PROTOCOL_VERSION: u64 = 4; +// Amount of bytes added when encrypting with encryptECIES. +const ECIES_OVERHEAD: usize = 113; impl Handshake { /// Create a new handshake object @@ -82,8 +93,9 @@ impl Handshake { state: HandshakeState::New, ecdhe: try!(KeyPair::create()), nonce: nonce.clone(), - remote_public: Public::new(), + remote_ephemeral: Public::new(), remote_nonce: H256::new(), + remote_version: PROTOCOL_VERSION, auth_cipher: Bytes::new(), ack_cipher: Bytes::new(), expired: false, @@ -115,11 +127,11 @@ impl Handshake { self.originated = originated; io.register_timer(self.connection.token, HANDSHAKE_TIMEOUT).ok(); if originated { - try!(self.write_auth(host)); + try!(self.write_auth(host.secret(), host.id())); } else { self.state = HandshakeState::ReadingAuth; - self.connection.expect(AUTH_PACKET_SIZE); + self.connection.expect(V4_AUTH_PACKET_SIZE); }; Ok(()) } @@ -134,20 +146,28 @@ impl Handshake { if !self.expired() { io.clear_timer(self.connection.token).unwrap(); match self.state { + HandshakeState::New => {} HandshakeState::ReadingAuth => { if let Some(data) = try!(self.connection.readable()) { - try!(self.read_auth(host, &data)); - try!(self.write_ack()); + try!(self.read_auth(host.secret(), &data)); + }; + }, + HandshakeState::ReadingAuthEip8 => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_auth_eip8(host.secret(), &data)); }; }, HandshakeState::ReadingAck => { if let Some(data) = try!(self.connection.readable()) { - try!(self.read_ack(host, &data)); - self.state = HandshakeState::StartSession; + try!(self.read_ack(host.secret(), &data)); + }; + }, + HandshakeState::ReadingAckEip8 => { + if let Some(data) = try!(self.connection.readable()) { + try!(self.read_ack_eip8(host.secret(), &data)); }; }, HandshakeState::StartSession => {}, - _ => { panic!("Unexpected state"); } } if self.state != HandshakeState::StartSession { try!(io.update_registration(self.connection.token)); @@ -190,48 +210,105 @@ impl Handshake { Ok(()) } + fn set_auth(&mut self, host_secret: &Secret, sig: &[u8], remote_public: &[u8], remote_nonce: &[u8], remote_version: u64) -> Result<(), UtilError> { + self.id.clone_from_slice(remote_public); + self.remote_nonce.clone_from_slice(remote_nonce); + self.remote_version = remote_version; + let shared = try!(ecdh::agree(host_secret, &self.id)); + let signature = Signature::from_slice(sig); + self.remote_ephemeral = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); + Ok(()) + } + /// Parse, validate and confirm auth message - fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); - if data.len() != AUTH_PACKET_SIZE { + fn read_auth(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received handshake auth from {:?}", self.connection.socket.peer_addr()); + if data.len() != V4_AUTH_PACKET_SIZE { debug!(target:"net", "Wrong auth packet size"); return Err(From::from(NetworkError::BadProtocol)); } self.auth_cipher = data.to_vec(); - let auth = try!(ecies::decrypt(host.secret(), data)); - let (sig, rest) = auth.split_at(65); - let (hepubk, rest) = rest.split_at(32); - let (pubk, rest) = rest.split_at(64); - let (nonce, _) = rest.split_at(32); - self.id.clone_from_slice(pubk); - self.remote_nonce.clone_from_slice(nonce); - let shared = try!(ecdh::agree(host.secret(), &self.id)); - let signature = Signature::from_slice(sig); - let spub = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); - self.remote_public = spub.clone(); - if &spub.sha3()[..] != hepubk { - trace!(target:"net", "Handshake hash mismath with {:?}", self.connection.socket.peer_addr()); - return Err(From::from(NetworkError::Auth)); - }; + match ecies::decrypt(secret, &[], data) { + Ok(auth) => { + let (sig, rest) = auth.split_at(65); + let (_, rest) = rest.split_at(32); + let (pubk, rest) = rest.split_at(64); + let (nonce, _) = rest.split_at(32); + try!(self.set_auth(secret, sig, pubk, nonce, PROTOCOL_VERSION)); + try!(self.write_ack()); + } + Err(_) => { + // Try to interpret as EIP-8 packet + let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2; + if total < V4_AUTH_PACKET_SIZE { + debug!(target:"net", "Wrong EIP8 auth packet size"); + return Err(From::from(NetworkError::BadProtocol)); + } + let rest = total - data.len(); + self.state = HandshakeState::ReadingAuthEip8; + self.connection.expect(rest); + } + } + Ok(()) + } + + fn read_auth_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + self.auth_cipher.extend_from_slice(data); + let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..])); + let rlp = UntrustedRlp::new(&auth); + let signature: Signature = try!(rlp.val_at(0)); + let remote_public: Public = try!(rlp.val_at(1)); + let remote_nonce: H256 = try!(rlp.val_at(2)); + let remote_version: u64 = try!(rlp.val_at(3)); + try!(self.set_auth(secret, &signature, &remote_public, &remote_nonce, remote_version)); + try!(self.write_ack_eip8()); Ok(()) } /// Parse and validate ack message - fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { + fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); - if data.len() != ACK_PACKET_SIZE { + if data.len() != V4_ACK_PACKET_SIZE { debug!(target:"net", "Wrong ack packet size"); return Err(From::from(NetworkError::BadProtocol)); } self.ack_cipher = data.to_vec(); - let ack = try!(ecies::decrypt(host.secret(), data)); - self.remote_public.clone_from_slice(&ack[0..64]); - self.remote_nonce.clone_from_slice(&ack[64..(64+32)]); + match ecies::decrypt(secret, &[], data) { + Ok(ack) => { + self.remote_ephemeral.clone_from_slice(&ack[0..64]); + self.remote_nonce.clone_from_slice(&ack[64..(64+32)]); + self.state = HandshakeState::StartSession; + } + Err(_) => { + // Try to interpret as EIP-8 packet + let total = (((data[0] as u16) << 8 | (data[1] as u16)) as usize) + 2; + if total < V4_ACK_PACKET_SIZE { + debug!(target:"net", "Wrong EIP8 ack packet size"); + return Err(From::from(NetworkError::BadProtocol)); + } + let rest = total - data.len(); + self.state = HandshakeState::ReadingAckEip8; + self.connection.expect(rest); + } + } + Ok(()) + } + + fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { + trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + self.ack_cipher.extend_from_slice(data); + let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..])); + let rlp = UntrustedRlp::new(&ack); + self.remote_ephemeral = try!(rlp.val_at(0)); + self.remote_nonce = try!(rlp.val_at(1)); + self.remote_version = try!(rlp.val_at(2)); + self.state = HandshakeState::StartSession; Ok(()) } /// Sends auth message - fn write_auth(&mut self, host: &HostInfo) -> Result<(), UtilError> { + fn write_auth(&mut self, secret: &Secret, public: &Public) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants let len = data.len(); @@ -243,16 +320,16 @@ impl Handshake { let (nonce, _) = rest.split_at_mut(32); // E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0) - let shared = try!(crypto::ecdh::agree(host.secret(), &self.id)); + let shared = try!(crypto::ecdh::agree(secret, &self.id)); try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); self.ecdhe.public().sha3_into(hepubk); - host.id().copy_to(pubk); + public.copy_to(pubk); self.nonce.copy_to(nonce); } - let message = try!(crypto::ecies::encrypt(&self.id, &data)); + let message = try!(crypto::ecies::encrypt(&self.id, &[], &data)); self.auth_cipher = message.clone(); self.connection.send(message); - self.connection.expect(ACK_PACKET_SIZE); + self.connection.expect(V4_ACK_PACKET_SIZE); self.state = HandshakeState::ReadingAck; Ok(()) } @@ -269,10 +346,222 @@ impl Handshake { self.ecdhe.public().copy_to(epubk); self.nonce.copy_to(nonce); } - let message = try!(crypto::ecies::encrypt(&self.id, &data)); + let message = try!(crypto::ecies::encrypt(&self.id, &[], &data)); self.ack_cipher = message.clone(); self.connection.send(message); self.state = HandshakeState::StartSession; Ok(()) } + + /// Sends EIP8 ack message + fn write_ack_eip8(&mut self) -> Result<(), UtilError> { + trace!(target:"net", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr()); + let mut rlp = RlpStream::new_list(3); + rlp.append(self.ecdhe.public()); + rlp.append(&self.nonce); + rlp.append(&PROTOCOL_VERSION); + + let pad_array = [0u8; 200]; + let pad = &pad_array[0 .. 100 + random::() % 100]; + rlp.append_raw(pad, 0); + + let encoded = rlp.drain(); + let len = (encoded.len() + ECIES_OVERHEAD) as u16; + let prefix = [ (len >> 8) as u8, (len & 0xff) as u8 ]; + let message = try!(crypto::ecies::encrypt(&self.id, &prefix, &encoded)); + self.ack_cipher.extend_from_slice(&prefix); + self.ack_cipher.extend_from_slice(&message); + self.connection.send(self.ack_cipher.clone()); + self.state = HandshakeState::StartSession; + Ok(()) + } } + +#[cfg(test)] +mod test { + use std::sync::Arc; + use std::str::FromStr; + use rustc_serialize::hex::FromHex; + use super::*; + use crypto::*; + use hash::*; + use std::net::SocketAddr; + use mio::tcp::TcpStream; + use network::stats::NetworkStats; + + fn check_auth(h: &Handshake, version: u64) { + assert_eq!(h.id, Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap()); + assert_eq!(h.remote_nonce, H256::from_str("7e968bba13b6c50e2c4cd7f241cc0d64d1ac25c7f5952df231ac6a2bda8ee5d6").unwrap()); + assert_eq!(h.remote_ephemeral, Public::from_str("654d1044b69c577a44e5f01a1209523adb4026e70c62d1c13a067acabc09d2667a49821a0ad4b634554d330a15a58fe61f8a8e0544b310c6de7b0c8da7528a8d").unwrap()); + assert_eq!(h.remote_version, version); + } + + fn check_ack(h: &Handshake, version: u64) { + assert_eq!(h.remote_nonce, H256::from_str("559aead08264d5795d3909718cdd05abd49572e84fe55590eef31a88a08fdffd").unwrap()); + assert_eq!(h.remote_ephemeral, Public::from_str("b6d82fa3409da933dbf9cb0140c5dde89f4e64aec88d476af648880f4a10e1e49fe35ef3e69e93dd300b4797765a747c6384a6ecf5db9c2690398607a86181e4").unwrap()); + assert_eq!(h.remote_version, version); + } + + fn create_handshake(to: Option<&Public>) -> Handshake { + let addr = SocketAddr::from_str("127.0.0.1:50556").unwrap(); + let socket = TcpStream::connect(&addr).unwrap(); + let nonce = H256::new(); + Handshake::new(0, to, socket, &nonce, Arc::new(NetworkStats::new())).unwrap() + } + + #[test] + fn test_handshake_auth_plain() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 048ca79ad18e4b0659fab4853fe5bc58eb83992980f4c9cc147d2aa31532efd29a3d3dc6a3d89eaf\ + 913150cfc777ce0ce4af2758bf4810235f6e6ceccfee1acc6b22c005e9e3a49d6448610a58e98744\ + ba3ac0399e82692d67c1f58849050b3024e21a52c9d3b01d871ff5f210817912773e610443a9ef14\ + 2e91cdba0bd77b5fdf0769b05671fc35f83d83e4d3b0b000c6b2a1b1bba89e0fc51bf4e460df3105\ + c444f14be226458940d6061c296350937ffd5e3acaceeaaefd3c6f74be8e23e0f45163cc7ebd7622\ + 0f0128410fd05250273156d548a414444ae2f7dea4dfca2d43c057adb701a715bf59f6fb66b2d1d2\ + 0f2c703f851cbf5ac47396d9ca65b6260bd141ac4d53e2de585a73d1750780db4c9ee4cd4d225173\ + a4592ee77e2bd94d0be3691f3b406f9bba9b591fc63facc016bfa8\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 4); + } + + #[test] + fn test_handshake_auth_eip8() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 01b304ab7578555167be8154d5cc456f567d5ba302662433674222360f08d5f1534499d3678b513b\ + 0fca474f3a514b18e75683032eb63fccb16c156dc6eb2c0b1593f0d84ac74f6e475f1b8d56116b84\ + 9634a8c458705bf83a626ea0384d4d7341aae591fae42ce6bd5c850bfe0b999a694a49bbbaf3ef6c\ + da61110601d3b4c02ab6c30437257a6e0117792631a4b47c1d52fc0f8f89caadeb7d02770bf999cc\ + 147d2df3b62e1ffb2c9d8c125a3984865356266bca11ce7d3a688663a51d82defaa8aad69da39ab6\ + d5470e81ec5f2a7a47fb865ff7cca21516f9299a07b1bc63ba56c7a1a892112841ca44b6e0034dee\ + 70c9adabc15d76a54f443593fafdc3b27af8059703f88928e199cb122362a4b35f62386da7caad09\ + c001edaeb5f8a06d2b26fb6cb93c52a9fca51853b68193916982358fe1e5369e249875bb8d0d0ec3\ + 6f917bc5e1eafd5896d46bd61ff23f1a863a8a8dcd54c7b109b771c8e61ec9c8908c733c0263440e\ + 2aa067241aaa433f0bb053c7b31a838504b148f570c0ad62837129e547678c5190341e4f1693956c\ + 3bf7678318e2d5b5340c9e488eefea198576344afbdf66db5f51204a6961a63ce072c8926c\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8); + h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 4); + } + + #[test] + fn test_handshake_auth_eip8_2() { + let mut h = create_handshake(None); + let secret = Secret::from_str("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291").unwrap(); + let auth = + "\ + 01b8044c6c312173685d1edd268aa95e1d495474c6959bcdd10067ba4c9013df9e40ff45f5bfd6f7\ + 2471f93a91b493f8e00abc4b80f682973de715d77ba3a005a242eb859f9a211d93a347fa64b597bf\ + 280a6b88e26299cf263b01b8dfdb712278464fd1c25840b995e84d367d743f66c0e54a586725b7bb\ + f12acca27170ae3283c1073adda4b6d79f27656993aefccf16e0d0409fe07db2dc398a1b7e8ee93b\ + cd181485fd332f381d6a050fba4c7641a5112ac1b0b61168d20f01b479e19adf7fdbfa0905f63352\ + bfc7e23cf3357657455119d879c78d3cf8c8c06375f3f7d4861aa02a122467e069acaf513025ff19\ + 6641f6d2810ce493f51bee9c966b15c5043505350392b57645385a18c78f14669cc4d960446c1757\ + 1b7c5d725021babbcd786957f3d17089c084907bda22c2b2675b4378b114c601d858802a55345a15\ + 116bc61da4193996187ed70d16730e9ae6b3bb8787ebcaea1871d850997ddc08b4f4ea668fbf3740\ + 7ac044b55be0908ecb94d4ed172ece66fd31bfdadf2b97a8bc690163ee11f5b575a4b44e36e2bfb2\ + f0fce91676fd64c7773bac6a003f481fddd0bae0a1f31aa27504e2a533af4cef3b623f4791b2cca6\ + d490\ + ".from_hex().unwrap(); + + h.read_auth(&secret, &auth[0..super::V4_AUTH_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAuthEip8); + h.read_auth_eip8(&secret, &auth[super::V4_AUTH_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_auth(&h, 56); + let ack = h.ack_cipher.clone(); + let total = (((ack[0] as u16) << 8 | (ack[1] as u16)) as usize) + 2; + assert_eq!(ack.len(), total); + } + + #[test] + fn test_handshake_ack_plain() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 049f8abcfa9c0dc65b982e98af921bc0ba6e4243169348a236abe9df5f93aa69d99cadddaa387662\ + b0ff2c08e9006d5a11a278b1b3331e5aaabf0a32f01281b6f4ede0e09a2d5f585b26513cb794d963\ + 5a57563921c04a9090b4f14ee42be1a5461049af4ea7a7f49bf4c97a352d39c8d02ee4acc416388c\ + 1c66cec761d2bc1c72da6ba143477f049c9d2dde846c252c111b904f630ac98e51609b3b1f58168d\ + dca6505b7196532e5f85b259a20c45e1979491683fee108e9660edbf38f3add489ae73e3dda2c71b\ + d1497113d5c755e942d1\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 4); + } + + #[test] + fn test_handshake_ack_eip8() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 01ea0451958701280a56482929d3b0757da8f7fbe5286784beead59d95089c217c9b917788989470\ + b0e330cc6e4fb383c0340ed85fab836ec9fb8a49672712aeabbdfd1e837c1ff4cace34311cd7f4de\ + 05d59279e3524ab26ef753a0095637ac88f2b499b9914b5f64e143eae548a1066e14cd2f4bd7f814\ + c4652f11b254f8a2d0191e2f5546fae6055694aed14d906df79ad3b407d94692694e259191cde171\ + ad542fc588fa2b7333313d82a9f887332f1dfc36cea03f831cb9a23fea05b33deb999e85489e645f\ + 6aab1872475d488d7bd6c7c120caf28dbfc5d6833888155ed69d34dbdc39c1f299be1057810f34fb\ + e754d021bfca14dc989753d61c413d261934e1a9c67ee060a25eefb54e81a4d14baff922180c395d\ + 3f998d70f46f6b58306f969627ae364497e73fc27f6d17ae45a413d322cb8814276be6ddd13b885b\ + 201b943213656cde498fa0e9ddc8e0b8f8a53824fbd82254f3e2c17e8eaea009c38b4aa0a3f306e8\ + 797db43c25d68e86f262e564086f59a2fc60511c42abfb3057c247a8a8fe4fb3ccbadde17514b7ac\ + 8000cdb6a912778426260c47f38919a91f25f4b5ffb455d6aaaf150f7e5529c100ce62d6d92826a7\ + 1778d809bdf60232ae21ce8a437eca8223f45ac37f6487452ce626f549b3b5fdee26afd2072e4bc7\ + 5833c2464c805246155289f4\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAckEip8); + h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 4); + } + + #[test] + fn test_handshake_ack_eip8_2() { + let remote = Public::from_str("fda1cff674c90c9a197539fe3dfb53086ace64f83ed7c6eabec741f7f381cc803e52ab2cd55d5569bce4347107a310dfd5f88a010cd2ffd1005ca406f1842877").unwrap(); + let mut h = create_handshake(Some(&remote)); + let secret = Secret::from_str("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee").unwrap(); + let ack = + "\ + 01f004076e58aae772bb101ab1a8e64e01ee96e64857ce82b1113817c6cdd52c09d26f7b90981cd7\ + ae835aeac72e1573b8a0225dd56d157a010846d888dac7464baf53f2ad4e3d584531fa203658fab0\ + 3a06c9fd5e35737e417bc28c1cbf5e5dfc666de7090f69c3b29754725f84f75382891c561040ea1d\ + dc0d8f381ed1b9d0d4ad2a0ec021421d847820d6fa0ba66eaf58175f1b235e851c7e2124069fbc20\ + 2888ddb3ac4d56bcbd1b9b7eab59e78f2e2d400905050f4a92dec1c4bdf797b3fc9b2f8e84a482f3\ + d800386186712dae00d5c386ec9387a5e9c9a1aca5a573ca91082c7d68421f388e79127a5177d4f8\ + 590237364fd348c9611fa39f78dcdceee3f390f07991b7b47e1daa3ebcb6ccc9607811cb17ce51f1\ + c8c2c5098dbdd28fca547b3f58c01a424ac05f869f49c6a34672ea2cbbc558428aa1fe48bbfd6115\ + 8b1b735a65d99f21e70dbc020bfdface9f724a0d1fb5895db971cc81aa7608baa0920abb0a565c9c\ + 436e2fd13323428296c86385f2384e408a31e104670df0791d93e743a3a5194ee6b076fb6323ca59\ + 3011b7348c16cf58f66b9633906ba54a2ee803187344b394f75dd2e663a57b956cb830dd7a908d4f\ + 39a2336a61ef9fda549180d4ccde21514d117b6c6fd07a9102b5efe710a32af4eeacae2cb3b1dec0\ + 35b9593b48b9d3ca4c13d245d5f04169b0b1\ + ".from_hex().unwrap(); + + h.read_ack(&secret, &ack[0..super::V4_ACK_PACKET_SIZE]).unwrap(); + assert_eq!(h.state, super::HandshakeState::ReadingAckEip8); + h.read_ack_eip8(&secret, &ack[super::V4_ACK_PACKET_SIZE..]).unwrap(); + assert_eq!(h.state, super::HandshakeState::StartSession); + check_ack(&h, 57); + } +} + diff --git a/util/src/network/host.rs b/util/src/network/host.rs index f54c85855..42e8ff93d 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -376,7 +376,7 @@ impl Host where Message: Send + Sync + Clone { let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; self.pinned_nodes.push(n.id.clone()); self.nodes.write().unwrap().add_node(n); - if let &mut Some(ref mut discovery) = self.discovery.lock().unwrap().deref_mut() { + if let Some(ref mut discovery) = *self.discovery.lock().unwrap().deref_mut() { discovery.add_node(entry); } } @@ -418,7 +418,7 @@ impl Host where Message: Send + Sync + Clone { } Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } }; - + // Setup the server socket *tcp_listener = Some(TcpListener::bind(&listen_address).unwrap()); self.info.write().unwrap().public_endpoint = public_endpoint.clone(); @@ -697,7 +697,7 @@ impl Host where Message: Send + Sync + Clone { let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); let mut discovery = self.discovery.lock().unwrap(); - if let &mut Some(ref mut discovery) = discovery.deref_mut() { + if let Some(ref mut discovery) = *discovery.deref_mut() { discovery.add_node(entry); } } diff --git a/util/src/numbers.rs b/util/src/numbers.rs new file mode 100644 index 000000000..b79338fc1 --- /dev/null +++ b/util/src/numbers.rs @@ -0,0 +1,20 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Utils number types. + +pub use hash::*; +pub use bigint::uint::*; diff --git a/util/src/rlp/bytes.rs b/util/src/rlp/bytes.rs index 2ff6281cc..e8bfa57b0 100644 --- a/util/src/rlp/bytes.rs +++ b/util/src/rlp/bytes.rs @@ -21,7 +21,7 @@ use std::mem; use std::fmt; use std::cmp::Ordering; use std::error::Error as StdError; -use uint::{Uint, U128, U256}; +use bigint::uint::{Uint, U128, U256}; use hash::FixedHash; use elastic_array::*; diff --git a/util/src/rlp/tests.rs b/util/src/rlp/tests.rs index eb2039103..a92dd5c4a 100644 --- a/util/src/rlp/tests.rs +++ b/util/src/rlp/tests.rs @@ -21,7 +21,7 @@ use std::{fmt, cmp}; use std::str::FromStr; use rlp; use rlp::{UntrustedRlp, RlpStream, View, Stream, DecoderError}; -use uint::U256; +use bigint::uint::U256; #[test] fn rlp_at() { diff --git a/util/src/table.rs b/util/src/table.rs new file mode 100644 index 000000000..f04a498f8 --- /dev/null +++ b/util/src/table.rs @@ -0,0 +1,254 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! A collection associating pair of keys (row and column) with a single value. + +use std::hash::Hash; +use std::collections::HashMap; + +/// Structure to hold double-indexed values +/// +/// You can obviously use `HashMap<(Row,Col), Val>`, but this structure gives +/// you better access to all `Columns` in Specific `Row`. Namely you can get sub-hashmap +/// `HashMap` for specific `Row` +pub struct Table + where Row: Eq + Hash + Clone, + Col: Eq + Hash { + map: HashMap>, +} + +impl Table + where Row: Eq + Hash + Clone, + Col: Eq + Hash { + /// Creates new Table + pub fn new() -> Table { + Table { + map: HashMap::new(), + } + } + + /// Removes all elements from this Table + pub fn clear(&mut self) { + self.map.clear(); + } + + /// Returns length of the Table (number of (row, col, val) tuples) + pub fn len(&self) -> usize { + self.map.values().fold(0, |acc, v| acc + v.len()) + } + + /// Check if there is any element in this Table + pub fn is_empty(&self) -> bool { + self.map.is_empty() || self.map.values().all(|v| v.is_empty()) + } + + /// Get mutable reference for single Table row. + pub fn row_mut(&mut self, row: &Row) -> Option<&mut HashMap> { + self.map.get_mut(row) + } + + /// Checks if row is defined for that table (note that even if defined it might be empty) + pub fn has_row(&self, row: &Row) -> bool { + self.map.contains_key(row) + } + + /// Get immutable reference for single row in this Table + pub fn row(&self, row: &Row) -> Option<&HashMap> { + self.map.get(row) + } + + /// Get element in cell described by `(row, col)` + pub fn get(&self, row: &Row, col: &Col) -> Option<&Val> { + self.map.get(row).and_then(|r| r.get(col)) + } + + /// Remove value from specific cell + /// + /// It will remove the row if it's the last value in it + pub fn remove(&mut self, row: &Row, col: &Col) -> Option { + let (val, is_empty) = { + let row_map = self.map.get_mut(row); + if let None = row_map { + return None; + } + let mut row_map = row_map.unwrap(); + let val = row_map.remove(col); + (val, row_map.is_empty()) + }; + // Clean row + if is_empty { + self.map.remove(row); + } + val + } + + /// Remove given row from Table if there are no values defined in it + /// + /// When using `#row_mut` it may happen that all values from some row are drained. + /// Table however will not be aware that row is empty. + /// You can use this method to explicitly remove row entry from the Table. + pub fn clear_if_empty(&mut self, row: &Row) { + let is_empty = self.map.get(row).map_or(false, |m| m.is_empty()); + if is_empty { + self.map.remove(row); + } + } + + /// Inserts new value to specified cell + /// + /// Returns previous value (if any) + pub fn insert(&mut self, row: Row, col: Col, val: Val) -> Option { + self.map.entry(row).or_insert_with(|| HashMap::new()).insert(col, val) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn should_create_empty_table() { + // when + let table : Table = Table::new(); + + // then + assert!(table.is_empty()); + assert_eq!(table.len(), 0); + } + + #[test] + fn should_insert_elements_and_return_previous_if_any() { + // given + let mut table = Table::new(); + + // when + let r1 = table.insert(5, 4, true); + let r2 = table.insert(10, 4, true); + let r3 = table.insert(10, 10, true); + let r4 = table.insert(10, 10, false); + + // then + assert!(r1.is_none()); + assert!(r2.is_none()); + assert!(r3.is_none()); + assert!(r4.is_some()); + assert!(!table.is_empty()); + assert_eq!(r4.unwrap(), true); + assert_eq!(table.len(), 3); + } + + #[test] + fn should_remove_element() { + // given + let mut table = Table::new(); + table.insert(5, 4, true); + assert!(!table.is_empty()); + assert_eq!(table.len(), 1); + + // when + let r = table.remove(&5, &4); + + // then + assert!(table.is_empty()); + assert_eq!(table.len() ,0); + assert_eq!(r.unwrap(), true); + } + + #[test] + fn should_return_none_if_trying_to_remove_non_existing_element() { + // given + let mut table : Table = Table::new(); + assert!(table.is_empty()); + + // when + let r = table.remove(&5, &4); + + // then + assert!(r.is_none()); + } + + #[test] + fn should_clear_row_if_removing_last_element() { + // given + let mut table = Table::new(); + table.insert(5, 4, true); + assert!(table.has_row(&5)); + + // when + let r = table.remove(&5, &4); + + // then + assert!(r.is_some()); + assert!(!table.has_row(&5)); + } + + #[test] + fn should_return_element_given_row_and_col() { + // given + let mut table = Table::new(); + table.insert(1551, 1234, 123); + + // when + let r1 = table.get(&1551, &1234); + let r2 = table.get(&5, &4); + + // then + assert!(r1.is_some()); + assert!(r2.is_none()); + assert_eq!(r1.unwrap(), &123); + } + + #[test] + fn should_clear_table() { + // given + let mut table = Table::new(); + table.insert(1, 1, true); + table.insert(1, 2, false); + table.insert(2, 2, false); + assert_eq!(table.len(), 3); + + // when + table.clear(); + + // then + assert!(table.is_empty()); + assert_eq!(table.len(), 0); + assert_eq!(table.has_row(&1), false); + assert_eq!(table.has_row(&2), false); + } + + #[test] + fn should_return_mutable_row() { + // given + let mut table = Table::new(); + table.insert(1, 1, true); + table.insert(1, 2, false); + table.insert(2, 2, false); + + // when + { + let mut row = table.row_mut(&1).unwrap(); + row.remove(&1); + row.remove(&2); + } + assert!(table.has_row(&1)); + table.clear_if_empty(&1); + + // then + assert!(!table.has_row(&1)); + assert_eq!(table.len(), 1); + } +} diff --git a/util/src/trie/standardmap.rs b/util/src/trie/standardmap.rs index 98a5ae0f0..b7f3a9500 100644 --- a/util/src/trie/standardmap.rs +++ b/util/src/trie/standardmap.rs @@ -20,6 +20,7 @@ extern crate rand; use bytes::*; use sha3::*; use hash::*; +use rlp::encode; /// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { @@ -39,6 +40,8 @@ pub enum ValueMode { Mirror, /// Randomly (50:50) 1 or 32 byte randomly string. Random, + /// RLP-encoded index. + Index, } /// Standard test map for profiling tries. @@ -89,19 +92,27 @@ impl StandardMap { /// Create the standard map (set of keys and values) for the object's fields. pub fn make(&self) -> Vec<(Bytes, Bytes)> { + self.make_with(&mut H256::new()) + } + + /// Create the standard map (set of keys and values) for the object's fields, using the given seed. + pub fn make_with(&self, seed: &mut H256) -> Vec<(Bytes, Bytes)> { let low = b"abcdef"; let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); - for _ in 0..self.count { + for index in 0..self.count { let k = match self.alphabet { - Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), - Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), - Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), - Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), + Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, seed), + }; + let v = match self.value_mode { + ValueMode::Mirror => k.clone(), + ValueMode::Random => Self::random_value(seed), + ValueMode::Index => encode(&index).to_vec(), }; - let v = match self.value_mode { ValueMode::Mirror => k.clone(), ValueMode::Random => Self::random_value(&mut seed) }; d.push((k, v)) } d diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 2b4567264..829c1e518 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -687,31 +687,10 @@ mod tests { use super::*; use nibbleslice::*; use rlp::*; - use rand::random; - use std::collections::HashSet; - use bytes::{ToPretty,Bytes,Populatable}; + use bytes::ToPretty; use super::super::node::*; use super::super::trietraits::*; - - fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec { - let mut ret: Vec = Vec::new(); - let r = min_count + if journal_count > 0 {random::() % journal_count} else {0}; - for _ in 0..r { - ret.push(alphabet[random::() % alphabet.len()]); - } - ret - } - - fn random_value_indexed(j: usize) -> Bytes { - match random::() % 2 { - 0 => encode(&j).to_vec(), - _ => { - let mut h = H256::new(); - h.as_slice_mut()[31] = j as u8; - encode(&h).to_vec() - }, - } - } + use super::super::standardmap::*; fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { let mut t = TrieDBMut::new(db, root); @@ -756,20 +735,18 @@ mod tests { };*/ // panic!(); + let mut seed = H256::new(); for test_i in 0..1 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); } - let mut x: Vec<(Vec, Vec)> = Vec::new(); - let mut got: HashSet> = HashSet::new(); - let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - for j in 0..100usize { - let key = random_key(alphabet, 5, 0); - if !got.contains(&key) { - x.push((key.clone(), random_value_indexed(j))); - got.insert(key); - } - } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 100, + }.make_with(&mut seed); let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); @@ -1049,13 +1026,16 @@ mod tests { #[test] fn stress() { + let mut seed = H256::new(); for _ in 0..50 { - let mut x: Vec<(Vec, Vec)> = Vec::new(); - let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - for j in 0..4u32 { - let key = random_key(alphabet, 5, 1); - x.push((key, encode(&j).to_vec())); - } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); let mut root = H256::new();