From 434e0185841cca5e890ce9f8ea90cb1d9041938f Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 5 Jul 2018 15:18:50 +0200 Subject: [PATCH 1/3] bump fs-swap to 0.2.3 so it is compatible with osx 10.11 again (#9050) --- Cargo.lock | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 855cc5e62..96f58d541 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,7 +558,6 @@ dependencies = [ "ethstore 0.2.0", "evm 0.1.0", "fake-hardware-wallet 0.0.1", - "fetch 0.1.0", "hardware-wallet 1.12.0", "hashdb 0.2.0", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1154,10 +1153,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "fs-swap" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1567,7 +1568,7 @@ version = "0.1.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "fs-swap 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fs-swap 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1603,6 +1604,15 @@ name = "libc" version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libloading" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libusb" version = "0.3.0" @@ -3974,7 +3984,7 @@ dependencies = [ "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" "checksum flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909" "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344" -"checksum fs-swap 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31a94e9407e53addc49de767234a0b000978523c59117e5badb575ccbb8370f6" +"checksum fs-swap 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67f816b2a5f8a6628764a4323d1a8d9ad5303266c4e4e4486ba680f477ba7e62" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" @@ -4015,6 +4025,7 @@ dependencies = [ "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" "checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121" +"checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" "checksum libusb 0.3.0 (git+https://github.com/paritytech/libusb-rs)" = "" "checksum libusb-sys 0.2.4 (git+https://github.com/paritytech/libusb-sys)" = "" "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939" From 802d68499449551b6cc8d63002eb26ac9cf273f3 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 5 Jul 2018 17:15:03 +0200 Subject: [PATCH 2/3] reduce number of constraints for triedb types (#9043) * NodeHandle does not require Hasher * Node does not require Hasher * change name of the template typo from I to O --- Cargo.lock | 1 + util/hashdb/src/lib.rs | 2 +- util/patricia_trie/Cargo.toml | 5 +- util/patricia_trie/src/lib.rs | 5 +- util/patricia_trie/src/triedbmut.rs | 85 +++++++++++++++-------------- 5 files changed, 53 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96f58d541..7f20bcbb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2485,6 +2485,7 @@ dependencies = [ "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "keccak-hasher 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 5961d90f9..4f5bed48c 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -23,7 +23,7 @@ use heapsize::HeapSizeOf; use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; -/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// Trait describing an object that can hash a slice of bytes. Used to abstract /// other types over the hashing algorithm. Defines a single `hash` method and an /// `Out` associated type with the necessary bounds. pub trait Hasher: Sync + Send { diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 532465678..ebd665045 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -7,10 +7,11 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" +ethcore-bytes = { version = "0.1.0", path = "../bytes" } +hashdb = { version = "0.2", path = "../hashdb" } +heapsize = "0.4" log = "0.3" rand = "0.4" -hashdb = { version = "0.2", path = "../hashdb" } -ethcore-bytes = { version = "0.1.0", path = "../bytes" } [dev-dependencies] env_logger = "0.5" diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 7cc623b1a..a028be87a 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -18,6 +18,7 @@ extern crate elastic_array; extern crate ethcore_bytes as bytes; extern crate hashdb; +extern crate heapsize; extern crate rand; #[macro_use] extern crate log; @@ -277,8 +278,8 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { } impl<'db, H, C> TrieFactory -where - H: Hasher, +where + H: Hasher, C: NodeCodec + 'db { /// Creates new factory. diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index f5c28bac3..4490285d5 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,6 +31,8 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; +use heapsize::HeapSizeOf; +use std::{fmt::Debug, hash::Hash}; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -39,20 +41,20 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H::Out), + Hash(H), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -fn empty_children() -> Box<[Option>; 16]> { +fn empty_children() -> Box<[Option>; 16]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, @@ -61,7 +63,7 @@ fn empty_children() -> Box<[Option>; 16]> { /// Node types in the Trie. #[derive(Debug)] -enum Node { +enum Node { /// Empty node. Empty, /// A leaf node contains the end of a key and a value. @@ -77,22 +79,24 @@ enum Node { Branch(Box<[Option>; 16]>, Option) } -impl Node { +impl Node where O: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle - where C: NodeCodec + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + where C: NodeCodec, + H: Hasher, { C::try_decode_hash(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { - let child = Node::from_encoded::(node, db, storage); + let child = Node::from_encoded::(node, db, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) }) } // decode a node from encoded bytes without getting its children. - fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self - where C: NodeCodec + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self + where C: NodeCodec, + H: Hasher, { match C::decode(data).expect("encoded bytes read from db; qed") { EncodedNode::Empty => Node::Empty, @@ -100,13 +104,13 @@ impl Node { EncodedNode::Extension(key, cb) => { Node::Extension( key.encoded(false), - Self::inline_or_hash::(cb, db, storage)) + Self::inline_or_hash::(cb, db, storage)) } EncodedNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; if !C::is_empty_node(raw) { - Some(Self::inline_or_hash::(raw, db, storage)) + Some(Self::inline_or_hash::(raw, db, storage)) } else { None } @@ -125,10 +129,11 @@ impl Node { } // TODO: parallelize - fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 + fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, - F: FnMut(NodeHandle) -> ChildReference + F: FnMut(NodeHandle) -> ChildReference, + H: Hasher, { match self { Node::Empty => C::empty_node(), @@ -139,7 +144,7 @@ impl Node { // map the `NodeHandle`s from the Branch to `ChildReferences` children.iter_mut() .map(Option::take) - .map(|maybe_child| + .map(|maybe_child| maybe_child.map(|child| child_cb(child)) ), value @@ -150,7 +155,7 @@ impl Node { } // post-inspect action. -enum Action { +enum Action { // Replace a node with a new one. Replace(Node), // Restore the original node. This trusts that the node is actually the original. @@ -160,14 +165,14 @@ enum Action { } // post-insert action. Same as action without delete -enum InsertAction { +enum InsertAction { // Replace a node with a new one. Replace(Node), // Restore the original node. Restore(Node), } -impl InsertAction { +impl InsertAction { fn into_action(self) -> Action { match self { InsertAction::Replace(n) => Action::Replace(n), @@ -184,11 +189,11 @@ impl InsertAction { } // What kind of node is stored here. -enum Stored { +enum Stored { // A new node. New(Node), // A cached node, loaded from the DB. - Cached(Node, H::Out), + Cached(Node, H), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s @@ -198,12 +203,12 @@ pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hash } /// Compact and cache-friendly storage for Trie nodes. -struct NodeStorage { +struct NodeStorage { nodes: Vec>, free_indices: VecDeque, } -impl NodeStorage { +impl NodeStorage { /// Create a new storage. fn empty() -> Self { NodeStorage { @@ -232,7 +237,7 @@ impl NodeStorage { } } -impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { +impl<'a, H> Index<&'a StorageHandle> for NodeStorage { type Output = Node; fn index(&self, handle: &'a StorageHandle) -> &Node { @@ -284,10 +289,10 @@ where H: Hasher + 'a, C: NodeCodec { - storage: NodeStorage, + storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, - root_handle: NodeHandle, + root_handle: NodeHandle, death_row: HashSet, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. @@ -347,7 +352,7 @@ where // cache a node by hash fn cache(&mut self, hash: H::Out) -> Result { let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_encoded::( + let node = Node::from_encoded::( &node_encoded, &*self.db, &mut self.storage @@ -357,8 +362,8 @@ where // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> - where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> + where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -380,7 +385,7 @@ where } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> where 'x: 'key { let mut handle = handle; @@ -429,7 +434,7 @@ where } /// insert a key-value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -443,7 +448,7 @@ where } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -604,7 +609,7 @@ where } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -619,7 +624,7 @@ where } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -705,7 +710,7 @@ where /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { + fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -825,7 +830,7 @@ where match self.storage.destroy(handle) { Stored::New(node) => { - let encoded_root = node.into_encoded::<_, C>(|child| self.commit_child(child) ); + let encoded_root = node.into_encoded::<_, C, H>(|child| self.commit_child(child) ); *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; @@ -845,14 +850,14 @@ where /// case where we can fit the actual data in the `Hasher`s output type, we /// store the data inline. This function is used as the callback to the /// `into_encoded` method of `Node`. - fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { + fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { match handle { NodeHandle::Hash(hash) => ChildReference::Hash(hash), NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { - let encoded = node.into_encoded::<_, C>(|node_handle| self.commit_child(node_handle) ); + let encoded = node.into_encoded::<_, C, H>(|node_handle| self.commit_child(node_handle) ); if encoded.len() >= H::LENGTH { let hash = self.db.insert(&encoded[..]); self.hash_count +=1; @@ -871,7 +876,7 @@ where } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle { + fn root_handle(&self) -> NodeHandle { match self.root_handle { NodeHandle::Hash(h) => NodeHandle::Hash(h), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), @@ -880,7 +885,7 @@ where } impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> -where +where H: Hasher, C: NodeCodec { From aa67bd5d00e48bac71ab81a384ac2902757eebed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 5 Jul 2018 17:27:48 +0200 Subject: [PATCH 3/3] A last bunch of txqueue performance optimizations (#9024) * Clear cache only when block is enacted. * Add tracing for cull. * Cull split. * Cull after creating pending block. * Add constant, remove sync::read tracing. * Reset debug. * Remove excessive tracing. * Use struct for NonceCache. * Fix build * Remove warnings. * Fix build again. --- ethcore/private-tx/src/lib.rs | 11 ++++--- ethcore/src/client/config.rs | 2 +- ethcore/src/miner/miner.rs | 31 +++++++++++------- ethcore/src/miner/pool_client.rs | 55 +++++++++++++++++++++++--------- ethcore/sync/src/api.rs | 1 - miner/src/pool/queue.rs | 26 ++++++++++++--- transaction-pool/src/pool.rs | 5 +++ 7 files changed, 94 insertions(+), 37 deletions(-) diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index 968b73be8..2034ea7fa 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -83,7 +83,7 @@ use ethcore::client::{ Client, ChainNotify, ChainRoute, ChainMessageType, ClientIoMessage, BlockId, CallContract }; use ethcore::account_provider::AccountProvider; -use ethcore::miner::{self, Miner, MinerService}; +use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; use ethcore::trace::{Tracer, VMTracer}; use rustc_hex::FromHex; use ethkey::Password; @@ -96,6 +96,9 @@ use_contract!(private, "PrivateContract", "res/private.json"); /// Initialization vector length. const INIT_VEC_LEN: usize = 16; +/// Size of nonce cache +const NONCE_CACHE_SIZE: usize = 128; + /// Configurtion for private transaction provider #[derive(Default, PartialEq, Debug, Clone)] pub struct ProviderConfig { @@ -245,7 +248,7 @@ impl Provider where { Ok(original_transaction) } - fn pool_client<'a>(&'a self, nonce_cache: &'a RwLock>) -> miner::pool_client::PoolClient<'a, Client> { + fn pool_client<'a>(&'a self, nonce_cache: &'a NonceCache) -> miner::pool_client::PoolClient<'a, Client> { let engine = self.client.engine(); let refuse_service_transactions = true; miner::pool_client::PoolClient::new( @@ -264,7 +267,7 @@ impl Provider where { /// can be replaced with a single `drain()` method instead. /// Thanks to this we also don't really need to lock the entire verification for the time of execution. fn process_queue(&self) -> Result<(), Error> { - let nonce_cache = Default::default(); + let nonce_cache = NonceCache::new(NONCE_CACHE_SIZE); let mut verification_queue = self.transactions_for_verification.lock(); let ready_transactions = verification_queue.ready_transactions(self.pool_client(&nonce_cache)); for transaction in ready_transactions { @@ -585,7 +588,7 @@ impl Importer for Arc { trace!("Validating transaction: {:?}", original_tx); // Verify with the first account available trace!("The following account will be used for verification: {:?}", validation_account); - let nonce_cache = Default::default(); + let nonce_cache = NonceCache::new(NONCE_CACHE_SIZE); self.transactions_for_verification.lock().add_transaction( original_tx, contract, diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 1045ea610..c8b931dee 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -152,7 +152,7 @@ impl Default for ClientConfig { } #[cfg(test)] mod test { - use super::{DatabaseCompactionProfile, Mode}; + use super::{DatabaseCompactionProfile}; #[test] fn test_default_compaction_profile() { diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 81ace93fe..d196dc2f0 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -14,8 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::cmp; use std::time::{Instant, Duration}; -use std::collections::{BTreeMap, BTreeSet, HashSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::sync::Arc; use ansi_term::Colour; @@ -47,7 +48,7 @@ use client::BlockId; use executive::contract_address; use header::{Header, BlockNumber}; use miner; -use miner::pool_client::{PoolClient, CachedNonceClient}; +use miner::pool_client::{PoolClient, CachedNonceClient, NonceCache}; use receipt::{Receipt, RichReceipt}; use spec::Spec; use state::State; @@ -203,7 +204,7 @@ pub struct Miner { params: RwLock, #[cfg(feature = "work-notify")] listeners: RwLock>>, - nonce_cache: RwLock>, + nonce_cache: NonceCache, gas_pricer: Mutex, options: MinerOptions, // TODO [ToDr] Arc is only required because of price updater @@ -230,6 +231,7 @@ impl Miner { let limits = options.pool_limits.clone(); let verifier_options = options.pool_verification_options.clone(); let tx_queue_strategy = options.tx_queue_strategy; + let nonce_cache_size = cmp::max(4096, limits.max_count / 4); Miner { sealing: Mutex::new(SealingWork { @@ -244,7 +246,7 @@ impl Miner { #[cfg(feature = "work-notify")] listeners: RwLock::new(vec![]), gas_pricer: Mutex::new(gas_pricer), - nonce_cache: RwLock::new(HashMap::with_capacity(1024)), + nonce_cache: NonceCache::new(nonce_cache_size), options, transaction_queue: Arc::new(TransactionQueue::new(limits, verifier_options, tx_queue_strategy)), accounts, @@ -883,7 +885,7 @@ impl miner::MinerService for Miner { let chain_info = chain.chain_info(); let from_queue = || self.transaction_queue.pending_hashes( - |sender| self.nonce_cache.read().get(sender).cloned(), + |sender| self.nonce_cache.get(sender), ); let from_pending = || { @@ -1126,14 +1128,15 @@ impl miner::MinerService for Miner { if has_new_best_block { // Clear nonce cache - self.nonce_cache.write().clear(); + self.nonce_cache.clear(); } // First update gas limit in transaction queue and minimal gas price. let gas_limit = *chain.best_block_header().gas_limit(); self.update_transaction_queue_limits(gas_limit); - // Then import all transactions... + + // Then import all transactions from retracted blocks. let client = self.pool_client(chain); { retracted @@ -1152,11 +1155,6 @@ impl miner::MinerService for Miner { }); } - if has_new_best_block { - // ...and at the end remove the old ones - self.transaction_queue.cull(client); - } - if has_new_best_block || (imported.len() > 0 && self.options.reseal_on_uncle) { // Reset `next_allowed_reseal` in case a block is imported. // Even if min_period is high, we will always attempt to create @@ -1171,6 +1169,15 @@ impl miner::MinerService for Miner { self.update_sealing(chain); } } + + if has_new_best_block { + // Make sure to cull transactions after we update sealing. + // Not culling won't lead to old transactions being added to the block + // (thanks to Ready), but culling can take significant amount of time, + // so best to leave it after we create some work for miners to prevent increased + // uncle rate. + self.transaction_queue.cull(client); + } } fn pending_state(&self, latest_block_number: BlockNumber) -> Option { diff --git a/ethcore/src/miner/pool_client.rs b/ethcore/src/miner/pool_client.rs index bcf93d375..f537a2757 100644 --- a/ethcore/src/miner/pool_client.rs +++ b/ethcore/src/miner/pool_client.rs @@ -36,10 +36,32 @@ use header::Header; use miner; use miner::service_transaction_checker::ServiceTransactionChecker; -type NoncesCache = RwLock>; +/// Cache for state nonces. +#[derive(Debug)] +pub struct NonceCache { + nonces: RwLock>, + limit: usize +} -const MAX_NONCE_CACHE_SIZE: usize = 4096; -const EXPECTED_NONCE_CACHE_SIZE: usize = 2048; +impl NonceCache { + /// Create new cache with a limit of `limit` entries. + pub fn new(limit: usize) -> Self { + NonceCache { + nonces: RwLock::new(HashMap::with_capacity(limit / 2)), + limit, + } + } + + /// Retrieve a cached nonce for given sender. + pub fn get(&self, sender: &Address) -> Option { + self.nonces.read().get(sender).cloned() + } + + /// Clear all entries from the cache. + pub fn clear(&self) { + self.nonces.write().clear(); + } +} /// Blockchain accesss for transaction pool. pub struct PoolClient<'a, C: 'a> { @@ -70,7 +92,7 @@ C: BlockInfo + CallContract, /// Creates new client given chain, nonce cache, accounts and service transaction verifier. pub fn new( chain: &'a C, - cache: &'a NoncesCache, + cache: &'a NonceCache, engine: &'a EthEngine, accounts: Option<&'a AccountProvider>, refuse_service_transactions: bool, @@ -161,7 +183,7 @@ impl<'a, C: 'a> NonceClient for PoolClient<'a, C> where pub(crate) struct CachedNonceClient<'a, C: 'a> { client: &'a C, - cache: &'a NoncesCache, + cache: &'a NonceCache, } impl<'a, C: 'a> Clone for CachedNonceClient<'a, C> { @@ -176,13 +198,14 @@ impl<'a, C: 'a> Clone for CachedNonceClient<'a, C> { impl<'a, C: 'a> fmt::Debug for CachedNonceClient<'a, C> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("CachedNonceClient") - .field("cache", &self.cache.read().len()) + .field("cache", &self.cache.nonces.read().len()) + .field("limit", &self.cache.limit) .finish() } } impl<'a, C: 'a> CachedNonceClient<'a, C> { - pub fn new(client: &'a C, cache: &'a NoncesCache) -> Self { + pub fn new(client: &'a C, cache: &'a NonceCache) -> Self { CachedNonceClient { client, cache, @@ -194,27 +217,29 @@ impl<'a, C: 'a> NonceClient for CachedNonceClient<'a, C> where C: Nonce + Sync, { fn account_nonce(&self, address: &Address) -> U256 { - if let Some(nonce) = self.cache.read().get(address) { + if let Some(nonce) = self.cache.nonces.read().get(address) { return *nonce; } // We don't check again if cache has been populated. // It's not THAT expensive to fetch the nonce from state. - let mut cache = self.cache.write(); + let mut cache = self.cache.nonces.write(); let nonce = self.client.latest_nonce(address); cache.insert(*address, nonce); - if cache.len() < MAX_NONCE_CACHE_SIZE { + if cache.len() < self.cache.limit { return nonce } + debug!(target: "txpool", "NonceCache: reached limit."); + trace_time!("nonce_cache:clear"); + // Remove excessive amount of entries from the cache - while cache.len() > EXPECTED_NONCE_CACHE_SIZE { - // Just remove random entry - if let Some(key) = cache.keys().next().cloned() { - cache.remove(&key); - } + let to_remove: Vec<_> = cache.keys().take(self.cache.limit / 2).cloned().collect(); + for x in to_remove { + cache.remove(&x); } + nonce } } diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 56bc579ad..ef54a4802 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -384,7 +384,6 @@ impl NetworkProtocolHandler for SyncProtocolHandler { } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { - trace_time!("sync::read"); ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); } diff --git a/miner/src/pool/queue.rs b/miner/src/pool/queue.rs index 40f3840d8..24a56c226 100644 --- a/miner/src/pool/queue.rs +++ b/miner/src/pool/queue.rs @@ -43,6 +43,14 @@ type Pool = txpool::Pool( + pub fn cull( &self, client: C, ) { + trace_time!("pool::cull"); // We don't care about future transactions, so nonce_cap is not important. let nonce_cap = None; // We want to clear stale transactions from the queue as well. @@ -416,10 +425,19 @@ impl TransactionQueue { current_id.checked_sub(gap) }; - let state_readiness = ready::State::new(client, stale_id, nonce_cap); - self.recently_rejected.clear(); - let removed = self.pool.write().cull(None, state_readiness); + + let mut removed = 0; + let senders: Vec<_> = { + let pool = self.pool.read(); + let senders = pool.senders().cloned().collect(); + senders + }; + for chunk in senders.chunks(CULL_SENDERS_CHUNK) { + trace_time!("pool::cull::chunk"); + let state_readiness = ready::State::new(client.clone(), stale_id, nonce_cap); + removed += self.pool.write().cull(Some(chunk), state_readiness); + } debug!(target: "txqueue", "Removed {} stalled transactions. {}", removed, self.status()); } diff --git a/transaction-pool/src/pool.rs b/transaction-pool/src/pool.rs index 6fa17e1b2..e2fa36c0e 100644 --- a/transaction-pool/src/pool.rs +++ b/transaction-pool/src/pool.rs @@ -414,6 +414,11 @@ impl Pool where || self.mem_usage >= self.options.max_mem_usage } + /// Returns senders ordered by priority of their transactions. + pub fn senders(&self) -> impl Iterator { + self.best_transactions.iter().map(|tx| tx.transaction.sender()) + } + /// Returns an iterator of pending (ready) transactions. pub fn pending>(&self, ready: R) -> PendingIterator { PendingIterator {