From 802d68499449551b6cc8d63002eb26ac9cf273f3 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 5 Jul 2018 17:15:03 +0200 Subject: [PATCH] reduce number of constraints for triedb types (#9043) * NodeHandle does not require Hasher * Node does not require Hasher * change name of the template typo from I to O --- Cargo.lock | 1 + util/hashdb/src/lib.rs | 2 +- util/patricia_trie/Cargo.toml | 5 +- util/patricia_trie/src/lib.rs | 5 +- util/patricia_trie/src/triedbmut.rs | 85 +++++++++++++++-------------- 5 files changed, 53 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96f58d541..7f20bcbb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2485,6 +2485,7 @@ dependencies = [ "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "keccak-hasher 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 5961d90f9..4f5bed48c 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -23,7 +23,7 @@ use heapsize::HeapSizeOf; use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; -/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// Trait describing an object that can hash a slice of bytes. Used to abstract /// other types over the hashing algorithm. Defines a single `hash` method and an /// `Out` associated type with the necessary bounds. pub trait Hasher: Sync + Send { diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 532465678..ebd665045 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -7,10 +7,11 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" +ethcore-bytes = { version = "0.1.0", path = "../bytes" } +hashdb = { version = "0.2", path = "../hashdb" } +heapsize = "0.4" log = "0.3" rand = "0.4" -hashdb = { version = "0.2", path = "../hashdb" } -ethcore-bytes = { version = "0.1.0", path = "../bytes" } [dev-dependencies] env_logger = "0.5" diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 7cc623b1a..a028be87a 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -18,6 +18,7 @@ extern crate elastic_array; extern crate ethcore_bytes as bytes; extern crate hashdb; +extern crate heapsize; extern crate rand; #[macro_use] extern crate log; @@ -277,8 +278,8 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { } impl<'db, H, C> TrieFactory -where - H: Hasher, +where + H: Hasher, C: NodeCodec + 'db { /// Creates new factory. diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index f5c28bac3..4490285d5 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,6 +31,8 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; +use heapsize::HeapSizeOf; +use std::{fmt::Debug, hash::Hash}; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -39,20 +41,20 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H::Out), + Hash(H), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -fn empty_children() -> Box<[Option>; 16]> { +fn empty_children() -> Box<[Option>; 16]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, @@ -61,7 +63,7 @@ fn empty_children() -> Box<[Option>; 16]> { /// Node types in the Trie. #[derive(Debug)] -enum Node { +enum Node { /// Empty node. Empty, /// A leaf node contains the end of a key and a value. @@ -77,22 +79,24 @@ enum Node { Branch(Box<[Option>; 16]>, Option) } -impl Node { +impl Node where O: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle - where C: NodeCodec + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + where C: NodeCodec, + H: Hasher, { C::try_decode_hash(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { - let child = Node::from_encoded::(node, db, storage); + let child = Node::from_encoded::(node, db, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) }) } // decode a node from encoded bytes without getting its children. - fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self - where C: NodeCodec + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self + where C: NodeCodec, + H: Hasher, { match C::decode(data).expect("encoded bytes read from db; qed") { EncodedNode::Empty => Node::Empty, @@ -100,13 +104,13 @@ impl Node { EncodedNode::Extension(key, cb) => { Node::Extension( key.encoded(false), - Self::inline_or_hash::(cb, db, storage)) + Self::inline_or_hash::(cb, db, storage)) } EncodedNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; if !C::is_empty_node(raw) { - Some(Self::inline_or_hash::(raw, db, storage)) + Some(Self::inline_or_hash::(raw, db, storage)) } else { None } @@ -125,10 +129,11 @@ impl Node { } // TODO: parallelize - fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 + fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, - F: FnMut(NodeHandle) -> ChildReference + F: FnMut(NodeHandle) -> ChildReference, + H: Hasher, { match self { Node::Empty => C::empty_node(), @@ -139,7 +144,7 @@ impl Node { // map the `NodeHandle`s from the Branch to `ChildReferences` children.iter_mut() .map(Option::take) - .map(|maybe_child| + .map(|maybe_child| maybe_child.map(|child| child_cb(child)) ), value @@ -150,7 +155,7 @@ impl Node { } // post-inspect action. -enum Action { +enum Action { // Replace a node with a new one. Replace(Node), // Restore the original node. This trusts that the node is actually the original. @@ -160,14 +165,14 @@ enum Action { } // post-insert action. Same as action without delete -enum InsertAction { +enum InsertAction { // Replace a node with a new one. Replace(Node), // Restore the original node. Restore(Node), } -impl InsertAction { +impl InsertAction { fn into_action(self) -> Action { match self { InsertAction::Replace(n) => Action::Replace(n), @@ -184,11 +189,11 @@ impl InsertAction { } // What kind of node is stored here. -enum Stored { +enum Stored { // A new node. New(Node), // A cached node, loaded from the DB. - Cached(Node, H::Out), + Cached(Node, H), } /// Used to build a collection of child nodes from a collection of `NodeHandle`s @@ -198,12 +203,12 @@ pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hash } /// Compact and cache-friendly storage for Trie nodes. -struct NodeStorage { +struct NodeStorage { nodes: Vec>, free_indices: VecDeque, } -impl NodeStorage { +impl NodeStorage { /// Create a new storage. fn empty() -> Self { NodeStorage { @@ -232,7 +237,7 @@ impl NodeStorage { } } -impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { +impl<'a, H> Index<&'a StorageHandle> for NodeStorage { type Output = Node; fn index(&self, handle: &'a StorageHandle) -> &Node { @@ -284,10 +289,10 @@ where H: Hasher + 'a, C: NodeCodec { - storage: NodeStorage, + storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, - root_handle: NodeHandle, + root_handle: NodeHandle, death_row: HashSet, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. @@ -347,7 +352,7 @@ where // cache a node by hash fn cache(&mut self, hash: H::Out) -> Result { let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_encoded::( + let node = Node::from_encoded::( &node_encoded, &*self.db, &mut self.storage @@ -357,8 +362,8 @@ where // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> - where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> + where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -380,7 +385,7 @@ where } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> where 'x: 'key { let mut handle = handle; @@ -429,7 +434,7 @@ where } /// insert a key-value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -443,7 +448,7 @@ where } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -604,7 +609,7 @@ where } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -619,7 +624,7 @@ where } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -705,7 +710,7 @@ where /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { + fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -825,7 +830,7 @@ where match self.storage.destroy(handle) { Stored::New(node) => { - let encoded_root = node.into_encoded::<_, C>(|child| self.commit_child(child) ); + let encoded_root = node.into_encoded::<_, C, H>(|child| self.commit_child(child) ); *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; @@ -845,14 +850,14 @@ where /// case where we can fit the actual data in the `Hasher`s output type, we /// store the data inline. This function is used as the callback to the /// `into_encoded` method of `Node`. - fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { + fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { match handle { NodeHandle::Hash(hash) => ChildReference::Hash(hash), NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { - let encoded = node.into_encoded::<_, C>(|node_handle| self.commit_child(node_handle) ); + let encoded = node.into_encoded::<_, C, H>(|node_handle| self.commit_child(node_handle) ); if encoded.len() >= H::LENGTH { let hash = self.db.insert(&encoded[..]); self.hash_count +=1; @@ -871,7 +876,7 @@ where } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle { + fn root_handle(&self) -> NodeHandle { match self.root_handle { NodeHandle::Hash(h) => NodeHandle::Hash(h), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), @@ -880,7 +885,7 @@ where } impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> -where +where H: Hasher, C: NodeCodec {