From ca1a6bd7912e77901c74abc9ebf45e875dd53d31 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:23:08 +0100 Subject: [PATCH 1/3] heapsize && squeeze --- Cargo.toml | 1 + src/heapsizeof.rs | 5 +++++ src/lib.rs | 6 ++++++ src/squeeze.rs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 src/heapsizeof.rs create mode 100644 src/squeeze.rs diff --git a/Cargo.toml b/Cargo.toml index 13295f766..4d23f49f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" elastic-array = "0.4" +heapsize = "0.2" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/heapsizeof.rs b/src/heapsizeof.rs new file mode 100644 index 000000000..c6d4cace4 --- /dev/null +++ b/src/heapsizeof.rs @@ -0,0 +1,5 @@ +use uint::*; +use hash::*; + +known_heap_size!(0, H32, H64, H128, Address, H256, H264, H512, H520, H1024, H2048); +known_heap_size!(0, U128, U256); diff --git a/src/lib.rs b/src/lib.rs index 24b404a88..9120f0977 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,10 +32,14 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; + +#[macro_use] +extern crate heapsize; #[macro_use] extern crate log; #[macro_use] extern crate lazy_static; + extern crate env_logger; extern crate time; @@ -60,5 +64,7 @@ pub mod crypto; pub mod triehash; pub mod trie; pub mod nibbleslice; +pub mod heapsizeof; +pub mod squeeze; //pub mod network; diff --git a/src/squeeze.rs b/src/squeeze.rs new file mode 100644 index 000000000..220c61800 --- /dev/null +++ b/src/squeeze.rs @@ -0,0 +1,33 @@ +use std::collections::HashMap; +use std::hash::Hash; +use heapsize::HeapSizeOf; + +/// Should be used to squeeze collections to certain size in bytes +trait Squeeze { + fn squeeze(&mut self, size: usize); +} + +impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: HeapSizeOf { + fn squeeze(&mut self, size: usize) { + if self.len() == 0 { + return + } + + let size_of_entry = self.heap_size_of_children() / self.capacity(); + let mut shrinked_size = size_of_entry * self.len(); + + while self.len() > 0 || shrinked_size > size { + // could be optimized + let key = self.keys().next().unwrap().clone(); + self.remove(&key); + shrinked_size -= size_of_entry; + } + + self.shrink_to_fit(); + + // if we havent shrinked enough, squeeze again + if self.heap_size_of_children() > size { + self.squeeze(size); + } + } +} From 56de081381f18e35bcc10c38858f13cd0ffc9768 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:27:27 +0100 Subject: [PATCH 2/3] squeeze is pub --- src/squeeze.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index 220c61800..ddec2a3a9 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use heapsize::HeapSizeOf; /// Should be used to squeeze collections to certain size in bytes -trait Squeeze { +pub trait Squeeze { fn squeeze(&mut self, size: usize); } From 926efb6fc8c5eff915201bb01fe7f8435e4efc2f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 17:12:20 +0100 Subject: [PATCH 3/3] docs, tests and bugfixes for squeeze --- src/squeeze.rs | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index ddec2a3a9..e81a13793 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -1,3 +1,35 @@ +//! Helper module that should be used to randomly squeeze +//! caches to a given size in bytes +//! +//! ``` +//! extern crate heapsize; +//! extern crate ethcore_util as util; +//! use std::collections::HashMap; +//! use std::mem::size_of; +//! use heapsize::HeapSizeOf; +//! use util::squeeze::Squeeze; +//! +//! fn main() { +//! let initial_size = 60; +//! let mut map: HashMap = HashMap::with_capacity(initial_size); +//! assert!(map.capacity() >= initial_size); +//! for i in 0..initial_size { +//! map.insert(i as u8, i as u8); +//! } +//! +//! assert_eq!(map.heap_size_of_children(), map.capacity() * 2 * size_of::()); +//! assert_eq!(map.len(), initial_size); +//! let initial_heap_size = map.heap_size_of_children(); +//! +//! // squeeze it to size of key and value +//! map.squeeze(2 * size_of::()); +//! assert_eq!(map.len(), 1); +//! +//! // its likely that heap size was reduced, but we can't be 100% sure +//! assert!(initial_heap_size >= map.heap_size_of_children()); +//! } +//! ``` + use std::collections::HashMap; use std::hash::Hash; use heapsize::HeapSizeOf; @@ -14,9 +46,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: } let size_of_entry = self.heap_size_of_children() / self.capacity(); - let mut shrinked_size = size_of_entry * self.len(); + let all_entries = size_of_entry * self.len(); + let mut shrinked_size = all_entries; - while self.len() > 0 || shrinked_size > size { + while self.len() > 0 && shrinked_size > size { // could be optimized let key = self.keys().next().unwrap().clone(); self.remove(&key); @@ -25,9 +58,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: self.shrink_to_fit(); - // if we havent shrinked enough, squeeze again - if self.heap_size_of_children() > size { + // if we squeezed something, but not enough, squeeze again + if all_entries != shrinked_size && self.heap_size_of_children() > size { self.squeeze(size); } } } +