grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

util cleanup (#1474)

Browse Source 
* removed old json-tests

* simplify folds in triehash.rs

* removed unused json_aid

* removed unused squeeze.rs

* json branching tests for trie

* loading trie consensus tests
This commit is contained in:
Marek Kotewicz
2016-07-05 15:16:27 +02:00
committed by Gav Wood
parent 4c1b74a42e
commit 62b9c1b14f
42 changed files with 425 additions and 1738 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
util/src/json_aid.rs
View File

@@ -1,154 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use common::*;
/// Remove the `"0x"`, if present, from the left of `s`, returning the remaining slice.
pub fn clean(s: &str) -> &str {
if s.len() >= 2 && &s[0..2] == "0x" {
&s[2..]
} else {
s
}
}
fn u256_from_str(s: &str) -> U256 {
if s.len() >= 2 && &s[0..2] == "0x" {
U256::from_str(&s[2..]).unwrap_or_else(|_| U256::zero())
} else {
U256::from_dec_str(s).unwrap_or_else(|_| U256::zero())
}
}
impl FromJson for Bytes {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref s) => match s.len() % 2 {
0 => FromHex::from_hex(clean(s)).unwrap_or_else(|_| vec![]),
_ => FromHex::from_hex(&("0".to_owned() + &(clean(s).to_owned()))[..]).unwrap_or_else(|_| vec![]),
},
_ => vec![],
}
}
}
impl FromJson for BTreeMap<H256, H256> {
fn from_json(json: &Json) -> Self {
match *json {
Json::Object(ref o) => o.iter().map(|(key, value)| (u256_from_str(key).into(), U256::from_json(value).into())).collect(),
_ => BTreeMap::new(),
}
}
}
impl<T> FromJson for Vec<T> where T: FromJson {
fn from_json(json: &Json) -> Self {
match *json {
Json::Array(ref o) => o.iter().map(|x|T::from_json(x)).collect(),
_ => Vec::new(),
}
}
}
impl<T> FromJson for Option<T> where T: FromJson {
fn from_json(json: &Json) -> Self {
match *json {
Json::String(ref o) if o.is_empty() => None,
Json::Null => None,
_ => Some(FromJson::from_json(json)),
}
}
}
impl FromJson for u64 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64()
}
}
impl FromJson for u32 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64() as u32
}
}
impl FromJson for u16 {
fn from_json(json: &Json) -> Self {
U256::from_json(json).low_u64() as u16
}
}
#[test]
fn u256_from_json() {
let j = Json::from_str("{ \"dec\": \"10\", \"hex\": \"0x0a\", \"int\": 10 }").unwrap();
let v: U256 = xjson!(&j["dec"]);
assert_eq!(U256::from(10), v);
let v: U256 = xjson!(&j["hex"]);
assert_eq!(U256::from(10), v);
let v: U256 = xjson!(&j["int"]);
assert_eq!(U256::from(10), v);
}
#[test]
fn h256_from_json() {
let j = Json::from_str("{ \"with\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"without\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\" }").unwrap();
let v: H256 = xjson!(&j["with"]);
assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v);
let v: H256 = xjson!(&j["without"]);
assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v);
}
#[test]
fn vec_u256_from_json() {
let j = Json::from_str("{ \"array\": [ \"10\", \"0x0a\", 10] }").unwrap();
let v: Vec<U256> = xjson!(&j["array"]);
assert_eq!(vec![U256::from(10); 3], v);
}
#[test]
fn vec_h256_from_json() {
let j = Json::from_str("{ \"array\": [ \"1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\"] }").unwrap();
let v: Vec<H256> = xjson!(&j["array"]);
assert_eq!(vec![H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(); 2], v);
}
#[test]
fn simple_types() {
let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap();
let v: u16 = xjson!(&j["int"]);
assert_eq!(42u16, v);
let v: u32 = xjson!(&j["dec"]);
assert_eq!(42u32, v);
let v: u64 = xjson!(&j["hex"]);
assert_eq!(42u64, v);
}
#[test]
fn option_types() {
let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap();
let v: Option<u16> = xjson!(&j["int"]);
assert_eq!(Some(42u16), v);
let v: Option<u16> = xjson!(&j["dec"]);
assert_eq!(Some(42u16), v);
let v: Option<u16> = xjson!(&j["null"]);
assert_eq!(None, v);
let v: Option<u16> = xjson!(&j["empty"]);
assert_eq!(None, v);
}

4
util/src/lib.rs
View File

@@ -131,7 +131,6 @@ pub mod bytes;
pub mod rlp;
pub mod misc;
pub mod using_queue;
mod json_aid;
pub mod vector;
pub mod sha3;
pub mod hashdb;
@@ -147,7 +146,6 @@ pub mod trie;
pub mod nibbleslice;
pub mod nibblevec;
mod heapsizeof;
pub mod squeeze;
pub mod semantic_version;
pub mod io;
pub mod network;
@@ -161,7 +159,6 @@ mod timer;
pub use common::*;
pub use misc::*;
pub use using_queue::*;
pub use json_aid::*;
pub use rlp::*;
pub use hashdb::*;
pub use memorydb::*;
@@ -172,7 +169,6 @@ pub use crypto::*;
pub use triehash::*;
pub use trie::*;
pub use nibbleslice::*;
pub use squeeze::*;
pub use semantic_version::*;
pub use network::*;
pub use io::*;

23
util/src/rlp/tests.rs
View File

@@ -14,9 +14,6 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate json_tests;
use self::json_tests::execute_tests_from_directory;
use self::json_tests::rlp as rlptest;
use std::{fmt, cmp};
use std::str::FromStr;
use rlp;
@@ -340,26 +337,6 @@ fn decode_untrusted_vector_of_vectors_str() {
run_decode_tests(tests);
}
#[test]
fn test_rlp_json() {
println!("Json rlp test: ");
execute_tests_from_directory::<rlptest::RlpStreamTest, _>("json-tests/json/rlp/stream/*.json", &mut | file, input, output | {
println!("file: {}", file);
let mut stream = RlpStream::new();
for operation in input.into_iter() {
match operation {
rlptest::Operation::Append(ref v) => stream.append(v),
rlptest::Operation::AppendList(len) => stream.begin_list(len),
rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len),
rlptest::Operation::AppendEmpty => stream.append_empty_data()
};
}
assert_eq!(stream.out(), output);
});
}
#[test]
fn test_decoding_array() {
let v = vec![5u16, 2u16];

84
util/src/squeeze.rs
View File

@@ -1,84 +0,0 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Helper module that should be used to randomly squeeze
//! caches to a given size in bytes
//!
//! ```
//! extern crate heapsize;
//! extern crate ethcore_util as util;
//! use std::collections::HashMap;
//! use std::mem::size_of;
//! use heapsize::HeapSizeOf;
//! use util::squeeze::Squeeze;
//!
//! fn main() {
//! let initial_size = 60;
//! let mut map: HashMap<u8, u8> = HashMap::with_capacity(initial_size);
//! assert!(map.capacity() >= initial_size);
//! for i in 0..initial_size {
//! map.insert(i as u8, i as u8);
//! }
//!
//! assert_eq!(map.heap_size_of_children(), map.capacity() * 2 * size_of::<u8>());
//! assert_eq!(map.len(), initial_size);
//! let initial_heap_size = map.heap_size_of_children();
//!
//! // squeeze it to size of key and value
//! map.squeeze(2 * size_of::<u8>());
//! assert_eq!(map.len(), 1);
//!
//! // its likely that heap size was reduced, but we can't be 100% sure
//! assert!(initial_heap_size >= map.heap_size_of_children());
//! }
//! ```
use std::collections::HashMap;
use std::hash::Hash;
use heapsize::HeapSizeOf;
/// Should be used to squeeze collections to certain size in bytes
pub trait Squeeze {
/// Try to reduce collection size to `size` bytes
fn squeeze(&mut self, size: usize);
}
impl<K, T> Squeeze for HashMap<K, T> where K: Eq + Hash + Clone + HeapSizeOf, T: HeapSizeOf {
fn squeeze(&mut self, size: usize) {
if self.is_empty() {
return
}
let size_of_entry = self.heap_size_of_children() / self.capacity();
let all_entries = size_of_entry * self.len();
let mut shrinked_size = all_entries;
while !self.is_empty() && shrinked_size > size {
// could be optimized
let key = self.keys().next().unwrap().clone();
self.remove(&key);
shrinked_size -= size_of_entry;
}
self.shrink_to_fit();
// if we squeezed something, but not enough, squeeze again
if all_entries != shrinked_size && self.heap_size_of_children() > size {
self.squeeze(size);
}
}
}

90
util/src/trie/triedbmut.rs
View File

@@ -676,8 +676,6 @@ impl<'db> fmt::Debug for TrieDBMut<'db> {
#[cfg(test)]
mod tests {
extern crate json_tests;
use self::json_tests::{trie, execute_tests_from_directory};
use triehash::*;
use hash::*;
use hashdb::*;
@@ -858,6 +856,21 @@ mod tests {
]));
}
#[test]
fn insert_out_of_order() {
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]);
assert_eq!(*t.root(), trie_root(vec![
(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
(vec![0x81u8, 0x23], vec![0x81u8, 0x23]),
(vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]),
]));
}
#[test]
fn insert_value_into_branch_root() {
let mut memdb = MemoryDB::new();
@@ -1063,23 +1076,64 @@ mod tests {
}
#[test]
fn test_trie_json() {
println!("Json trie test: ");
execute_tests_from_directory::<trie::TrieTest, _>("json-tests/json/trie/*.json", &mut | file, input, output | {
println!("file: {}", file);
fn branching_test() {
use std::str::FromStr;
use rustc_serialize::hex::FromHex;
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
for operation in input.into_iter() {
match operation {
trie::Operation::Insert(key, value) => t.insert(&key, &value),
trie::Operation::Remove(key) => t.remove(&key)
}
}
assert_eq!(*t.root(), H256::from_slice(&output));
});
let mut memdb = MemoryDB::new();
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&"04110d816c380812a427968ece99b1c963dfbce6".from_hex().unwrap(), b"something");
t.insert(&"095e7baea6a6c7c4c2dfeb977efac326af552d87".from_hex().unwrap(), b"something");
t.insert(&"0a517d755cebbf66312b30fff713666a9cb917e0".from_hex().unwrap(), b"something");
t.insert(&"24dd378f51adc67a50e339e8031fe9bd4aafab36".from_hex().unwrap(), b"something");
t.insert(&"293f982d000532a7861ab122bdc4bbfd26bf9030".from_hex().unwrap(), b"something");
t.insert(&"2cf5732f017b0cf1b1f13a1478e10239716bf6b5".from_hex().unwrap(), b"something");
t.insert(&"31c640b92c21a1f1465c91070b4b3b4d6854195f".from_hex().unwrap(), b"something");
t.insert(&"37f998764813b136ddf5a754f34063fd03065e36".from_hex().unwrap(), b"something");
t.insert(&"37fa399a749c121f8a15ce77e3d9f9bec8020d7a".from_hex().unwrap(), b"something");
t.insert(&"4f36659fa632310b6ec438dea4085b522a2dd077".from_hex().unwrap(), b"something");
t.insert(&"62c01474f089b07dae603491675dc5b5748f7049".from_hex().unwrap(), b"something");
t.insert(&"729af7294be595a0efd7d891c9e51f89c07950c7".from_hex().unwrap(), b"something");
t.insert(&"83e3e5a16d3b696a0314b30b2534804dd5e11197".from_hex().unwrap(), b"something");
t.insert(&"8703df2417e0d7c59d063caa9583cb10a4d20532".from_hex().unwrap(), b"something");
t.insert(&"8dffcd74e5b5923512916c6a64b502689cfa65e1".from_hex().unwrap(), b"something");
t.insert(&"95a4d7cccb5204733874fa87285a176fe1e9e240".from_hex().unwrap(), b"something");
t.insert(&"99b2fcba8120bedd048fe79f5262a6690ed38c39".from_hex().unwrap(), b"something");
t.insert(&"a4202b8b8afd5354e3e40a219bdc17f6001bf2cf".from_hex().unwrap(), b"something");
t.insert(&"a94f5374fce5edbc8e2a8697c15331677e6ebf0b".from_hex().unwrap(), b"something");
t.insert(&"a9647f4a0a14042d91dc33c0328030a7157c93ae".from_hex().unwrap(), b"something");
t.insert(&"aa6cffe5185732689c18f37a7f86170cb7304c2a".from_hex().unwrap(), b"something");
t.insert(&"aae4a2e3c51c04606dcb3723456e58f3ed214f45".from_hex().unwrap(), b"something");
t.insert(&"c37a43e940dfb5baf581a0b82b351d48305fc885".from_hex().unwrap(), b"something");
t.insert(&"d2571607e241ecf590ed94b12d87c94babe36db6".from_hex().unwrap(), b"something");
t.insert(&"f735071cbee190d76b704ce68384fc21e389fbe7".from_hex().unwrap(), b"something");
t.insert(&"04110d816c380812a427968ece99b1c963dfbce6".from_hex().unwrap(), &[]);
t.insert(&"095e7baea6a6c7c4c2dfeb977efac326af552d87".from_hex().unwrap(), &[]);
t.insert(&"0a517d755cebbf66312b30fff713666a9cb917e0".from_hex().unwrap(), &[]);
t.insert(&"24dd378f51adc67a50e339e8031fe9bd4aafab36".from_hex().unwrap(), &[]);
t.insert(&"293f982d000532a7861ab122bdc4bbfd26bf9030".from_hex().unwrap(), &[]);
t.insert(&"2cf5732f017b0cf1b1f13a1478e10239716bf6b5".from_hex().unwrap(), &[]);
t.insert(&"31c640b92c21a1f1465c91070b4b3b4d6854195f".from_hex().unwrap(), &[]);
t.insert(&"37f998764813b136ddf5a754f34063fd03065e36".from_hex().unwrap(), &[]);
t.insert(&"37fa399a749c121f8a15ce77e3d9f9bec8020d7a".from_hex().unwrap(), &[]);
t.insert(&"4f36659fa632310b6ec438dea4085b522a2dd077".from_hex().unwrap(), &[]);
t.insert(&"62c01474f089b07dae603491675dc5b5748f7049".from_hex().unwrap(), &[]);
t.insert(&"729af7294be595a0efd7d891c9e51f89c07950c7".from_hex().unwrap(), &[]);
t.insert(&"83e3e5a16d3b696a0314b30b2534804dd5e11197".from_hex().unwrap(), &[]);
t.insert(&"8703df2417e0d7c59d063caa9583cb10a4d20532".from_hex().unwrap(), &[]);
t.insert(&"8dffcd74e5b5923512916c6a64b502689cfa65e1".from_hex().unwrap(), &[]);
t.insert(&"95a4d7cccb5204733874fa87285a176fe1e9e240".from_hex().unwrap(), &[]);
t.insert(&"99b2fcba8120bedd048fe79f5262a6690ed38c39".from_hex().unwrap(), &[]);
t.insert(&"a4202b8b8afd5354e3e40a219bdc17f6001bf2cf".from_hex().unwrap(), &[]);
t.insert(&"a94f5374fce5edbc8e2a8697c15331677e6ebf0b".from_hex().unwrap(), &[]);
t.insert(&"a9647f4a0a14042d91dc33c0328030a7157c93ae".from_hex().unwrap(), &[]);
t.insert(&"aa6cffe5185732689c18f37a7f86170cb7304c2a".from_hex().unwrap(), &[]);
t.insert(&"aae4a2e3c51c04606dcb3723456e58f3ed214f45".from_hex().unwrap(), &[]);
t.insert(&"c37a43e940dfb5baf581a0b82b351d48305fc885".from_hex().unwrap(), &[]);
t.insert(&"d2571607e241ecf590ed94b12d87c94babe36db6".from_hex().unwrap(), &[]);
t.insert(&"f735071cbee190d76b704ce68384fc21e389fbe7".from_hex().unwrap(), &[]);
assert_eq!(*t.root(), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap());
}
#[test]

37
util/src/triehash.rs
View File

@@ -46,7 +46,8 @@ pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
// optimize it later
.into_iter()
.enumerate()
.fold(BTreeMap::new(), | mut acc, (i, vec) | { acc.insert(rlp::encode(&i).to_vec(), vec); acc })
.map(|(i, vec)| (rlp::encode(&i).to_vec(), vec))
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -78,10 +79,7 @@ pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
let gen_input = input
// first put elements into btree to sort them and to remove duplicates
.into_iter()
.fold(BTreeMap::new(), | mut acc, (k, v) | {
acc.insert(k, v);
acc
})
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -97,7 +95,7 @@ pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
/// use std::str::FromStr;
/// use util::triehash::*;
/// use util::hash::*;
///
///
/// fn main() {
/// let v = vec![
/// (From::from("doe"), From::from("reindeer")),
@@ -113,10 +111,8 @@ pub fn sec_trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
let gen_input = input
// first put elements into btree to sort them and to remove duplicates
.into_iter()
.fold(BTreeMap::new(), | mut acc, (k, v) | {
acc.insert(k.sha3().to_vec(), v);
acc
})
.map(|(k, v)| (k.sha3().to_vec(), v))
.collect::<BTreeMap<_, _>>()
// then move them to a vector
.into_iter()
.map(|(k, v)| (as_nibbles(&k), v) )
@@ -324,10 +320,16 @@ fn test_hex_prefix_encode() {
#[cfg(test)]
mod tests {
extern crate json_tests;
use self::json_tests::*;
use hash::*;
use triehash::*;
use std::str::FromStr;
use hash::H256;
use super::trie_root;
#[test]
fn simple_test() {
assert_eq!(trie_root(vec![
(b"A".to_vec(), b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_vec())
]), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap());
}
#[test]
fn test_triehash_out_of_order() {
@@ -343,11 +345,4 @@ mod tests {
]));
}
#[test]
fn test_triehash_json() {
execute_tests_from_directory::<trie::TriehashTest, _>("json-tests/json/trie/*.json", &mut | file, input, output | {
println!("file: {}, output: {:?}", file, output);
assert_eq!(trie_root(input), H256::from_slice(&output));
});
}
}