From 1cb6cc0b87a830e283226c761ed13036d24fc349 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Dec 2015 18:42:51 +0100 Subject: [PATCH 01/47] Stress-tests for trie. All passing. --- src/trie.rs | 135 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 9b31545db..81ac4f99b 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -801,28 +801,86 @@ mod tests { use rlp; use env_logger; use rand::random; - use bytes::ToPretty; + use std::collections::HashSet; + use bytes::{ToPretty,Bytes}; + + fn random_key() -> Vec { + let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + let mut ret: Vec = Vec::new(); + let r = random::() % 4 + 1; + for _ in 0..r { + ret.push(chars[random::() % chars.len()]); + } + ret + } + + fn random_value(i: usize) -> Bytes { + match random::() % 2 { + 0 => rlp::encode(&i), + _ => { + let mut h = H256::new(); + h.mut_bytes()[31] = i as u8; + rlp::encode(&h) + }, + } + } + + fn populate_trie(v: &Vec<(Vec, Vec)>) -> TrieDB { + let mut t = TrieDB::new_memory(); + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + t + } + + fn unpopulate_trie(t: &mut TrieDB, v: &Vec<(Vec, Vec)>) { + for i in v.iter() { + let key: &[u8]= &i.0; + t.remove(&key); + } + } #[test] fn playpen() { env_logger::init().ok(); + for _ in 0..1000 { + let mut x: Vec<(Vec, Vec)> = Vec::new(); + let mut got: HashSet> = HashSet::new(); + for j in 0..10usize { + let key = random_key(); + if !got.contains(&key) { + x.push((key.clone(), random_value(j))); + got.insert(key); + } + } - let big_value = b"00000000000000000000000000000000"; - - let mut t1 = TrieDB::new_memory(); - t1.insert(&[0x01, 0x23], &big_value.to_vec()); - t1.insert(&[0x01, 0x34], &big_value.to_vec()); - trace!("keys remaining {:?}", t1.db_items_remaining()); - assert!(t1.db_items_remaining().is_empty()); - let mut t2 = TrieDB::new_memory(); - t2.insert(&[0x01], &big_value.to_vec()); - t2.insert(&[0x01, 0x23], &big_value.to_vec()); - t2.insert(&[0x01, 0x34], &big_value.to_vec()); - t2.remove(&[0x01]); - assert!(t2.db_items_remaining().is_empty()); - /*if t1.root() != t2.root()*/ { - trace!("{:?}", t1); - trace!("{:?}", t2); + let real = trie_root(x.clone()); + let mut memtrie = populate_trie(&x); + if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in x.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie); + } + assert_eq!(*memtrie.root(), real); + assert!(memtrie.db_items_remaining().is_empty()); + unpopulate_trie(&mut memtrie, &x); + if *memtrie.root() != SHA3_NULL_RLP || !memtrie.db_items_remaining().is_empty() { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in x.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie); + } + assert_eq!(*memtrie.root(), SHA3_NULL_RLP); + assert!(memtrie.db_items_remaining().is_empty()); } } @@ -842,6 +900,23 @@ mod tests { #[test] fn remove_to_empty() { + let big_value = b"00000000000000000000000000000000"; + + let mut t1 = TrieDB::new_memory(); + t1.insert(&[0x01, 0x23], &big_value.to_vec()); + t1.insert(&[0x01, 0x34], &big_value.to_vec()); + trace!("keys remaining {:?}", t1.db_items_remaining()); + assert!(t1.db_items_remaining().is_empty()); + let mut t2 = TrieDB::new_memory(); + t2.insert(&[0x01], &big_value.to_vec()); + t2.insert(&[0x01, 0x23], &big_value.to_vec()); + t2.insert(&[0x01, 0x34], &big_value.to_vec()); + t2.remove(&[0x01]); + assert!(t2.db_items_remaining().is_empty()); + /*if t1.root() != t2.root()*/ { + trace!("{:?}", t1); + trace!("{:?}", t2); + } } #[test] @@ -1022,16 +1097,6 @@ mod tests { //assert!(false); } - fn random_key() -> Vec { - let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; - let mut ret: Vec = Vec::new(); - let r = random::() % 4 + 1; - for _ in 0..r { - ret.push(chars[random::() % chars.len()]); - } - ret - } - #[test] fn stress() { for _ in 0..5000 { @@ -1041,10 +1106,10 @@ mod tests { x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); - let memtrie = trie_root_mem(&x); + let memtrie = populate_trie(&x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let memtrie_sorted = trie_root_mem(&y); + let memtrie_sorted = populate_trie(&y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1064,18 +1129,6 @@ mod tests { } } - fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> TrieDB { - let mut t = TrieDB::new_memory(); - - for i in 0..v.len() { - let key: &[u8]= &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(&key, &val); - } - - t - } - #[test] fn test_trie_json() { println!("Json trie test: "); From 40f5bb6da27384e954e3c74fa473b3af63234c47 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 03:02:53 +0100 Subject: [PATCH 02/47] cleanedup uint.rs, converted spaces to tabs --- src/uint.rs | 937 +++++++++++++++++++++++----------------------------- 1 file changed, 412 insertions(+), 525 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 693b9f471..e017de385 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -22,452 +22,339 @@ ///! use std::fmt; -use std::cmp::{Ord, PartialOrd, Ordering}; +use std::cmp::*; use std::ops::*; use std::str::FromStr; use rustc_serialize::hex::{FromHex, FromHexError}; macro_rules! impl_map_from { - ($thing:ident, $from:ty, $to:ty) => { - impl From<$from> for $thing { - fn from(value: $from) -> $thing { - From::from(value as $to) - } - } - } -} - -macro_rules! impl_array_newtype { - ($thing:ident, $ty:ty, $len:expr) => { - impl $thing { - #[inline] - /// Converts the object to a raw pointer - pub fn as_ptr(&self) -> *const $ty { - let &$thing(ref dat) = self; - dat.as_ptr() - } - - #[inline] - /// Converts the object to a mutable raw pointer - pub fn as_mut_ptr(&mut self) -> *mut $ty { - let &mut $thing(ref mut dat) = self; - dat.as_mut_ptr() - } - - #[inline] - /// Returns the length of the object as an array - pub fn len(&self) -> usize { $len } - - #[inline] - /// Returns whether the object, as an array, is empty. Always false. - pub fn is_empty(&self) -> bool { false } - } - - impl<'a> From<&'a [$ty]> for $thing { - fn from(data: &'a [$ty]) -> $thing { - assert_eq!(data.len(), $len); - unsafe { - use std::intrinsics::copy_nonoverlapping; - use std::mem; - let mut ret: $thing = mem::uninitialized(); - copy_nonoverlapping(data.as_ptr(), - ret.as_mut_ptr(), - mem::size_of::<$thing>()); - ret - } - } - } - - impl Index for $thing { - type Output = $ty; - - #[inline] - fn index(&self, index: usize) -> &$ty { - let &$thing(ref dat) = self; - &dat[index] - } - } - - impl_index_newtype!($thing, $ty); - - impl PartialEq for $thing { - #[inline] - fn eq(&self, other: &$thing) -> bool { - &self[..] == &other[..] - } - } - - impl Eq for $thing {} - - impl Clone for $thing { - #[inline] - fn clone(&self) -> $thing { - $thing::from(&self[..]) - } - } - - impl Copy for $thing {} - } -} - -macro_rules! impl_index_newtype { - ($thing:ident, $ty:ty) => { - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: Range) -> &[$ty] { - &self.0[index] - } - } - - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: RangeTo) -> &[$ty] { - &self.0[index] - } - } - - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: RangeFrom) -> &[$ty] { - &self.0[index] - } - } - - impl Index for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, _: RangeFull) -> &[$ty] { - &self.0[..] - } - } - } + ($thing:ident, $from:ty, $to:ty) => { + impl From<$from> for $thing { + fn from(value: $from) -> $thing { + From::from(value as $to) + } + } + } } macro_rules! construct_uint { - ($name:ident, $n_words:expr) => ( - /// Little-endian large integer type - pub struct $name(pub [u64; $n_words]); - impl_array_newtype!($name, u64, $n_words); + ($name:ident, $n_words:expr) => ( + /// Little-endian large integer type + #[derive(Copy, Clone, Eq, PartialEq)] + pub struct $name(pub [u64; $n_words]); - impl $name { - /// Conversion to u32 - #[inline] - fn low_u32(&self) -> u32 { - let &$name(ref arr) = self; - arr[0] as u32 - } + impl $name { + /// Conversion to u32 + #[inline] + fn low_u32(&self) -> u32 { + let &$name(ref arr) = self; + arr[0] as u32 + } - /// Return the least number of bits needed to represent the number - #[inline] - pub fn bits(&self) -> usize { - let &$name(ref arr) = self; - for i in 1..$n_words { - if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } - } - 0x40 - arr[0].leading_zeros() as usize - } + /// Return the least number of bits needed to represent the number + #[inline] + pub fn bits(&self) -> usize { + let &$name(ref arr) = self; + for i in 1..$n_words { + if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } + } + 0x40 - arr[0].leading_zeros() as usize + } - #[inline] - pub fn bit(&self, index: usize) -> bool { - let &$name(ref arr) = self; - arr[index / 64] & (1 << (index % 64)) != 0 - } + #[inline] + pub fn bit(&self, index: usize) -> bool { + let &$name(ref arr) = self; + arr[index / 64] & (1 << (index % 64)) != 0 + } - #[inline] - pub fn byte(&self, index: usize) -> u8 { - let &$name(ref arr) = self; - (arr[index / 8] >> ((index % 8)) * 8) as u8 - } + #[inline] + pub fn byte(&self, index: usize) -> u8 { + let &$name(ref arr) = self; + (arr[index / 8] >> ((index % 8)) * 8) as u8 + } - pub fn to_bytes(&self, bytes: &mut[u8]) { - assert!($n_words * 8 == bytes.len()); - let &$name(ref arr) = self; - for i in 0..bytes.len() { - let rev = bytes.len() - 1 - i; - let pos = rev / 8; - bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; - } - } - /// Multiplication by u32 - fn mul_u32(self, other: u32) -> $name { - let $name(ref arr) = self; - let mut carry = [0u64; $n_words]; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - let upper = other as u64 * (arr[i] >> 32); - let lower = other as u64 * (arr[i] & 0xFFFFFFFF); - if i < 3 { - carry[i + 1] += upper >> 32; - } - ret[i] = lower + (upper << 32); - } - $name(ret) + $name(carry) - } - } + pub fn to_bytes(&self, bytes: &mut[u8]) { + assert!($n_words * 8 == bytes.len()); + let &$name(ref arr) = self; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; + } + } + /// Multiplication by u32 + fn mul_u32(self, other: u32) -> $name { + let $name(ref arr) = self; + let mut carry = [0u64; $n_words]; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + let upper = other as u64 * (arr[i] >> 32); + let lower = other as u64 * (arr[i] & 0xFFFFFFFF); + if i < 3 { + carry[i + 1] += upper >> 32; + } + ret[i] = lower + (upper << 32); + } + $name(ret) + $name(carry) + } + } - impl From for $name { - fn from(value: u64) -> $name { - let mut ret = [0; $n_words]; - ret[0] = value; - $name(ret) - } - } + impl From for $name { + fn from(value: u64) -> $name { + let mut ret = [0; $n_words]; + ret[0] = value; + $name(ret) + } + } - impl_map_from!($name, u8, u64); - impl_map_from!($name, u16, u64); - impl_map_from!($name, u32, u64); + impl_map_from!($name, u8, u64); + impl_map_from!($name, u16, u64); + impl_map_from!($name, u32, u64); - impl<'a> From<&'a [u8]> for $name { - fn from(bytes: &[u8]) -> $name { - assert!($n_words * 8 >= bytes.len()); + impl<'a> From<&'a [u8]> for $name { + fn from(bytes: &[u8]) -> $name { + assert!($n_words * 8 >= bytes.len()); - let mut ret = [0; $n_words]; - for i in 0..bytes.len() { - let rev = bytes.len() - 1 - i; - let pos = rev / 8; - ret[pos] += (bytes[i] as u64) << (rev % 8) * 8; - } - $name(ret) - } - } + let mut ret = [0; $n_words]; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + ret[pos] += (bytes[i] as u64) << (rev % 8) * 8; + } + $name(ret) + } + } - impl FromStr for $name { - type Err = FromHexError; + impl FromStr for $name { + type Err = FromHexError; - fn from_str(value: &str) -> Result<$name, Self::Err> { - let bytes: &[u8] = &try!(value.from_hex()); - Ok(From::from(bytes)) - } - } + fn from_str(value: &str) -> Result<$name, Self::Err> { + let bytes: &[u8] = &try!(value.from_hex()); + Ok(From::from(bytes)) + } + } - impl Add<$name> for $name { - type Output = $name; + impl Add<$name> for $name { + type Output = $name; - fn add(self, other: $name) -> $name { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - if i < $n_words - 1 && ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; - } - } - if b_carry { $name(ret) + $name(carry) } else { $name(ret) } - } - } + fn add(self, other: $name) -> $name { + let $name(ref me) = self; + let $name(ref you) = other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + if i < $n_words - 1 && ret[i] < me[i] { + carry[i + 1] = 1; + b_carry = true; + } + } + if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + } + } - impl Sub<$name> for $name { - type Output = $name; + impl Sub<$name> for $name { + type Output = $name; - #[inline] - fn sub(self, other: $name) -> $name { - self + !other + From::from(1u64) - } - } + #[inline] + fn sub(self, other: $name) -> $name { + self + !other + From::from(1u64) + } + } - impl Mul<$name> for $name { - type Output = $name; + impl Mul<$name> for $name { + type Output = $name; - fn mul(self, other: $name) -> $name { - let mut me = self; - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); - } - me - } - } + fn mul(self, other: $name) -> $name { + let mut me = self; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); + } + me + } + } - impl Div<$name> for $name { - type Output = $name; + impl Div<$name> for $name { + type Output = $name; - fn div(self, other: $name) -> $name { - let mut sub_copy = self; - let mut shift_copy = other; - let mut ret = [0u64; $n_words]; + fn div(self, other: $name) -> $name { + let mut sub_copy = self; + let mut shift_copy = other; + let mut ret = [0u64; $n_words]; - let my_bits = self.bits(); - let your_bits = other.bits(); + let my_bits = self.bits(); + let your_bits = other.bits(); - // Check for division by 0 - assert!(your_bits != 0); + // Check for division by 0 + assert!(your_bits != 0); - // Early return in case we are dividing by a larger number than us - if my_bits < your_bits { - return $name(ret); - } + // Early return in case we are dividing by a larger number than us + if my_bits < your_bits { + return $name(ret); + } - // Bitwise long division - let mut shift = my_bits - your_bits; - shift_copy = shift_copy << shift; - loop { - if sub_copy >= shift_copy { - ret[shift / 64] |= 1 << (shift % 64); - sub_copy = sub_copy - shift_copy; - } - shift_copy = shift_copy >> 1; - if shift == 0 { break; } - shift -= 1; - } + // Bitwise long division + let mut shift = my_bits - your_bits; + shift_copy = shift_copy << shift; + loop { + if sub_copy >= shift_copy { + ret[shift / 64] |= 1 << (shift % 64); + sub_copy = sub_copy - shift_copy; + } + shift_copy = shift_copy >> 1; + if shift == 0 { break; } + shift -= 1; + } - $name(ret) - } - } + $name(ret) + } + } - impl BitAnd<$name> for $name { - type Output = $name; + impl BitAnd<$name> for $name { + type Output = $name; - #[inline] - fn bitand(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] & arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitand(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] & arr2[i]; + } + $name(ret) + } + } - impl BitXor<$name> for $name { - type Output = $name; + impl BitXor<$name> for $name { + type Output = $name; - #[inline] - fn bitxor(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] ^ arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitxor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] ^ arr2[i]; + } + $name(ret) + } + } - impl BitOr<$name> for $name { - type Output = $name; + impl BitOr<$name> for $name { + type Output = $name; - #[inline] - fn bitor(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] | arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] | arr2[i]; + } + $name(ret) + } + } - impl Not for $name { - type Output = $name; + impl Not for $name { + type Output = $name; - #[inline] - fn not(self) -> $name { - let $name(ref arr) = self; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = !arr[i]; - } - $name(ret) - } - } + #[inline] + fn not(self) -> $name { + let $name(ref arr) = self; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = !arr[i]; + } + $name(ret) + } + } - impl Shl for $name { - type Output = $name; + impl Shl for $name { + type Output = $name; - fn shl(self, shift: usize) -> $name { - let $name(ref original) = self; - let mut ret = [0u64; $n_words]; - let word_shift = shift / 64; - let bit_shift = shift % 64; - for i in 0..$n_words { - // Shift - if bit_shift < 64 && i + word_shift < $n_words { - ret[i + word_shift] += original[i] << bit_shift; - } - // Carry - if bit_shift > 0 && i + word_shift + 1 < $n_words { - ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); - } - } - $name(ret) - } - } + fn shl(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in 0..$n_words { + // Shift + if bit_shift < 64 && i + word_shift < $n_words { + ret[i + word_shift] += original[i] << bit_shift; + } + // Carry + if bit_shift > 0 && i + word_shift + 1 < $n_words { + ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); + } + } + $name(ret) + } + } - impl Shr for $name { - type Output = $name; + impl Shr for $name { + type Output = $name; - fn shr(self, shift: usize) -> $name { - let $name(ref original) = self; - let mut ret = [0u64; $n_words]; - let word_shift = shift / 64; - let bit_shift = shift % 64; - for i in word_shift..$n_words { - // Shift - ret[i - word_shift] += original[i] >> bit_shift; - // Carry - if bit_shift > 0 && i < $n_words - 1 { - ret[i - word_shift] += original[i + 1] << (64 - bit_shift); - } - } - $name(ret) - } - } + fn shr(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in word_shift..$n_words { + // Shift + ret[i - word_shift] += original[i] >> bit_shift; + // Carry + if bit_shift > 0 && i < $n_words - 1 { + ret[i - word_shift] += original[i + 1] << (64 - bit_shift); + } + } + $name(ret) + } + } - impl Ord for $name { - fn cmp(&self, other: &$name) -> Ordering { - let &$name(ref me) = self; - let &$name(ref you) = other; - for i in 0..$n_words { - if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Ordering::Less; } - if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Ordering::Greater; } - } - Ordering::Equal - } - } + impl Ord for $name { + fn cmp(&self, other: &$name) -> Ordering { + let &$name(ref me) = self; + let &$name(ref you) = other; + for i in 0..$n_words { + if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Ordering::Less; } + if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Ordering::Greater; } + } + Ordering::Equal + } + } - impl PartialOrd for $name { - fn partial_cmp(&self, other: &$name) -> Option { - Some(self.cmp(other)) - } - } + impl PartialOrd for $name { + fn partial_cmp(&self, other: &$name) -> Option { + Some(self.cmp(other)) + } + } - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let &$name(ref data) = self; - try!(write!(f, "0x")); - for ch in data.iter().rev() { - try!(write!(f, "{:02x}", ch)); - } - Ok(()) - } - } - ); + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let &$name(ref data) = self; + try!(write!(f, "0x")); + for ch in data.iter().rev() { + try!(write!(f, "{:02x}", ch)); + } + Ok(()) + } + } + ); } construct_uint!(U256, 4); construct_uint!(U128, 2); impl From for U256 { - fn from(value: U128) -> U256 { - let U128(ref arr) = value; - let mut ret = [0; 4]; - ret[0] = arr[0]; - ret[1] = arr[1]; - U256(ret) - } + fn from(value: U128) -> U256 { + let U128(ref arr) = value; + let mut ret = [0; 4]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U256(ret) + } } pub const ZERO_U256: U256 = U256([0x00u64; 4]); @@ -476,158 +363,158 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { - use uint::U256; - use std::str::FromStr; + use uint::U256; + use std::str::FromStr; - #[test] - pub fn uint256_from() { - let e = U256([10, 0, 0, 0]); + #[test] + pub fn uint256_from() { + let e = U256([10, 0, 0, 0]); - // test unsigned initialization - let ua = U256::from(10u8); - let ub = U256::from(10u16); - let uc = U256::from(10u32); - let ud = U256::from(10u64); - assert_eq!(e, ua); - assert_eq!(e, ub); - assert_eq!(e, uc); - assert_eq!(e, ud); + // test unsigned initialization + let ua = U256::from(10u8); + let ub = U256::from(10u16); + let uc = U256::from(10u32); + let ud = U256::from(10u64); + assert_eq!(e, ua); + assert_eq!(e, ub); + assert_eq!(e, uc); + assert_eq!(e, ud); - // test initialization from bytes - let va = U256::from(&[10u8][..]); - assert_eq!(e, va); + // test initialization from bytes + let va = U256::from(&[10u8][..]); + assert_eq!(e, va); - // more tests for initialization from bytes - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ - 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ - 0x24u8, 0x37, 0x10, - 0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..])); + // more tests for initialization from bytes + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ + 0x24u8, 0x37, 0x10, + 0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..])); - // test initializtion from string - let sa = U256::from_str("0a").unwrap(); - assert_eq!(e, sa); - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); - assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); - } + // test initializtion from string + let sa = U256::from_str("0a").unwrap(); + assert_eq!(e, sa); + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); + } - #[test] - pub fn uint256_to() { + #[test] + pub fn uint256_to() { let hex = "8090a0b0c0d0e0f00910203040506077583a2cf8264910e1436bda32571012f0"; let uint = U256::from_str(hex).unwrap(); let mut bytes = [0u8; 32]; uint.to_bytes(&mut bytes); let uint2 = U256::from(&bytes[..]); - assert_eq!(uint, uint2); - } + assert_eq!(uint, uint2); + } - #[test] - pub fn uint256_bits_test() { - assert_eq!(U256::from(0u64).bits(), 0); - assert_eq!(U256::from(255u64).bits(), 8); - assert_eq!(U256::from(256u64).bits(), 9); - assert_eq!(U256::from(300u64).bits(), 9); - assert_eq!(U256::from(60000u64).bits(), 16); - assert_eq!(U256::from(70000u64).bits(), 17); + #[test] + pub fn uint256_bits_test() { + assert_eq!(U256::from(0u64).bits(), 0); + assert_eq!(U256::from(255u64).bits(), 8); + assert_eq!(U256::from(256u64).bits(), 9); + assert_eq!(U256::from(300u64).bits(), 9); + assert_eq!(U256::from(60000u64).bits(), 16); + assert_eq!(U256::from(70000u64).bits(), 17); - //// Try to read the following lines out loud quickly - let mut shl = U256::from(70000u64); - shl = shl << 100; - assert_eq!(shl.bits(), 117); - shl = shl << 100; - assert_eq!(shl.bits(), 217); - shl = shl << 100; - assert_eq!(shl.bits(), 0); + //// Try to read the following lines out loud quickly + let mut shl = U256::from(70000u64); + shl = shl << 100; + assert_eq!(shl.bits(), 117); + shl = shl << 100; + assert_eq!(shl.bits(), 217); + shl = shl << 100; + assert_eq!(shl.bits(), 0); - //// Bit set check - //// 01010 - assert!(!U256::from(10u8).bit(0)); - assert!(U256::from(10u8).bit(1)); - assert!(!U256::from(10u8).bit(2)); - assert!(U256::from(10u8).bit(3)); - assert!(!U256::from(10u8).bit(4)); + //// Bit set check + //// 01010 + assert!(!U256::from(10u8).bit(0)); + assert!(U256::from(10u8).bit(1)); + assert!(!U256::from(10u8).bit(2)); + assert!(U256::from(10u8).bit(3)); + assert!(!U256::from(10u8).bit(4)); - //// byte check - assert_eq!(U256::from(10u8).byte(0), 10); - assert_eq!(U256::from(0xffu64).byte(0), 0xff); - assert_eq!(U256::from(0xffu64).byte(1), 0); - assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); - assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); - assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); - assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); - assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); - } + //// byte check + assert_eq!(U256::from(10u8).byte(0), 10); + assert_eq!(U256::from(0xffu64).byte(0), 0xff); + assert_eq!(U256::from(0xffu64).byte(1), 0); + assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); + assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); + assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); + assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); + assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); + } - #[test] - pub fn uint256_comp_test() { - let small = U256([10u64, 0, 0, 0]); - let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); + #[test] + pub fn uint256_comp_test() { + let small = U256([10u64, 0, 0, 0]); + let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); - assert!(small < big); - assert!(big < bigger); - assert!(bigger < biggest); - assert!(bigger <= biggest); - assert!(biggest <= biggest); - assert!(bigger >= big); - assert!(bigger >= small); - assert!(small <= small); - } + assert!(small < big); + assert!(big < bigger); + assert!(bigger < biggest); + assert!(bigger <= biggest); + assert!(biggest <= biggest); + assert!(bigger >= big); + assert!(bigger >= small); + assert!(small <= small); + } - #[test] - pub fn uint256_arithmetic_test() { - let init = U256::from(0xDEADBEEFDEADBEEFu64); - let copy = init; + #[test] + pub fn uint256_arithmetic_test() { + let init = U256::from(0xDEADBEEFDEADBEEFu64); + let copy = init; - let add = init + copy; - assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); - // Bitshifts - let shl = add << 88; - assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); - let shr = shl >> 40; - assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Increment - let incr = shr + U256::from(1u64); - assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Subtraction - let sub = incr - init; - assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - // Multiplication - let mult = sub.mul_u32(300); - assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); - // Division - assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); - let div = mult / U256::from(300u16); - assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - //// TODO: bit inversion - } + let add = init + copy; + assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); + // Bitshifts + let shl = add << 88; + assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); + let shr = shl >> 40; + assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Increment + let incr = shr + U256::from(1u64); + assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Subtraction + let sub = incr - init; + assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + // Multiplication + let mult = sub.mul_u32(300); + assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); + // Division + assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); + let div = mult / U256::from(300u16); + assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + //// TODO: bit inversion + } - #[test] - pub fn uint256_extreme_bitshift_test() { - //// Shifting a u64 by 64 bits gives an undefined value, so make sure that - //// we're doing the Right Thing here - let init = U256::from(0xDEADBEEFDEADBEEFu64); + #[test] + pub fn uint256_extreme_bitshift_test() { + //// Shifting a u64 by 64 bits gives an undefined value, so make sure that + //// we're doing the Right Thing here + let init = U256::from(0xDEADBEEFDEADBEEFu64); - assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); - let add = (init << 64) + init; - assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); - assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); - } + assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); + let add = (init << 64) + init; + assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); + assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); + } } From 98feaa4f16cf222d6037d856867f4c25ee2a441a Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 03:39:13 +0100 Subject: [PATCH 03/47] docs --- src/lib.rs | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c52a1b2d0..1de5c7ca8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,31 @@ //! Ethcore-util library +//! +//! ### Rust version: +//! - beta +//! - nightly //! -//! TODO: check reexports +//! ### Supported platforms: +//! - OSX +//! - Linux +//! +//! ### Dependencies: +//! - RocksDB 3.13 +//! +//! ### Dependencies Installation: +//! +//! - OSX: +//! +//! ```bash +//! brew install rocksdb +//! ``` +//! +//! - From source: +//! +//! ```bash +//! wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz +//! tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib +//! sudo make install +//! ``` extern crate rustc_serialize; extern crate mio; @@ -39,12 +64,3 @@ pub mod trie; pub mod nibbleslice; //pub mod network; - -// reexports -pub use std::str::FromStr; -pub use hash::*; -pub use sha3::*; -pub use bytes::*; -pub use hashdb::*; -pub use memorydb::*; - From 2805dfe22dda728f062ca3f9535704c5ba7ca961 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 11:40:54 +0100 Subject: [PATCH 04/47] removed num library from dependencies --- Cargo.toml | 1 - src/chainfilter.rs | 17 ++++++++++++----- src/lib.rs | 1 - 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 41ca44580..65ea8508f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" rocksdb = "0.2.1" -num = "0.1" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 538d1883f..076263fbb 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -41,7 +41,6 @@ use std::collections::{HashMap}; use hash::*; use sha3::*; -use num::pow; /// Represents bloom index in cache /// @@ -119,13 +118,21 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource let mut filter = ChainFilter { data_source: data_source, index_size: index_size, - level_sizes: vec![] + // 0 level has always a size of 1 + level_sizes: vec![1] }; // cache level sizes, so we do not have to calculate them all the time - for i in 0..levels { - filter.level_sizes.push(pow(index_size, i as usize)); - } + // eg. if levels == 3, index_size = 16 + // level_sizes = [1, 16, 256] + let additional: Vec = (1..).into_iter() + .scan(1, |acc, _| { + *acc = *acc * index_size; + Some(*acc) + }) + .take(levels as usize - 1) + .collect(); + filter.level_sizes.extend(additional); filter } diff --git a/src/lib.rs b/src/lib.rs index 1de5c7ca8..6366d682e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,7 +32,6 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; -extern crate num; #[macro_use] extern crate log; #[macro_use] From 94edc594d7d023f82eab9618619be4aaaf646132 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 13:12:11 +0100 Subject: [PATCH 05/47] docs --- src/sha3.rs | 14 ++++++++++++++ src/triehash.rs | 3 +-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/sha3.rs b/src/sha3.rs index 11d7ca274..b3676720c 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,8 +1,22 @@ +//! Wrapper around tiny-keccak crate. + use std::mem::uninitialized; use tiny_keccak::Keccak; use bytes::BytesConvertable; use hash::{FixedHash, H256}; +/// Types implementing this trait are sha3able. +/// +/// ``` +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::sha3::*; +/// use util::hash::*; +/// +/// fn main() { +/// assert_eq!([0u8; 0].sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); +/// } +/// ``` pub trait Hashable { fn sha3(&self) -> H256; } diff --git a/src/triehash.rs b/src/triehash.rs index 93d21aacc..eea79e9d4 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,4 +1,4 @@ -//! Generete trie root. +//! Generetes trie root. //! //! This module should be used to generate trie root hash. @@ -10,7 +10,6 @@ use rlp; use rlp::RlpStream; use vector::SharedPrefix; -// todo: verify if example for ordered_trie_root is valid /// Generates a trie root hash for a vector of values /// /// ```rust From 53547adea8b72d36ad997894e32abf40b89bc809 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Dec 2015 18:05:59 +0100 Subject: [PATCH 06/47] Fix for trie. Benchmarks for Trie. --- benches/trie.rs | 128 ++++++++++++++++++++++++++++++++++++++ src/hashdb.rs | 2 +- src/memorydb.rs | 8 +-- src/overlaydb.rs | 12 ++-- src/trie.rs | 159 +++++++++++++++++++++++++++++++++++++---------- 5 files changed, 262 insertions(+), 47 deletions(-) create mode 100644 benches/trie.rs diff --git a/benches/trie.rs b/benches/trie.rs new file mode 100644 index 000000000..0b7055bde --- /dev/null +++ b/benches/trie.rs @@ -0,0 +1,128 @@ +#![feature(test)] + +extern crate test; +extern crate rand; +extern crate ethcore_util; +#[macro_use] +extern crate log; + +use test::Bencher; +use rand::random; +//use ethcore_util::BytesConvertable; +use ethcore_util::hash::*; +use ethcore_util::bytes::*; +use ethcore_util::trie::*; +use ethcore_util::sha3::*; +use ethcore_util::ToBytes::*; + + +fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); + } + ret +} + +fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + seed.bytes()[0..r].to_vec() +} + +fn random_value(seed: &mut H256) -> Bytes { + *seed = seed.sha3(); + match seed.bytes()[0] % 2 { + 1 => vec![seed.bytes()[31];1], + _ => seed.bytes().to_vec(), + } +} + +#[bench] +fn insertions_six_high(b: &mut Bencher) { + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_bytes(6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn insertions_six_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + debug!("hash_count={:?}", t.hash_count); + }) +} + +#[bench] +fn insertions_random_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 1, 5, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn insertions_six_low(b: &mut Bencher) { + let alphabet = b"abcdef"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn sha3x1000(b: &mut Bencher) { + b.iter(||{ + let mut seed = H256::new(); + for i in 0..1000 { + seed = seed.sha3() + } + }) +} diff --git a/src/hashdb.rs b/src/hashdb.rs index f893c8df8..207883e4b 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; pub trait HashDB { /// Get the keys in the database together with number of underlying references. - fn keys(&self) -> HashMap; + fn keys(&self) -> HashMap; /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. diff --git a/src/memorydb.rs b/src/memorydb.rs index 8c7eaff2c..680129670 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -116,10 +116,6 @@ impl MemoryDB { } self.data.get(key).unwrap() } - - pub fn raw_keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() - } } impl HashDB for MemoryDB { @@ -130,8 +126,8 @@ impl HashDB for MemoryDB { } } - fn keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 > 0 {Some((k.clone(), v.1 as u32))} else {None} ).collect::>() + fn keys(&self) -> HashMap { + self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() } fn exists(&self, key: &H256) -> bool { diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 78ca67d01..c13acfd6a 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -136,17 +136,17 @@ impl OverlayDB { } impl HashDB for OverlayDB { - fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); for (key, _) in self.backing.iterator().from_start() { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; - ret.insert(h, r); + ret.insert(h, r as i32); } - for (key, refs) in self.overlay.raw_keys().into_iter() { - let refs = *ret.get(&key).unwrap_or(&0u32) as i32 + refs as i32; - ret.insert(key, refs as u32); + for (key, refs) in self.overlay.keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0) + refs; + ret.insert(key, refs); } ret } diff --git a/src/trie.rs b/src/trie.rs index 81ac4f99b..a340d78f7 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -26,6 +26,67 @@ pub trait Trie { fn remove(&mut self, key: &[u8]); } +pub enum Alphabet { + All, + Low, + Mid, + Custom(Bytes), +} + +pub struct StandardMap { + alphabet: Alphabet, + min_key: usize, + diff_key: usize, + count: usize, +} + +impl StandardMap { + fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + seed.bytes()[0..r].to_vec() + } + + fn random_value(seed: &mut H256) -> Bytes { + *seed = seed.sha3(); + match seed.bytes()[0] % 2 { + 1 => vec![seed.bytes()[31];1], + _ => seed.bytes().to_vec(), + } + } + + fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); + } + ret + } + + pub fn make(&self) -> Vec<(Bytes, Bytes)> { + let low = b"abcdef"; + let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..self.count { + let k = match self.alphabet { + Alphabet::All => Self::random_bytes(self.min_key, self.diff_key, &mut seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.diff_key, &mut seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.diff_key, &mut seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.diff_key, &mut seed), + }; + let v = Self::random_value(&mut seed); + d.push((k, v)) + } + d + } +} + #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { Empty, @@ -34,11 +95,13 @@ pub enum Node<'a> { Branch([&'a[u8]; 16], Option<&'a [u8]>) } +#[derive(Debug)] enum Operation { New(H256, Bytes), Delete(H256), } +#[derive(Debug)] struct Diff (Vec); impl Diff { @@ -48,8 +111,9 @@ impl Diff { /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { - trace!("new_node: reference node {:?}", rlp.pretty()); let rlp_sha3 = rlp.sha3(); + + trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty()); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); } @@ -61,25 +125,18 @@ impl Diff { /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. fn delete_node_sha3(&mut self, old_sha3: H256) { + trace!("delete_node: {:?}", old_sha3); self.0.push(Operation::Delete(old_sha3)); } fn delete_node(&mut self, old: &Rlp) { if old.is_data() && old.size() == 32 { - self.0.push(Operation::Delete(H256::decode(old))); + self.delete_node_sha3(H256::decode(old)); } } fn delete_node_from_slice(&mut self, old: &[u8]) { - let r = Rlp::new(old); - if r.is_data() && r.size() == 32 { - self.0.push(Operation::Delete(H256::decode(&r))); - } - } - - fn replace_node(&mut self, old: &Rlp, rlp: Bytes, out: &mut RlpStream) { - self.delete_node(old); - self.new_node(rlp, out); + self.delete_node(&Rlp::new(old)); } } @@ -192,11 +249,12 @@ impl <'a>Node<'a> { pub struct TrieDB { db: Box, root: H256, + pub hash_count: usize, } impl fmt::Debug for TrieDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(writeln!(f, "[")); + try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); writeln!(f, "]") @@ -209,7 +267,7 @@ enum MaybeChanged<'a> { } impl TrieDB { - pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } + pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } @@ -220,6 +278,7 @@ impl TrieDB { fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); + self.hash_count += 1; trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } @@ -234,6 +293,7 @@ impl TrieDB { Operation::New(h, d) => { trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty()); self.db.emplace(h, d); + self.hash_count += 1; } } } @@ -272,14 +332,13 @@ impl TrieDB { r } - pub fn db_items_remaining(&self) -> HashMap { + pub fn db_items_remaining(&self) -> HashMap { let mut ret = self.db().keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { - let old = *ret.get(&k).expect("Node in trie is not in database!"); - assert!(old >= v); - match old > v { - true => ret.insert(k, old - v), - _ => ret.remove(&k), + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), }; } ret @@ -437,7 +496,11 @@ impl TrieDB { } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!"); + let r = self.db.lookup(&h).unwrap_or_else(||{ + println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); + println!("Diff: {:?}", diff); + panic!(); + }); trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); diff.delete_node_sha3(h); r @@ -518,7 +581,7 @@ impl TrieDB { diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), (true, i) => { // harder - original has something there already let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); - diff.replace_node(&orig.at(i), new, &mut s); + diff.new_node(new, &mut s); } (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } @@ -804,22 +867,21 @@ mod tests { use std::collections::HashSet; use bytes::{ToPretty,Bytes}; - fn random_key() -> Vec { - let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + fn random_key(alphabet: &[u8], min_count: usize, diff_count: usize) -> Vec { let mut ret: Vec = Vec::new(); - let r = random::() % 4 + 1; + let r = min_count + if diff_count > 0 {random::() % diff_count} else {0}; for _ in 0..r { - ret.push(chars[random::() % chars.len()]); + ret.push(alphabet[random::() % alphabet.len()]); } ret } - - fn random_value(i: usize) -> Bytes { + + fn random_value_indexed(j: usize) -> Bytes { match random::() % 2 { - 0 => rlp::encode(&i), + 0 => rlp::encode(&j), _ => { let mut h = H256::new(); - h.mut_bytes()[31] = i as u8; + h.mut_bytes()[31] = j as u8; rlp::encode(&h) }, } @@ -842,16 +904,44 @@ mod tests { } } + macro_rules! map({$($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + };); + #[test] fn playpen() { env_logger::init().ok(); - for _ in 0..1000 { + + let maps = map!{ + "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, diff_key: 0, count: 1000}, + "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, diff_key: 0, count: 1000}, + "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, diff_key: 0, count: 1000}, + "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, diff_key: 5, count: 1000} + }; + for sm in maps { + let m = sm.1.make(); + let t = populate_trie(&m); + println!("{:?}: root={:?}, hash_count={:?}", sm.0, t.root(), t.hash_count); + }; + panic!(); + + for test_i in 0..1 { + if test_i % 50 == 0 { + debug!("{:?} of 10000 stress tests done", test_i); + } let mut x: Vec<(Vec, Vec)> = Vec::new(); let mut got: HashSet> = HashSet::new(); - for j in 0..10usize { - let key = random_key(); + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + for j in 0..1000usize { + let key = random_key(alphabet, 5, 0); if !got.contains(&key) { - x.push((key.clone(), random_value(j))); + x.push((key.clone(), random_value_indexed(j))); got.insert(key); } } @@ -1101,8 +1191,9 @@ mod tests { fn stress() { for _ in 0..5000 { let mut x: Vec<(Vec, Vec)> = Vec::new(); + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { - let key = random_key(); + let key = random_key(alphabet, 5, 1); x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); From 38f0af1aa0e47195d74a9552cc0530945effe6ca Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Dec 2015 20:04:26 +0100 Subject: [PATCH 07/47] Code cleanups. --- src/trie.rs | 87 +++++++++++++++++------------------------------------ 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index a340d78f7..48fbe8d83 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -511,58 +511,24 @@ impl TrieDB { } } - /// Transform an existing extension or leaf node to an invalid single-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmuted_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_extension_to_branch"); - let mut s = RlpStream::new_list(17); - assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let index = orig_partial.at(0); - // orig is extension - orig_raw_payload is a node itself. - for i in 0..17 { - if index == i { - if orig_partial.len() > 1 { - // still need an extension - diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s); - } else { - // was an extension of length 1 - just redirect the payload into here. - s.append_raw(orig_raw_payload, 1); - } - } else { - s.append_empty_data(); - } - } - s.out() - } - - fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_leaf_to_branch"); + fn augmented_into_transmuted_branch(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + assert!(orig_is_leaf || !orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. let mut s = RlpStream::new_list(17); let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; - // orig is leaf - orig_raw_payload is data representing the actual value. for i in 0..17 { - match (index == i, i) { - (true, 16) => // leaf entry - just replace. - { s.append(&value); }, - (true, _) => // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&orig_partial.mid(1), value), &mut s), - (false, _) => { s.append_empty_data(); } + match orig_is_leaf { + // not us - empty. + _ if index != i => { s.append_empty_data(); }, + // just replace. + true if i == 16 => { s.append(&value); }, + // original has empty slot. + true => diff.new_node(Self::compose_leaf(&orig_partial.mid(1), Rlp::new(orig_raw_payload).data()), &mut s), + // + false if orig_partial.len() > 1 => diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s), + false => { s.append_raw(orig_raw_payload, 1); }, } - } - s.out() - } - - /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmuted_to_branch_and_augmented(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_to_branch_and_augmented"); - let intermediate = match orig_is_leaf { - true => Self::transmuted_leaf_to_branch(orig_partial, Rlp::new(orig_raw_payload).data(), diff), - false => Self::transmuted_extension_to_branch(orig_partial, orig_raw_payload, diff), }; - self.augmented(&intermediate, partial, value, diff) + self.augmented(&s.out(), partial, value, diff) // TODO: implement without having to make an intermediate representation. } @@ -573,17 +539,19 @@ impl TrieDB { trace!("augmented_into_branch"); let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; - for i in 0usize..17 { - match (index == i, i) { - (true, 16) => // leaf entry - just replace. - { s.append(&value); }, - (true, i) if orig.at(i).is_empty() => // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), - (true, i) => { // harder - original has something there already + for i in 0..17 { + match index == i { + // not us - leave alone. + false => { s.append_raw(orig.at(i).raw(), 1); }, + // branch-leaf entry - just replace. + true if i == 16 => { s.append(&value); }, + // original had empty slot - place a leaf there. + true if orig.at(i).is_empty() => diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + // original has something there already; augment. + true => { let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); diff.new_node(new, &mut s); } - (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } } s.out() @@ -619,17 +587,16 @@ impl TrieDB { (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); - self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) + self.augmented_into_transmuted_branch(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: // transform to an extension + augmented version of onward node. - let downstream_node: Bytes = if is_leaf { + let downstream_node: Bytes = match is_leaf { // no onward node because we're a leaf - create fake stub and use that. - self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff) - } else { - self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff) + true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff), + false => self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff), }; Self::create_extension(&existing_key, downstream_node, diff) }, From 5ba8f528ba3e10c8a45a9c55c48d0d778999307b Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Dec 2015 13:22:22 +0100 Subject: [PATCH 08/47] removed build warnings and errors --- benches/trie.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index 0b7055bde..2865c5778 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -7,13 +7,10 @@ extern crate ethcore_util; extern crate log; use test::Bencher; -use rand::random; -//use ethcore_util::BytesConvertable; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; use ethcore_util::sha3::*; -use ethcore_util::ToBytes::*; fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { @@ -121,7 +118,7 @@ fn insertions_six_low(b: &mut Bencher) { fn sha3x1000(b: &mut Bencher) { b.iter(||{ let mut seed = H256::new(); - for i in 0..1000 { + for _ in 0..1000 { seed = seed.sha3() } }) From 9b080c91619decc61a0d7b0fb2bafaf47d6383ae Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 6 Dec 2015 11:20:37 +0100 Subject: [PATCH 09/47] microoptimization --- src/rlp.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a7543aae5..bada78de0 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -844,8 +844,12 @@ impl RlpStream { // we may finish, if the appended list len is equal 0 self.encoder.bytes.push(0xc0u8); self.note_appended(1); - } - _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), + }, + _ => { + // reserve at least double size of the len + self.encoder.bytes.reserve(len * 2); + self.unfinished_lists.push_back(ListInfo::new(position, len)); + }, } // return chainable self From 182f356467213fbf80e4a224fa4b4e027eb9a3de Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 6 Dec 2015 11:42:15 +0100 Subject: [PATCH 10/47] replace linked list with vec --- src/rlp.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index bada78de0..624254fc6 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -32,7 +32,6 @@ use std::fmt; use std::cell::Cell; -use std::collections::LinkedList; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; @@ -778,7 +777,7 @@ impl ListInfo { /// Appendable rlp encoder. pub struct RlpStream { - unfinished_lists: LinkedList, + unfinished_lists: Vec, encoder: BasicEncoder, } @@ -786,7 +785,7 @@ impl RlpStream { /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { - unfinished_lists: LinkedList::new(), + unfinished_lists: vec![], encoder: BasicEncoder::new(), } } @@ -848,7 +847,7 @@ impl RlpStream { _ => { // reserve at least double size of the len self.encoder.bytes.reserve(len * 2); - self.unfinished_lists.push_back(ListInfo::new(position, len)); + self.unfinished_lists.push(ListInfo::new(position, len)); }, } @@ -930,7 +929,7 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } pub fn is_finished(&self) -> bool { - self.unfinished_lists.back().is_none() + self.unfinished_lists.len() == 0 } /// Streams out encoded bytes. @@ -945,7 +944,12 @@ impl RlpStream { /// Try to finish lists fn note_appended(&mut self, inserted_items: usize) -> () { - let should_finish = match self.unfinished_lists.back_mut() { + if self.unfinished_lists.len() == 0 { + return; + } + + let back = self.unfinished_lists.len() - 1; + let should_finish = match self.unfinished_lists.get_mut(back) { None => false, Some(ref mut x) => { x.current += inserted_items; @@ -957,7 +961,7 @@ impl RlpStream { }; if should_finish { - let x = self.unfinished_lists.pop_back().unwrap(); + let x = self.unfinished_lists.pop().unwrap(); let len = self.encoder.bytes.len() - x.position; self.encoder.insert_list_len_at_pos(len, x.position); self.note_appended(1); From 8cf437b34942a99798d13357859079032c371e89 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 11:27:12 +0100 Subject: [PATCH 11/47] rlp uses elastic array --- Cargo.toml | 1 + src/lib.rs | 1 + src/rlp.rs | 29 +++++++++++++++-------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 65ea8508f..13295f766 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ rocksdb = "0.2.1" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" +elastic-array = "0.4" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/lib.rs b/src/lib.rs index 6366d682e..883cac2f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ extern crate time; extern crate crypto as rcrypto; extern crate secp256k1; extern crate arrayvec; +extern crate elastic_array; pub mod macros; pub mod error; diff --git a/src/rlp.rs b/src/rlp.rs index 624254fc6..d0dece3ba 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -33,6 +33,7 @@ use std::fmt; use std::cell::Cell; use std::error::Error as StdError; +use elastic_array::*; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; @@ -758,7 +759,7 @@ impl Decoder for BasicDecoder { } } -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] struct ListInfo { position: usize, current: usize, @@ -777,7 +778,7 @@ impl ListInfo { /// Appendable rlp encoder. pub struct RlpStream { - unfinished_lists: Vec, + unfinished_lists: ElasticArray16, encoder: BasicEncoder, } @@ -785,7 +786,7 @@ impl RlpStream { /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { - unfinished_lists: vec![], + unfinished_lists: ElasticArray16::new(), encoder: BasicEncoder::new(), } } @@ -846,7 +847,7 @@ impl RlpStream { }, _ => { // reserve at least double size of the len - self.encoder.bytes.reserve(len * 2); + //self.encoder.bytes.reserve(len * 2); self.unfinished_lists.push(ListInfo::new(position, len)); }, } @@ -882,7 +883,7 @@ impl RlpStream { /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { // push raw items - self.encoder.bytes.extend(bytes); + self.encoder.bytes.append_slice(bytes); // try to finish and prepend the length self.note_appended(item_count); @@ -937,7 +938,7 @@ impl RlpStream { /// panic! if stream is not finished. pub fn out(self) -> Vec { match self.is_finished() { - true => self.encoder.out(), + true => self.encoder.out().to_vec(), false => panic!() } } @@ -985,7 +986,7 @@ pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); object.encode(&mut encoder); - encoder.out() + encoder.out().to_vec() } pub trait Encodable { @@ -1038,12 +1039,12 @@ impl Encodable for Vec { } struct BasicEncoder { - bytes: Vec, + bytes: ElasticArray1024, } impl BasicEncoder { fn new() -> BasicEncoder { - BasicEncoder { bytes: vec![] } + BasicEncoder { bytes: ElasticArray1024::new() } } /// inserts list prefix at given position @@ -1062,7 +1063,7 @@ impl BasicEncoder { } /// get encoded value - fn out(self) -> Vec { + fn out(self) -> ElasticArray1024 { self.bytes } } @@ -1073,17 +1074,17 @@ impl Encoder for BasicEncoder { // just 0 0 => self.bytes.push(0x80u8), // byte is its own encoding - 1 if bytes[0] < 0x80 => self.bytes.extend(bytes), + 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), // (prefix + length), followed by the string len @ 1 ... 55 => { self.bytes.push(0x80u8 + len as u8); - self.bytes.extend(bytes); + self.bytes.append_slice(bytes); } // (prefix + length of length), followed by the length, followd by the string len => { self.bytes.push(0xb7 + len.to_bytes_len() as u8); - self.bytes.extend(len.to_bytes()); - self.bytes.extend(bytes); + self.bytes.append_slice(&len.to_bytes()); + self.bytes.append_slice(bytes); } } } From fcae9034acbd17e136f5a062c3d4bc095c56bb3f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 7 Dec 2015 15:00:54 +0100 Subject: [PATCH 12/47] Trie documentation and code consolidation. --- src/trie.rs | 451 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 265 insertions(+), 186 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 48fbe8d83..0425565bc 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -15,17 +15,30 @@ use std::collections::HashMap; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); +/// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { + /// Return the root of the trie. fn root(&self) -> &H256; + + /// Is the trie empty? fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } - // TODO: consider returning &[u8]... + /// Does the trie contain a given key? fn contains(&self, key: &[u8]) -> bool; + + /// What is the value of the given key in this trie? fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; + + /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing + /// `key` from the trie. fn insert(&mut self, key: &[u8], value: &[u8]); + + /// Remove a `key` from the trie. Equivalent to making it equal to the empty + /// value. fn remove(&mut self, key: &[u8]); } +/// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { All, Low, @@ -33,21 +46,25 @@ pub enum Alphabet { Custom(Bytes), } +/// Standard test map for profiling tries. pub struct StandardMap { alphabet: Alphabet, min_key: usize, - diff_key: usize, + journal_key: usize, count: usize, } impl StandardMap { - fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + diff_count <= 32); + /// Get a bunch of random bytes, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// `seed` is mutated pseudoramdonly and used. + fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); seed.bytes()[0..r].to_vec() } + /// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used. fn random_value(seed: &mut H256) -> Bytes { *seed = seed.sha3(); match seed.bytes()[0] % 2 { @@ -56,10 +73,12 @@ impl StandardMap { } } - fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + diff_count <= 32); + /// Get a random word of, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// Each byte is an item from `alphabet`. `seed` is mutated pseudoramdonly and used. + fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); let mut ret: Vec = Vec::with_capacity(r); for i in 0..r { ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); @@ -67,6 +86,7 @@ impl StandardMap { ret } + /// Create the standard map (set of keys and values) for the object's fields. pub fn make(&self) -> Vec<(Bytes, Bytes)> { let low = b"abcdef"; let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; @@ -75,10 +95,10 @@ impl StandardMap { let mut seed = H256::new(); for _ in 0..self.count { let k = match self.alphabet { - Alphabet::All => Self::random_bytes(self.min_key, self.diff_key, &mut seed), - Alphabet::Low => Self::random_word(low, self.min_key, self.diff_key, &mut seed), - Alphabet::Mid => Self::random_word(mid, self.min_key, self.diff_key, &mut seed), - Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.diff_key, &mut seed), + Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), }; let v = Self::random_value(&mut seed); d.push((k, v)) @@ -87,6 +107,7 @@ impl StandardMap { } } +/// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { Empty, @@ -95,19 +116,22 @@ pub enum Node<'a> { Branch([&'a[u8]; 16], Option<&'a [u8]>) } +/// Type of operation for the backing database - either a new node or a node deletion. #[derive(Debug)] enum Operation { New(H256, Bytes), Delete(H256), } +/// A journal of operations on the backing database. #[derive(Debug)] -struct Diff (Vec); +struct Journal (Vec); -impl Diff { - fn new() -> Diff { Diff(vec![]) } +impl Journal { + /// Create a new, empty, object. + fn new() -> Journal { Journal(vec![]) } - /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` + /// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal` /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { @@ -123,24 +147,23 @@ impl Diff { } } - /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. + /// Given the RLP that encodes a now-unused node, leave `journal` in such a state that it is noted. fn delete_node_sha3(&mut self, old_sha3: H256) { trace!("delete_node: {:?}", old_sha3); self.0.push(Operation::Delete(old_sha3)); } - fn delete_node(&mut self, old: &Rlp) { - if old.is_data() && old.size() == 32 { - self.delete_node_sha3(H256::decode(old)); + /// Register an RLP-encoded node for deletion (given a slice), if it needs to be deleted. + fn delete_node(&mut self, old: &[u8]) { + let r = Rlp::new(old); + if r.is_data() && r.size() == 32 { + self.delete_node_sha3(H256::decode(&r)); } } - - fn delete_node_from_slice(&mut self, old: &[u8]) { - self.delete_node(&Rlp::new(old)); - } } impl <'a>Node<'a> { + /// Decode the `node_rlp` and return the Node. fn decoded(node_rlp: &'a [u8]) -> Node<'a> { let r = Rlp::new(node_rlp); match r.prototype() { @@ -168,7 +191,10 @@ impl <'a>Node<'a> { } } - // todo: should check length before encoding, cause it may just be sha3 of data + /// Encode the node into RLP. + /// + /// Will always return the direct node RLP even if it's 32 or more bytes. To get the + /// RLP which would be valid for using in another node, use `encoded_and_added()`. fn encoded(&self) -> Bytes { match *self { Node::Leaf(ref slice, ref value) => { @@ -202,7 +228,9 @@ impl <'a>Node<'a> { } } - fn encoded_and_added(&self, diff: &mut Diff) -> Bytes { + /// Encode the node, adding it to `journal` if necessary and return the RLP valid for + /// insertion into a parent node. + fn encoded_and_added(&self, journal: &mut Journal) -> Bytes { let mut stream = RlpStream::new(); match *self { Node::Leaf(ref slice, ref value) => { @@ -234,47 +262,96 @@ impl <'a>Node<'a> { 0 ... 31 => node, _ => { let mut stream = RlpStream::new(); - diff.new_node(node, &mut stream); + journal.new_node(node, &mut stream); stream.out() } } } } -//enum ValidationResult<'a> { - //Valid, - //Invalid { node: Node<'a>, depth: usize } -//} - +/// A `Trie` implementation using a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` +/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get +/// which items in the backing database do not belong to this trie. If this is the only trie in the +/// backing database, then `db_items_remaining()` should be empty. +/// +/// # Example +/// ``` +/// extern crate ethcore_util; +/// use ethcore_util::trie::*; +/// fn main() { +/// let mut t = TrieDB::new_memory(); +/// assert!(t.is_empty()); +/// assert_eq!(*t.root(), SHA3_NULL_RLP); +/// t.insert(b"foo", b"bar"); +/// assert!(t.contains(b"foo")); +/// assert_eq!(t.at(b"foo").unwrap(), b"bar"); +/// assert!(t.db_items_remaining().is_empty()); +/// t.remove(b"foo"); +/// assert!(!t.contains(b"foo")); +/// assert!(t.db_items_remaining().is_empty()); +/// } +/// ``` pub struct TrieDB { db: Box, root: H256, pub hash_count: usize, } -impl fmt::Debug for TrieDB { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(writeln!(f, "c={:?} [", self.hash_count)); - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); - writeln!(f, "]") - } -} - +/// Option-like type allowing either a Node object passthrough or Bytes in the case of data alteration. enum MaybeChanged<'a> { Same(Node<'a>), Changed(Bytes), } impl TrieDB { + /// Create a new trie with the boxed backing database `box_db`. pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } + /// Convenience function to create a new trie with the backing database `db`. pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } + /// Convenience function to create a new trie with a new `MemoryDB` based backing database. pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } + /// Get the backing database. pub fn db(&self) -> &HashDB { self.db.as_ref() } + /// Determine all the keys in the backing database that belong to the trie. + pub fn keys(&self) -> Vec { + let mut ret: Vec = Vec::new(); + ret.push(self.root.clone()); + self.accumulate_keys(self.root_node(), &mut ret); + ret + } + + /// Convert a vector of hashes to a hashmap of hash to occurances. + pub fn to_map(hashes: Vec) -> HashMap { + let mut r: HashMap = HashMap::new(); + for h in hashes.into_iter() { + let c = *r.get(&h).unwrap_or(&0); + r.insert(h, c + 1); + } + r + } + + /// Determine occurances of items in the backing database which are not related to this + /// trie. + pub fn db_items_remaining(&self) -> HashMap { + let mut ret = self.db().keys(); + for (k, v) in Self::to_map(self.keys()).into_iter() { + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), + }; + } + ret + } + + /// Set the trie to a new root node's RLP, inserting the new RLP into the backing database + /// and removing the old. fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); @@ -282,9 +359,10 @@ impl TrieDB { trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } - fn apply(&mut self, diff: Diff) { - trace!("applying {:?} changes", diff.0.len()); - for d in diff.0.into_iter() { + /// Apply the items in `journal` into the backing database. + fn apply(&mut self, journal: Journal) { + trace!("applying {:?} changes", journal.0.len()); + for d in journal.0.into_iter() { match d { Operation::Delete(h) => { trace!("TrieDB::apply --- {:?}", &h); @@ -299,13 +377,7 @@ impl TrieDB { } } - pub fn keys(&self) -> Vec { - let mut ret: Vec = Vec::new(); - ret.push(self.root.clone()); - self.accumulate_keys(self.root_node(), &mut ret); - ret - } - + /// Recursion helper for `keys`. fn accumulate_keys(&self, node: Node, acc: &mut Vec) { let mut handle_payload = |payload| { let p = Rlp::new(payload); @@ -323,27 +395,17 @@ impl TrieDB { } } - fn to_map(hashes: Vec) -> HashMap { - let mut r: HashMap = HashMap::new(); - for h in hashes.into_iter() { - let c = *r.get(&h).unwrap_or(&0); - r.insert(h, c + 1); - } - r + /// Get the root node's RLP. + fn root_node(&self) -> Node { + Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) } - pub fn db_items_remaining(&self) -> HashMap { - let mut ret = self.db().keys(); - for (k, v) in Self::to_map(self.keys()).into_iter() { - let keycount = *ret.get(&k).unwrap_or(&0); - match keycount == v as i32 { - true => ret.remove(&k), - _ => ret.insert(k, keycount - v as i32), - }; - } - ret + /// Get the root node as a `Node`. + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { + Node::decoded(self.get_raw_or_lookup(node)) } + /// Indentation helper for `formal_all`. fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { for _ in 0..size { try!(write!(f, " ")); @@ -351,14 +413,7 @@ impl TrieDB { Ok(()) } - fn root_node(&self) -> Node { - Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) - } - - fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { - Node::decoded(self.get_raw_or_lookup(node)) - } - + /// Recursion helper for implementation of formatting trait. fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { match node { Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), @@ -394,11 +449,16 @@ impl TrieDB { Ok(()) } + /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.get_from_node(&root_rlp, key) } + /// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no + /// value exists for the key. + /// + /// Note: Not a public API; use Trie trait functions. fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { match Node::decoded(node) { Node::Leaf(ref slice, ref value) if key == slice => Some(value), @@ -413,6 +473,9 @@ impl TrieDB { } } + /// Given some node-describing data `node`, return the actual node RLP. + /// This could be a simple identity operation in the case that the node is sufficiently small, but + /// may require a database lookup. fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { // check if its sha3 + len let r = Rlp::new(node); @@ -422,20 +485,26 @@ impl TrieDB { } } - fn add(&mut self, key: &NibbleSlice, value: &[u8]) { + /// Insert a `key` and `value` pair into the trie. + /// + /// Note: Not a public API; use Trie trait functions. + fn insert_ns(&mut self, key: &NibbleSlice, value: &[u8]) { trace!("ADD: {:?} {:?}", key, value.pretty()); // determine what the new root is, insert new nodes and remove old as necessary. - let mut todo: Diff = Diff::new(); + let mut todo: Journal = Journal::new(); let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); trace!("/"); } - fn delete(&mut self, key: &NibbleSlice) { + /// Remove a `key` and `value` pair from the trie. + /// + /// Note: Not a public API; use Trie trait functions. + fn remove_ns(&mut self, key: &NibbleSlice) { trace!("DELETE: {:?}", key); // determine what the new root is, insert new nodes and remove old as necessary. - let mut todo: Diff = Diff::new(); + let mut todo: Journal = Journal::new(); match self.cleared_from_slice(self.db.lookup(&self.root).expect("Trie root not found!"), key, &mut todo) { Some(root_rlp) => { self.apply(todo); @@ -448,6 +517,7 @@ impl TrieDB { trace!("/"); } + /// Compose a leaf node in RLP given the `partial` key and `value`. fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty()); let mut s = RlpStream::new_list(2); @@ -458,6 +528,7 @@ impl TrieDB { r } + /// Compose a raw extension/leaf node in RLP given the `partial` key, `raw_payload` and whether it `is_leaf`. fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { trace!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); @@ -468,6 +539,7 @@ impl TrieDB { r } + /// Compose a branch node in RLP with a particular `value` sitting in the value position (17th place). fn compose_stub_branch(value: &[u8]) -> Bytes { let mut s = RlpStream::new_list(17); for _ in 0..16 { s.append_empty_data(); } @@ -475,21 +547,14 @@ impl TrieDB { s.out() } + /// Compose an extension node's RLP with the `partial` key and `raw_payload`. fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } - fn create_extension(partial: &NibbleSlice, downstream_node: Bytes, diff: &mut Diff) -> Bytes { - trace!("create_extension partial: {:?}, downstream_node: {:?}", partial, downstream_node.pretty()); - let mut s = RlpStream::new_list(2); - s.append(&partial.encoded(false)); - diff.new_node(downstream_node, &mut s); - s.out() - } - - /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from - /// the trie. - fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { + /// Return the bytes encoding the node represented by `rlp`. `journal` will record necessary + /// removal instructions from the backing database. + fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, journal: &mut Journal) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { trace!("take_node {:?} (inline)", rlp.raw().pretty()); rlp.raw() @@ -498,11 +563,11 @@ impl TrieDB { let h = H256::decode(rlp); let r = self.db.lookup(&h).unwrap_or_else(||{ println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); - println!("Diff: {:?}", diff); + println!("Journal: {:?}", journal); panic!(); }); trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); - diff.delete_node_sha3(h); + journal.delete_node_sha3(h); r } else { @@ -511,61 +576,15 @@ impl TrieDB { } } - fn augmented_into_transmuted_branch(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - assert!(orig_is_leaf || !orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let mut s = RlpStream::new_list(17); - let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; - for i in 0..17 { - match orig_is_leaf { - // not us - empty. - _ if index != i => { s.append_empty_data(); }, - // just replace. - true if i == 16 => { s.append(&value); }, - // original has empty slot. - true => diff.new_node(Self::compose_leaf(&orig_partial.mid(1), Rlp::new(orig_raw_payload).data()), &mut s), - // - false if orig_partial.len() > 1 => diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s), - false => { s.append_raw(orig_raw_payload, 1); }, - } - }; - self.augmented(&s.out(), partial, value, diff) - // TODO: implement without having to make an intermediate representation. - } - - /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the - /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that - /// once applied the returned node is valid. - fn augmented_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("augmented_into_branch"); - let mut s = RlpStream::new_list(17); - let index = if partial.is_empty() {16} else {partial.at(0) as usize}; - for i in 0..17 { - match index == i { - // not us - leave alone. - false => { s.append_raw(orig.at(i).raw(), 1); }, - // branch-leaf entry - just replace. - true if i == 16 => { s.append(&value); }, - // original had empty slot - place a leaf there. - true if orig.at(i).is_empty() => diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), - // original has something there already; augment. - true => { - let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); - diff.new_node(new, &mut s); - } - } - } - s.out() - } - /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that includes the new node. /// - /// The database will be updated so as to make the returned RLP valid through inserting + /// `journal` will record the database updates so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// /// **This operation will not insert the new node nor destroy the original.** - fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], journal: &mut Journal) -> Bytes { trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); @@ -573,7 +592,24 @@ impl TrieDB { Prototype::List(17) => { trace!("branch: ROUTE,AUGMENT"); // already have a branch. route and augment. - self.augmented_into_branch(&old_rlp, partial, value, diff) + let mut s = RlpStream::new_list(17); + let index = if partial.is_empty() {16} else {partial.at(0) as usize}; + for i in 0..17 { + match index == i { + // not us - leave alone. + false => { s.append_raw(old_rlp.at(i).raw(), 1); }, + // branch-leaf entry - just replace. + true if i == 16 => { s.append(&value); }, + // original had empty slot - place a leaf there. + true if old_rlp.at(i).is_empty() => journal.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + // original has something there already; augment. + true => { + let new = self.augmented(self.take_node(&old_rlp.at(i), journal), &partial.mid(1), value, journal); + journal.new_node(new, &mut s); + } + } + } + s.out() }, Prototype::List(2) => { let existing_key_rlp = old_rlp.at(0); @@ -587,7 +623,24 @@ impl TrieDB { (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); - self.augmented_into_transmuted_branch(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) + assert!(is_leaf || !existing_key.is_empty()); // extension nodes are not allowed to have empty partial keys. + let mut s = RlpStream::new_list(17); + let index = if existing_key.is_empty() {16} else {existing_key.at(0)}; + for i in 0..17 { + match is_leaf { + // not us - empty. + _ if index != i => { s.append_empty_data(); }, + // branch-value: just replace. + true if i == 16 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + // direct extension: just replace. + false if existing_key.len() == 1 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + // original has empty slot. + true => journal.new_node(Self::compose_leaf(&existing_key.mid(1), old_rlp.at(1).data()), &mut s), + // additional work required after branching. + false => journal.new_node(Self::compose_extension(&existing_key.mid(1), old_rlp.at(1).raw()), &mut s), + } + }; + self.augmented(&s.out(), partial, value, journal) }, (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); @@ -595,10 +648,15 @@ impl TrieDB { // transform to an extension + augmented version of onward node. let downstream_node: Bytes = match is_leaf { // no onward node because we're a leaf - create fake stub and use that. - true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff), - false => self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff), + true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, journal), + false => self.augmented(self.take_node(&old_rlp.at(1), journal), &partial.mid(cp), value, journal), }; - Self::create_extension(&existing_key, downstream_node, diff) + + trace!("create_extension partial: {:?}, downstream_node: {:?}", existing_key, downstream_node.pretty()); + let mut s = RlpStream::new_list(2); + s.append(&existing_key.encoded(false)); + journal.new_node(downstream_node, &mut s); + s.out() }, (_, cp) => { // partially-shared prefix for this extension: @@ -612,12 +670,12 @@ impl TrieDB { // low (farther from root) let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).raw(), is_leaf); - let augmented_low = self.augmented(&low, &partial.mid(cp), value, diff); + let augmented_low = self.augmented(&low, &partial.mid(cp), value, journal); // high (closer to root) let mut s = RlpStream::new_list(2); s.append(&existing_key.encoded_leftmost(cp, false)); - diff.new_node(augmented_low, &mut s); + journal.new_node(augmented_low, &mut s); s.out() }, } @@ -630,6 +688,7 @@ impl TrieDB { } } + /// Given a `MaybeChanged` result `n`, return the node's RLP regardless of whether it changed. fn encoded(n: MaybeChanged) -> Bytes { match n { MaybeChanged::Same(n) => n.encoded(), @@ -637,18 +696,20 @@ impl TrieDB { } } - fn fixed_indirection<'a>(n: Node<'a>, diff: &mut Diff) -> MaybeChanged<'a> { + /// Fix the node payload's sizes in `n`, replacing any over-size payloads with the hashed reference + /// and placing the payload DB insertions in the `journal`. + fn fixed_indirection<'a>(n: Node<'a>, journal: &mut Journal) -> MaybeChanged<'a> { match n { Node::Extension(partial, payload) if payload.len() >= 32 && Rlp::new(payload).is_list() => { // make indirect - MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(diff)).encoded()) + MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(journal)).encoded()) }, Node::Branch(payloads, value) => { // check each child isn't too big // TODO OPTIMISE - should really check at the point of (re-)constructing the branch. for i in 0..16 { if payloads[i].len() >= 32 && Rlp::new(payloads[i]).is_list() { - let n = Node::decoded(payloads[i]).encoded_and_added(diff); + let n = Node::decoded(payloads[i]).encoded_and_added(journal); let mut new_nodes = payloads; new_nodes[i] = &n; return MaybeChanged::Changed(Node::Branch(new_nodes, value).encoded()) @@ -668,8 +729,11 @@ impl TrieDB { /// - Extension node followed by anything other than a Branch node. /// - Extension node with a child which has too many bytes to be inline. /// + /// `journal` will record the database updates so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + /// /// **This operation will not insert the new node nor destroy the original.** - fn fixed<'a, 'b>(&'a self, n: Node<'b>, diff: &mut Diff) -> MaybeChanged<'b> where 'a: 'b { + fn fixed<'a, 'b>(&'a self, n: Node<'b>, journal: &mut Journal) -> MaybeChanged<'b> where 'a: 'b { trace!("fixed node={:?}", n); match n { Node::Branch(nodes, node_value) => { @@ -681,9 +745,6 @@ impl TrieDB { Many, }; let mut used_index = UsedIndex::None; - // 0-15 -> index of a non-null branch - // 16 -> no non-null branch - // 17 -> multiple non-null branches for i in 0..16 { match (nodes[i] == NULL_RLP, &used_index) { (false, &UsedIndex::None) => used_index = UsedIndex::One(i as u8), @@ -699,17 +760,17 @@ impl TrieDB { // TODO: OPTIMISE: - don't call fixed again but put the right node in straight away here. // call fixed again since the transmute may cause invalidity. let new_partial: [u8; 1] = [a; 1]; - MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), diff))) + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), journal))) }, (UsedIndex::None, Some(value)) => { // one leaf value // transmute to leaf. // call fixed again since the transmute may cause invalidity. - MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), diff))) + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), journal))) } _ => { // onwards node(s) and/or leaf // no transmute needed, but should still fix the indirection. trace!("no-transmute: FIXINDIRECTION"); - Self::fixed_indirection(Node::Branch(nodes, node_value), diff) + Self::fixed_indirection(Node::Branch(nodes, node_value), journal) }, } }, @@ -717,16 +778,16 @@ impl TrieDB { match Node::decoded(self.get_raw_or_lookup(payload)) { Node::Extension(sub_partial, sub_payload) => { // combine with node below - diff.delete_node_from_slice(payload); - MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Extension(NibbleSlice::new_composed(&partial, &sub_partial), sub_payload), diff))) + journal.delete_node(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Extension(NibbleSlice::new_composed(&partial, &sub_partial), sub_payload), journal))) }, Node::Leaf(sub_partial, sub_value) => { // combine with node below - diff.delete_node_from_slice(payload); - MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Leaf(NibbleSlice::new_composed(&partial, &sub_partial), sub_value), diff))) + journal.delete_node(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Leaf(NibbleSlice::new_composed(&partial, &sub_partial), sub_value), journal))) }, // no change, might still have an oversize node inline - fix indirection - _ => Self::fixed_indirection(n, diff), + _ => Self::fixed_indirection(n, journal), } }, // leaf or empty. no change. @@ -739,34 +800,40 @@ impl TrieDB { /// it will just return the new RLP that represents the new node. /// `None` may be returned should no change be needed. /// - /// The database will be updated so as to make the returned RLP valid through inserting + /// `journal` will record the database updates so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// /// **This operation will not insert the new node nor destroy the original.** - fn cleared_from_slice(&self, old: &[u8], partial: &NibbleSlice, diff: &mut Diff) -> Option { - self.cleared(Node::decoded(old), partial, diff) + fn cleared_from_slice(&self, old: &[u8], partial: &NibbleSlice, journal: &mut Journal) -> Option { + self.cleared(Node::decoded(old), partial, journal) } - fn cleared(&self, n: Node, partial: &NibbleSlice, diff: &mut Diff) -> Option { + /// Compose the RLP of the node equivalent to `n` except with the `partial` key removed from its (sub-)trie. + /// + /// `journal` will record the database updates so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn cleared(&self, n: Node, partial: &NibbleSlice, journal: &mut Journal) -> Option { trace!("cleared old={:?}, partial={:?})", n, partial); match (n, partial.is_empty()) { (Node::Empty, _) => None, (Node::Branch(_, None), true) => { None }, - (Node::Branch(payloads, _), true) => Some(Self::encoded(self.fixed(Node::Branch(payloads, None), diff))), // matched as leaf-branch - give back fixed branch with it. + (Node::Branch(payloads, _), true) => Some(Self::encoded(self.fixed(Node::Branch(payloads, None), journal))), // matched as leaf-branch - give back fixed branch with it. (Node::Branch(payloads, value), false) => { // Branch with partial left - route, clear, fix. let i: usize = partial.at(0) as usize; trace!("branch-with-partial node[{:?}]={:?}", i, payloads[i].pretty()); - self.cleared(self.get_node(payloads[i]), &partial.mid(1), diff).map(|new_payload| { + self.cleared(self.get_node(payloads[i]), &partial.mid(1), journal).map(|new_payload| { trace!("branch-new-payload={:?}; delete-old={:?}", new_payload.pretty(), payloads[i].pretty()); // downsteam node needed to be changed. - diff.delete_node_from_slice(payloads[i]); + journal.delete_node(payloads[i]); // return fixed up new node. let mut new_payloads = payloads; new_payloads[i] = &new_payload; - Self::encoded(self.fixed(Node::Branch(new_payloads, value), diff)) + Self::encoded(self.fixed(Node::Branch(new_payloads, value), journal)) }) }, (Node::Leaf(node_partial, _), _) => { @@ -785,12 +852,12 @@ impl TrieDB { cp if cp == node_partial.len() => { trace!("matching-prefix (cp={:?}): SKIP,CLEAR,FIXUP", cp); // key at end of extension - skip, clear, fix - self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), diff).map(|new_payload| { + self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), journal).map(|new_payload| { trace!("extension-new-payload={:?}; delete-old={:?}", new_payload.pretty(), node_payload.pretty()); // downsteam node needed to be changed. - diff.delete_node_from_slice(node_payload); + journal.delete_node(node_payload); // return fixed up new node. - Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), diff)) + Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), journal)) }) }, _ => None, // key in the middle of an extension - doesn't exist. @@ -812,11 +879,23 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - self.add(&NibbleSlice::new(key), value); + match value.is_empty() { + false => self.insert_ns(&NibbleSlice::new(key), value), + true => self.remove_ns(&NibbleSlice::new(key)), + } } fn remove(&mut self, key: &[u8]) { - self.delete(&NibbleSlice::new(key)); + self.remove_ns(&NibbleSlice::new(key)); + } +} + +impl fmt::Debug for TrieDB { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "c={:?} [", self.hash_count)); + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); + writeln!(f, "]") } } @@ -834,9 +913,9 @@ mod tests { use std::collections::HashSet; use bytes::{ToPretty,Bytes}; - fn random_key(alphabet: &[u8], min_count: usize, diff_count: usize) -> Vec { + fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec { let mut ret: Vec = Vec::new(); - let r = min_count + if diff_count > 0 {random::() % diff_count} else {0}; + let r = min_count + if journal_count > 0 {random::() % journal_count} else {0}; for _ in 0..r { ret.push(alphabet[random::() % alphabet.len()]); } @@ -885,18 +964,18 @@ mod tests { fn playpen() { env_logger::init().ok(); - let maps = map!{ - "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, diff_key: 0, count: 1000}, - "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, diff_key: 0, count: 1000}, - "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, diff_key: 0, count: 1000}, - "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, diff_key: 5, count: 1000} + /*let maps = map!{ + "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, journal_key: 0, count: 1000}, + "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, journal_key: 0, count: 1000}, + "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, journal_key: 0, count: 1000}, + "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, journal_key: 5, count: 1000} }; for sm in maps { let m = sm.1.make(); let t = populate_trie(&m); println!("{:?}: root={:?}, hash_count={:?}", sm.0, t.root(), t.hash_count); - }; - panic!(); + };*/ +// panic!(); for test_i in 0..1 { if test_i % 50 == 0 { @@ -1156,7 +1235,7 @@ mod tests { #[test] fn stress() { - for _ in 0..5000 { + for _ in 0..500 { let mut x: Vec<(Vec, Vec)> = Vec::new(); let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { From 36bea4692e01dd84cd52d79b20571a9f128f9c02 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 16:32:06 +0100 Subject: [PATCH 13/47] refector init --- benches/rlp.rs | 2 +- src/rlp/faces.rs | 23 ++++++ src/rlp/mod.rs | 7 ++ src/{rlp.rs => rlp/old.rs} | 15 ++-- src/rlp/rlp.rs | 72 +++++++++++++++++++ src/rlp/rlpstream.rs | 0 src/rlp/untrusted_rlp.rs | 143 +++++++++++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 src/rlp/faces.rs create mode 100644 src/rlp/mod.rs rename src/{rlp.rs => rlp/old.rs} (99%) create mode 100644 src/rlp/rlp.rs create mode 100644 src/rlp/rlpstream.rs create mode 100644 src/rlp/untrusted_rlp.rs diff --git a/benches/rlp.rs b/benches/rlp.rs index 1160e311f..62f6c6112 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -91,6 +91,6 @@ fn bench_stream_1000_empty_lists(b: &mut Bencher) { for _ in 0..1000 { stream.append_list(0); } - let _ = stream.out(); + //let _ = stream.out(); }); } diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs new file mode 100644 index 000000000..8ebe962c5 --- /dev/null +++ b/src/rlp/faces.rs @@ -0,0 +1,23 @@ +pub trait Reader<'a, 'view>: Sized { + type Prototype; + type PayloadInfo; + type Data; + type Item; + + fn new(bytes: &'a [u8]) -> Self; + fn raw(&'view self) -> &'a [u8]; + fn prototype(&self) -> Self::Prototype; + fn payload_info(&self) -> Self::PayloadInfo; + fn data(&'view self) -> Self::Data; + fn item_count(&self) -> usize; + fn size(&self) -> usize; + fn at(&'view self, index: usize) -> Self::Item; + fn is_null(&self) -> bool; + fn is_empty(&self) -> bool; + fn is_list(&self) -> bool; + fn is_data(&self) -> bool; + fn is_int(&self) -> bool; +} + +pub trait Stream { +} diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs new file mode 100644 index 000000000..98e73665f --- /dev/null +++ b/src/rlp/mod.rs @@ -0,0 +1,7 @@ +pub mod old; + +pub mod faces; +pub mod rlp; +pub mod untrusted_rlp; + +pub use self::old::*; diff --git a/src/rlp.rs b/src/rlp/old.rs similarity index 99% rename from src/rlp.rs rename to src/rlp/old.rs index d0dece3ba..63c3040e0 100644 --- a/src/rlp.rs +++ b/src/rlp/old.rs @@ -32,10 +32,10 @@ use std::fmt; use std::cell::Cell; +use std::ops::Deref; use std::error::Error as StdError; use elastic_array::*; use bytes::{ToBytes, FromBytes, FromBytesError}; -use vector::InsertSlice; /// Data-oriented view onto rlp-slice. /// @@ -113,6 +113,14 @@ pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } +//impl<'a> Deref for Rlp<'a> { + //type Target = UntrustedRlp<'a>; + + //fn deref(&self) -> &Self::Target { + //&self.rlp + //} +//} + impl<'a> From> for Rlp<'a> { fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { Rlp { rlp: rlp } @@ -837,8 +845,6 @@ impl RlpStream { /// } /// ``` pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - // push new list - let position = self.encoder.bytes.len(); match len { 0 => { // we may finish, if the appended list len is equal 0 @@ -846,8 +852,7 @@ impl RlpStream { self.note_appended(1); }, _ => { - // reserve at least double size of the len - //self.encoder.bytes.reserve(len * 2); + let position = self.encoder.bytes.len(); self.unfinished_lists.push(ListInfo::new(position, len)); }, } diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs new file mode 100644 index 000000000..603ba425d --- /dev/null +++ b/src/rlp/rlp.rs @@ -0,0 +1,72 @@ +use super::faces::Reader; +use super::untrusted_rlp::*; + +/// Data-oriented view onto trusted rlp-slice. +/// +/// Unlikely to `UntrustedRlp` doesn't bother you with error +/// handling. It assumes that you know what you are doing. +pub struct Rlp<'a> { + rlp: UntrustedRlp<'a> +} + +impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { + type Prototype = Prototype; + type PayloadInfo = PayloadInfo; + type Data = &'a [u8]; + type Item = Rlp<'a>; + + /// Create a new instance of `Rlp` + fn new(bytes: &'a [u8]) -> Rlp<'a> { + Rlp { + rlp: UntrustedRlp::new(bytes) + } + } + + fn raw(&'view self) -> &'a [u8] { + self.rlp.raw() + } + + fn prototype(&self) -> Self::Prototype { + unimplemented!() + } + + fn payload_info(&self) -> Self::PayloadInfo { + unimplemented!() + } + + fn data(&'view self) -> Self::Data { + unimplemented!() + } + + fn item_count(&self) -> usize { + unimplemented!() + } + + fn size(&self) -> usize { + unimplemented!() + } + + fn at(&'view self, index: usize) -> Self::Item { + unimplemented!() + } + + fn is_null(&self) -> bool { + unimplemented!() + } + + fn is_empty(&self) -> bool { + unimplemented!() + } + + fn is_list(&self) -> bool { + unimplemented!() + } + + fn is_data(&self) -> bool { + unimplemented!() + } + + fn is_int(&self) -> bool { + unimplemented!() + } +} diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs new file mode 100644 index 000000000..8d875324e --- /dev/null +++ b/src/rlp/untrusted_rlp.rs @@ -0,0 +1,143 @@ +use std::fmt; +use std::cell::Cell; +use std::error::Error as StdError; +use bytes::{FromBytesError}; +use super::faces::Reader; + +/// rlp offset +#[derive(Copy, Clone, Debug)] +struct OffsetCache { + index: usize, + offset: usize, +} + +impl OffsetCache { + fn new(index: usize, offset: usize) -> OffsetCache { + OffsetCache { + index: index, + offset: offset, + } + } +} + +#[derive(Debug)] +pub enum Prototype { + Null, + Data(usize), + List(usize), +} + +/// Stores basic information about item +pub struct PayloadInfo { + pub header_len: usize, + pub value_len: usize, +} + +impl PayloadInfo { + fn new(header_len: usize, value_len: usize) -> PayloadInfo { + PayloadInfo { + header_len: header_len, + value_len: value_len, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} + +/// Data-oriented view onto rlp-slice. +/// +/// This is immutable structere. No operations change it. +/// +/// Should be used in places where, error handling is required, +/// eg. on input +#[derive(Debug)] +pub struct UntrustedRlp<'a> { + bytes: &'a [u8], + cache: Cell, +} + +impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { + type Prototype = Result; + type PayloadInfo = Result; + type Data = Result<&'a [u8], DecoderError>; + type Item = Result, DecoderError>; + + //returns new instance of `UntrustedRlp` + fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { + UntrustedRlp { + bytes: bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), + } + } + + fn raw(&'view self) -> &'a [u8] { + self.bytes + } + + fn prototype(&self) -> Self::Prototype { + unimplemented!() + } + + fn payload_info(&self) -> Self::PayloadInfo { + unimplemented!() + } + + fn data(&'view self) -> Self::Data { + unimplemented!() + } + + fn item_count(&self) -> usize { + unimplemented!() + } + + fn size(&self) -> usize { + unimplemented!() + } + + fn at(&'view self, index: usize) -> Self::Item { + unimplemented!() + } + + fn is_null(&self) -> bool { + unimplemented!() + } + + fn is_empty(&self) -> bool { + unimplemented!() + } + + fn is_list(&self) -> bool { + unimplemented!() + } + + fn is_data(&self) -> bool { + unimplemented!() + } + + fn is_int(&self) -> bool { + unimplemented!() + } +} From 03674cad3be346b00e19641a7c254dbaa15fbc85 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 7 Dec 2015 17:20:15 +0100 Subject: [PATCH 14/47] Documentation. --- src/error.rs | 3 +++ src/hash.rs | 6 +++++- src/hashdb.rs | 2 ++ src/trie.rs | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index f6c64a54f..71514d045 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,3 +1,5 @@ +//! General error types for use in ethcore. + use rustc_serialize::hex::*; #[derive(Debug)] @@ -6,6 +8,7 @@ pub enum BaseDataError { } #[derive(Debug)] +/// General error type which should be capable of representing all errors in ethcore. pub enum EthcoreError { FromHex(FromHexError), BaseData(BaseDataError), diff --git a/src/hash.rs b/src/hash.rs index 5662d713c..b51bbdacf 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,3 +1,5 @@ +//! General hash types, a fixed-size raw-data type used as the output of hash functions. + use std::str::FromStr; use std::fmt; use std::ops; @@ -11,7 +13,9 @@ use bytes::BytesConvertable; use math::log2; use uint::U256; -/// types implementing FixedHash must be also BytesConvertable +/// Trait for a fixed-size byte array to be used as the output of hash functions. +/// +/// Note: types implementing `FixedHash` must be also `BytesConvertable`. pub trait FixedHash: Sized + BytesConvertable { fn new() -> Self; fn random() -> Self; diff --git a/src/hashdb.rs b/src/hashdb.rs index 207883e4b..e63d031e5 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,7 +1,9 @@ +//! Database of byte-slices keyed to their Keccak hash. use hash::*; use bytes::*; use std::collections::HashMap; +/// Trait modelling datastore keyed by a 32-byte Keccak hash. pub trait HashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; diff --git a/src/trie.rs b/src/trie.rs index 0425565bc..5c4420787 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,3 +1,4 @@ +//! Key-value datastore with a modified Merkle tree. extern crate rand; use std::fmt; From 7e287755ca491bde2c9a27bb91c2bc43f64c1c4a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 23:47:26 +0100 Subject: [PATCH 15/47] rlp view interface + refactor --- src/rlp/faces.rs | 179 +++++++++++++++++++++++++++++- src/rlp/mod.rs | 1 + src/rlp/rlp.rs | 83 +++++++++++--- src/rlp/untrusted_rlp.rs | 228 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 462 insertions(+), 29 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 8ebe962c5..69e39b06d 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -1,23 +1,198 @@ -pub trait Reader<'a, 'view>: Sized { +pub trait Decoder { + type Error; + + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result; + + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result; +} + +pub trait Decodable: Sized { + fn decode(decoder: &D) -> Result where D: Decoder; +} + +pub trait View<'a, 'view>: Sized { type Prototype; type PayloadInfo; type Data; type Item; + type Iter; + type Error; + /// Creates a new instance of `Rlp` reader fn new(bytes: &'a [u8]) -> Self; + + /// The raw data of the RLP. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = rlp.at(1).raw(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` fn raw(&'view self) -> &'a [u8]; + + /// Get the prototype of the RLP. fn prototype(&self) -> Self::Prototype; + fn payload_info(&self) -> Self::PayloadInfo; + fn data(&'view self) -> Self::Data; + + /// Returns number of RLP items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.item_count(), 2); + /// let view = rlp.at(1); + /// assert_eq!(view.item_count(), 0); + /// } + /// ``` fn item_count(&self) -> usize; + + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1); + /// assert_eq!(view.size(), 3); + /// } + /// ``` fn size(&self) -> usize; + + /// Get view onto RLP-slice at index. + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = String::decode(&rlp.at(1)); + /// assert_eq!(dog, "dog".to_string()); + /// } fn at(&'view self, index: usize) -> Self::Item; + + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` fn is_null(&self) -> bool; + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` fn is_empty(&self) -> bool; + + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` fn is_list(&self) -> bool; + + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.at(1).is_data()); + /// } + /// ``` fn is_data(&self) -> bool; + + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).is_int(), true); + /// } + /// ``` fn is_int(&self) -> bool; + + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// } + /// ``` + fn iter(&'view self) -> Self::Iter; + + fn as_val(&self) -> Result where T: Decodable; } -pub trait Stream { +pub trait Encoder { + fn emit_value(&mut self, bytes: &[u8]) -> (); + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); +} + +pub trait Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder; +} + +pub trait Stream: Sized { + fn new() -> Self; + fn new_list(len: usize) -> Self; + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self; } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 98e73665f..af6da14a2 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -1,6 +1,7 @@ pub mod old; pub mod faces; +pub mod coders; pub mod rlp; pub mod untrusted_rlp; diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 603ba425d..54807bfda 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,19 +1,28 @@ -use super::faces::Reader; +use super::faces::{View, Decodable}; use super::untrusted_rlp::*; +impl<'a> From> for Rlp<'a> { + fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { + Rlp { rlp: rlp } + } +} + /// Data-oriented view onto trusted rlp-slice. /// /// Unlikely to `UntrustedRlp` doesn't bother you with error /// handling. It assumes that you know what you are doing. +#[derive(Debug)] pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } -impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { +impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { type Prototype = Prototype; type PayloadInfo = PayloadInfo; type Data = &'a [u8]; type Item = Rlp<'a>; + type Iter = RlpIterator<'a, 'view>; + type Error = DecoderError; /// Create a new instance of `Rlp` fn new(bytes: &'a [u8]) -> Rlp<'a> { @@ -27,46 +36,94 @@ impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { } fn prototype(&self) -> Self::Prototype { - unimplemented!() + self.rlp.prototype().unwrap() } fn payload_info(&self) -> Self::PayloadInfo { - unimplemented!() + self.rlp.payload_info().unwrap() } fn data(&'view self) -> Self::Data { - unimplemented!() + self.rlp.data().unwrap() } fn item_count(&self) -> usize { - unimplemented!() + self.rlp.item_count() } fn size(&self) -> usize { - unimplemented!() + self.rlp.size() } fn at(&'view self, index: usize) -> Self::Item { - unimplemented!() + From::from(self.rlp.at(index).unwrap()) } fn is_null(&self) -> bool { - unimplemented!() + self.rlp.is_null() } fn is_empty(&self) -> bool { - unimplemented!() + self.rlp.is_empty() } fn is_list(&self) -> bool { - unimplemented!() + self.rlp.is_list() } fn is_data(&self) -> bool { - unimplemented!() + self.rlp.is_data() } fn is_int(&self) -> bool { - unimplemented!() + self.rlp.is_int() + } + + fn iter(&'view self) -> Self::Iter { + self.into_iter() + } + + fn as_val(&self) -> Result where T: Decodable { + self.rlp.as_val() + } +} + +impl <'a, 'view> Rlp<'a> where 'a: 'view { + fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { + let res: Result = r.as_val(); + res.unwrap_or_else(|_| panic!()) + } + + pub fn as_val(&self) -> T where T: Decodable { + Self::reader_as_val(self) + } +} + +/// Iterator over trusted rlp-slice list elements. +pub struct RlpIterator<'a, 'view> where 'a: 'view { + rlp: &'view Rlp<'a>, + index: usize +} + +impl<'a, 'view> IntoIterator for &'view Rlp<'a> where 'a: 'view { + type Item = Rlp<'a>; + type IntoIter = RlpIterator<'a, 'view>; + + fn into_iter(self) -> Self::IntoIter { + RlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a, 'view> Iterator for RlpIterator<'a, 'view> { + type Item = Rlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); + self.index += 1; + result } } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 8d875324e..910696565 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,8 +1,8 @@ use std::fmt; use std::cell::Cell; use std::error::Error as StdError; -use bytes::{FromBytesError}; -use super::faces::Reader; +use bytes::{FromBytes, FromBytesError}; +use super::faces::{View, Decoder, Decodable}; /// rlp offset #[derive(Copy, Clone, Debug)] @@ -79,11 +79,22 @@ pub struct UntrustedRlp<'a> { cache: Cell, } -impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { +impl<'a> Clone for UntrustedRlp<'a> { + fn clone(&self) -> UntrustedRlp<'a> { + UntrustedRlp { + bytes: self.bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)) + } + } +} + +impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { type Prototype = Result; type PayloadInfo = Result; type Data = Result<&'a [u8], DecoderError>; type Item = Result, DecoderError>; + type Iter = UntrustedRlpIterator<'a, 'view>; + type Error = DecoderError; //returns new instance of `UntrustedRlp` fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { @@ -98,46 +109,235 @@ impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { } fn prototype(&self) -> Self::Prototype { - unimplemented!() + // optimize? && return appropriate errors + if self.is_data() { + Ok(Prototype::Data(self.size())) + } else if self.is_list() { + Ok(Prototype::List(self.item_count())) + } else { + Ok(Prototype::Null) + } } fn payload_info(&self) -> Self::PayloadInfo { - unimplemented!() + BasicDecoder::payload_info(self.bytes) } fn data(&'view self) -> Self::Data { - unimplemented!() + let pi = try!(BasicDecoder::payload_info(self.bytes)); + Ok(&self.bytes[pi.header_len..(pi.header_len + pi.value_len)]) } fn item_count(&self) -> usize { - unimplemented!() + match self.is_list() { + true => self.iter().count(), + false => 0 + } } fn size(&self) -> usize { - unimplemented!() + match self.is_data() { + // we can safely unwrap (?) cause its data + true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, + false => 0 + } } fn at(&'view self, index: usize) -> Self::Item { - unimplemented!() + if !self.is_list() { + return Err(DecoderError::RlpExpectedToBeList); + } + + // move to cached position if it's index is less or equal to + // current search index, otherwise move to beginning of list + let c = self.cache.get(); + let (mut bytes, to_skip) = match c.index <= index { + true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), + false => (try!(self.consume_list_prefix()), index), + }; + + // skip up to x items + bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); + + // update the cache + self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); + + // construct new rlp + let found = try!(BasicDecoder::payload_info(bytes)); + Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) } fn is_null(&self) -> bool { - unimplemented!() + self.bytes.len() == 0 } fn is_empty(&self) -> bool { - unimplemented!() + !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) } fn is_list(&self) -> bool { - unimplemented!() + !self.is_null() && self.bytes[0] >= 0xc0 } fn is_data(&self) -> bool { - unimplemented!() + !self.is_null() && self.bytes[0] < 0xc0 } fn is_int(&self) -> bool { - unimplemented!() + if self.is_null() { + return false; + } + + match self.bytes[0] { + 0...0x80 => true, + 0x81...0xb7 => self.bytes[1] != 0, + b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, + _ => false + } + } + + fn iter(&'view self) -> Self::Iter { + self.into_iter() + } + + fn as_val(&self) -> Result where T: Decodable { + // optimize, so it doesn't use clone (although This clone is cheap) + T::decode(&BasicDecoder::new(self.clone())) } } + +impl<'a> UntrustedRlp<'a> { + /// consumes first found prefix + fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { + let item = try!(BasicDecoder::payload_info(self.bytes)); + let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); + Ok(bytes) + } + + /// consumes fixed number of items + fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { + let mut result = bytes; + for _ in 0..items { + let i = try!(BasicDecoder::payload_info(result)); + result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); + } + Ok(result) + } + + + /// consumes slice prefix of length `len` + fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { + match bytes.len() >= len { + true => Ok(&bytes[len..]), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + +/// Iterator over rlp-slice list elements. +pub struct UntrustedRlpIterator<'a, 'view> where 'a: 'view { + rlp: &'view UntrustedRlp<'a>, + index: usize, +} + +impl<'a, 'view> IntoIterator for &'view UntrustedRlp<'a> where 'a: 'view { + type Item = UntrustedRlp<'a>; + type IntoIter = UntrustedRlpIterator<'a, 'view>; + + fn into_iter(self) -> Self::IntoIter { + UntrustedRlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a, 'view> Iterator for UntrustedRlpIterator<'a, 'view> { + type Item = UntrustedRlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.at(index).ok(); + self.index += 1; + result + } +} + +struct BasicDecoder<'a> { + rlp: UntrustedRlp<'a> +} + +impl<'a> BasicDecoder<'a> { + pub fn new(rlp: UntrustedRlp<'a>) -> BasicDecoder<'a> { + BasicDecoder { + rlp: rlp + } + } + + /// Return first item info + fn payload_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + }, + // we cant reach this place, but rust requires _ to be implemented + _ => { panic!(); } + }; + + match item.header_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + +impl<'a> Decoder for BasicDecoder<'a> { + type Error = DecoderError; + + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result { + + let bytes = self.rlp.raw(); + + match bytes.first().map(|&x| x) { + // rlp is too short + None => Err(DecoderError::RlpIsTooShort), + // single byt value + Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), + // 0-55 bytes + Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), + // longer than 55 bytes + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let begin_of_value = 1 as usize + len_of_len; + let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); + Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) + } + // we are reading value, not a list! + _ => { unreachable!(); } + } + } + + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result { + + let v: Vec> = self.rlp.iter() + .map(| i | BasicDecoder::new(i)) + .collect(); + f(&v) + } +} + From 0de75185aad990fcd8a43a69e59e527cc9190807 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:07:07 +0100 Subject: [PATCH 16/47] decodable trait impl in progress --- src/rlp/faces.rs | 45 +++++++++++++++++++++++------ src/rlp/mod.rs | 34 +++++++++++++++++++++- src/rlp/rlp.rs | 7 ++--- src/rlp/untrusted_rlp.rs | 61 ++++++++++++++++------------------------ 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 69e39b06d..d72f2f96c 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -1,15 +1,43 @@ +use std::fmt; +use std::error::Error as StdError; +use bytes::FromBytesError; + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} + +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} + pub trait Decoder { - type Error; + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result; - fn read_value(&self, f: F) -> Result - where F: FnOnce(&[u8]) -> Result; - - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result; + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result; } pub trait Decodable: Sized { - fn decode(decoder: &D) -> Result where D: Decoder; + fn decode(decoder: &D) -> Result where D: Decoder; } pub trait View<'a, 'view>: Sized { @@ -18,7 +46,6 @@ pub trait View<'a, 'view>: Sized { type Data; type Item; type Iter; - type Error; /// Creates a new instance of `Rlp` reader fn new(bytes: &'a [u8]) -> Self; @@ -179,7 +206,7 @@ pub trait View<'a, 'view>: Sized { /// ``` fn iter(&'view self) -> Self::Iter; - fn as_val(&self) -> Result where T: Decodable; + fn as_val(&self) -> Result where T: Decodable; } pub trait Encoder { diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index af6da14a2..98cb43e80 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -1,8 +1,40 @@ +//! Rlp serialization module +//! +//! Allows encoding, decoding, and view onto rlp-slice +//! +//!# What should you use when? +//! +//!### Use `encode` function when: +//! * You want to encode something inline. +//! * You do not work on big set of data. +//! * You want to encode whole data structure at once. +//! +//!### Use `decode` function when: +//! * You want to decode something inline. +//! * You do not work on big set of data. +//! * You want to decode whole rlp at once. +//! +//!### Use `RlpStream` when: +//! * You want to encode something in portions. +//! * You encode a big set of data. +//! +//!### Use `Rlp` when: +//! * You are working on trusted data (not corrupted). +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. +//! +//!### Use `UntrustedRlp` when: +//! * You are working on untrusted data (~corrupted). +//! * You need to handle data corruption errors. +//! * You are working on input data. +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. + pub mod old; pub mod faces; -pub mod coders; pub mod rlp; pub mod untrusted_rlp; +pub mod rlpstream; pub use self::old::*; diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 54807bfda..2effb7297 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,4 +1,4 @@ -use super::faces::{View, Decodable}; +use super::faces::{View, Decodable, DecoderError}; use super::untrusted_rlp::*; impl<'a> From> for Rlp<'a> { @@ -22,7 +22,6 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { type Data = &'a [u8]; type Item = Rlp<'a>; type Iter = RlpIterator<'a, 'view>; - type Error = DecoderError; /// Create a new instance of `Rlp` fn new(bytes: &'a [u8]) -> Rlp<'a> { @@ -83,14 +82,14 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { self.into_iter() } - fn as_val(&self) -> Result where T: Decodable { + fn as_val(&self) -> Result where T: Decodable { self.rlp.as_val() } } impl <'a, 'view> Rlp<'a> where 'a: 'view { fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { - let res: Result = r.as_val(); + let res: Result = r.as_val(); res.unwrap_or_else(|_| panic!()) } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 910696565..89c3b664f 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,8 +1,6 @@ -use std::fmt; use std::cell::Cell; -use std::error::Error as StdError; -use bytes::{FromBytes, FromBytesError}; -use super::faces::{View, Decoder, Decodable}; +use bytes::{FromBytes}; +use super::faces::{View, Decoder, Decodable, DecoderError}; /// rlp offset #[derive(Copy, Clone, Debug)] @@ -42,31 +40,6 @@ impl PayloadInfo { } } -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} - /// Data-oriented view onto rlp-slice. /// /// This is immutable structere. No operations change it. @@ -94,7 +67,6 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { type Data = Result<&'a [u8], DecoderError>; type Item = Result, DecoderError>; type Iter = UntrustedRlpIterator<'a, 'view>; - type Error = DecoderError; //returns new instance of `UntrustedRlp` fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { @@ -200,7 +172,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { self.into_iter() } - fn as_val(&self) -> Result where T: Decodable { + fn as_val(&self) -> Result where T: Decodable { // optimize, so it doesn't use clone (although This clone is cheap) T::decode(&BasicDecoder::new(self.clone())) } @@ -305,10 +277,8 @@ impl<'a> BasicDecoder<'a> { } impl<'a> Decoder for BasicDecoder<'a> { - type Error = DecoderError; - - fn read_value(&self, f: F) -> Result - where F: FnOnce(&[u8]) -> Result { + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result { let bytes = self.rlp.raw(); @@ -331,8 +301,8 @@ impl<'a> Decoder for BasicDecoder<'a> { } } - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result { + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result { let v: Vec> = self.rlp.iter() .map(| i | BasicDecoder::new(i)) @@ -341,3 +311,20 @@ impl<'a> Decoder for BasicDecoder<'a> { } } +impl Decodable for T where T: FromBytes { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} + +impl Decodable for Vec where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} + +impl Decodable for Vec { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} From cb5ec8c0afe77c6221d0c6facb01bd8086739661 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:27:12 +0100 Subject: [PATCH 17/47] decodable implementations --- src/rlp/faces.rs | 2 +- src/rlp/untrusted_rlp.rs | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index d72f2f96c..392908f16 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -37,7 +37,7 @@ pub trait Decoder { } pub trait Decodable: Sized { - fn decode(decoder: &D) -> Result where D: Decoder; + fn decode(decoder: &D) -> Result where D: Decoder; } pub trait View<'a, 'view>: Sized { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 89c3b664f..fd385b99d 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -312,19 +312,27 @@ impl<'a> Decoder for BasicDecoder<'a> { } impl Decodable for T where T: FromBytes { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + Ok(try!(T::from_bytes(bytes))) + }) } } impl Decodable for Vec where T: Decodable { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_list(| decoders | { + decoders.iter().map(|d| T::decode(d)).collect() + }) } } impl Decodable for Vec { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + let mut res = vec![]; + res.extend(bytes); + Ok(res) + }) } } From 4419924154fdf3edf7efed68401a9620e38ca02b Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:45:04 +0100 Subject: [PATCH 18/47] decoder refactor applied --- src/overlaydb.rs | 2 +- src/rlp/mod.rs | 12 +++++++++++- src/rlp/old.rs | 26 +++++++++++++++----------- src/trie.rs | 8 ++++---- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index c13acfd6a..b5bbf4299 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -122,7 +122,7 @@ impl OverlayDB { .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { let r = Rlp::new(d.deref()); - (Bytes::decode(&r.at(1)), u32::decode(&r.at(0))) + (r.at(1).as_val(), r.at(0).as_val()) }) } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 98cb43e80..e14b4ad25 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -37,4 +37,14 @@ pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::old::*; +pub use self::faces::{DecoderError, Decoder, Decodable, View}; +pub use self::rlp::*; +pub use self::untrusted_rlp::*; + +pub use self::old::{encode, RlpStream, Encodable}; +//pub use self::old::*; + +pub fn decode(bytes: &[u8]) -> T where T: Decodable { + let rlp = Rlp::new(bytes); + rlp.as_val() +} diff --git a/src/rlp/old.rs b/src/rlp/old.rs index 63c3040e0..dd09ea548 100644 --- a/src/rlp/old.rs +++ b/src/rlp/old.rs @@ -1119,7 +1119,7 @@ mod tests { use std::{fmt, cmp}; use std::str::FromStr; use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable}; + use rlp::{UntrustedRlp, RlpStream, Decodable, View}; use uint::U256; #[test] @@ -1128,23 +1128,27 @@ mod tests { { let rlp = UntrustedRlp::new(&data); assert!(rlp.is_list()); - let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + let animals: Vec = rlp.as_val().unwrap(); assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); assert!(cat.is_data()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); assert!(dog.is_data()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); - assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); } } @@ -1172,18 +1176,18 @@ mod tests { let cat = iter.next().unwrap(); assert!(cat.is_data()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); let dog = iter.next().unwrap(); assert!(dog.is_data()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); let none = iter.next(); assert!(none.is_none()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); } } diff --git a/src/trie.rs b/src/trie.rs index a340d78f7..0cbfca189 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -131,7 +131,7 @@ impl Diff { fn delete_node(&mut self, old: &Rlp) { if old.is_data() && old.size() == 32 { - self.delete_node_sha3(H256::decode(old)); + self.delete_node_sha3(old.as_val()); } } @@ -310,7 +310,7 @@ impl TrieDB { let mut handle_payload = |payload| { let p = Rlp::new(payload); if p.is_data() && p.size() == 32 { - acc.push(H256::decode(&p)); + acc.push(p.as_val()); } self.accumulate_keys(self.get_node(payload), acc); @@ -417,7 +417,7 @@ impl TrieDB { // check if its sha3 + len let r = Rlp::new(node); match r.is_data() && r.size() == 32 { - true => self.db.lookup(&H256::decode(&r)).expect("Not found!"), + true => self.db.lookup(&r.as_val::()).expect("Not found!"), false => node } } @@ -495,7 +495,7 @@ impl TrieDB { rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { - let h = H256::decode(rlp); + let h = rlp.as_val(); let r = self.db.lookup(&h).unwrap_or_else(||{ println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); println!("Diff: {:?}", diff); From 701aaf126d2977802ec46d15b85d74668c669800 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:53:28 +0100 Subject: [PATCH 19/47] fixed rlp reexports --- src/rlp/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index e14b4ad25..a790a480c 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -38,11 +38,10 @@ pub mod untrusted_rlp; pub mod rlpstream; pub use self::faces::{DecoderError, Decoder, Decodable, View}; -pub use self::rlp::*; -pub use self::untrusted_rlp::*; +pub use self::rlp::{Rlp, RlpIterator}; +pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::old::{encode, RlpStream, Encodable}; -//pub use self::old::*; pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); From 51eff239fa3c6416aaf61e297605744ae0b26d97 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:33:05 +0100 Subject: [PATCH 20/47] rlpstream separated to its own submodule --- src/rlp/faces.rs | 92 ++++++++++++++- src/rlp/mod.rs | 36 +++++- src/rlp/rlpstream.rs | 247 +++++++++++++++++++++++++++++++++++++++ src/rlp/untrusted_rlp.rs | 4 +- src/triehash.rs | 2 +- 5 files changed, 375 insertions(+), 6 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 392908f16..f55b5584e 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -219,7 +219,97 @@ pub trait Encodable { } pub trait Stream: Sized { + + /// Initializes instance of empty `Stream`. fn new() -> Self; + + /// Initializes the `Stream` as a list. fn new_list(len: usize) -> Self; - fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self; + + /// Apends value to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + /// ``` + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self where E: Encodable; + + /// Declare appending the list of given size, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); + /// } + /// ``` + fn append_list<'a>(&'a mut self, len: usize) -> &'a mut Self; + + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` + fn append_empty_data<'a>(&'a mut self) -> &'a mut Self; + + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; + + /// Clear the output stream so far. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// } + fn clear(&mut self); + + /// Returns true if stream doesnt expect any more items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + fn is_finished(&self) -> bool; + + /// Streams out encoded bytes. + /// + /// panic! if stream is not finished. + fn out(self) -> Vec; } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index a790a480c..c7d657fa0 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -37,13 +37,45 @@ pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::faces::{DecoderError, Decoder, Decodable, View}; +pub use self::faces::{DecoderError, Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::rlp::{Rlp, RlpIterator}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; +pub use self::rlpstream::{RlpStream}; -pub use self::old::{encode, RlpStream, Encodable}; +//pub use self::old::{encode, RlpStream, Encodable}; +/// Shortcut function to decode trusted rlp +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +/// let animals: Vec = decode(&data); +/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); +/// } +/// ``` pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); rlp.as_val() } + +/// Shortcut function to encode structure into rlp. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let animals = vec!["cat", "dog"]; +/// let out = encode(&animals); +/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +/// } +/// ``` +pub fn encode(object: &E) -> Vec where E: Encodable +{ + let mut stream = RlpStream::new(); + stream.append(object); + stream.out() +} diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index e69de29bb..379852083 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -0,0 +1,247 @@ +use elastic_array::*; +use bytes::ToBytes; +use super::faces::{Stream, Encoder, Encodable}; + +#[derive(Debug, Copy, Clone)] +struct ListInfo { + position: usize, + current: usize, + max: usize, +} + +impl ListInfo { + fn new(position: usize, max: usize) -> ListInfo { + ListInfo { + position: position, + current: 0, + max: max, + } + } +} + +/// Appendable rlp encoder. +pub struct RlpStream { + unfinished_lists: ElasticArray16, + encoder: BasicEncoder, +} + +impl Stream for RlpStream { + fn new() -> Self { + RlpStream { + unfinished_lists: ElasticArray16::new(), + encoder: BasicEncoder::new(), + } + } + + fn new_list(len: usize) -> Self { + let mut stream = RlpStream::new(); + stream.append_list(len); + stream + } + + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { + // encode given value and add it at the end of the stream + object.encode(&mut self.encoder); + + // if list is finished, prepend the length + self.note_appended(1); + + // return chainable self + self + } + + fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { + match len { + 0 => { + // we may finish, if the appended list len is equal 0 + self.encoder.bytes.push(0xc0u8); + self.note_appended(1); + }, + _ => { + let position = self.encoder.bytes.len(); + self.unfinished_lists.push(ListInfo::new(position, len)); + }, + } + + // return chainable self + self + } + + fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { + // self push raw item + self.encoder.bytes.push(0x80); + + // try to finish and prepend the length + self.note_appended(1); + + // return chainable self + self + } + + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + // push raw items + self.encoder.bytes.append_slice(bytes); + + // try to finish and prepend the length + self.note_appended(item_count); + + // return chainable self + self + } + + fn clear(&mut self) { + // clear bytes + self.encoder.bytes.clear(); + + // clear lists + self.unfinished_lists.clear(); + } + + fn is_finished(&self) -> bool { + self.unfinished_lists.len() == 0 + } + + fn out(self) -> Vec { + match self.is_finished() { + true => self.encoder.out().to_vec(), + false => panic!() + } + } +} + +impl RlpStream { + + /// Try to finish lists + fn note_appended(&mut self, inserted_items: usize) -> () { + if self.unfinished_lists.len() == 0 { + return; + } + + let back = self.unfinished_lists.len() - 1; + let should_finish = match self.unfinished_lists.get_mut(back) { + None => false, + Some(ref mut x) => { + x.current += inserted_items; + if x.current > x.max { + panic!("You cannot append more items then you expect!"); + } + x.current == x.max + } + }; + + if should_finish { + let x = self.unfinished_lists.pop().unwrap(); + let len = self.encoder.bytes.len() - x.position; + self.encoder.insert_list_len_at_pos(len, x.position); + self.note_appended(1); + } + } +} + +struct BasicEncoder { + bytes: ElasticArray1024, +} + +impl BasicEncoder { + fn new() -> BasicEncoder { + BasicEncoder { bytes: ElasticArray1024::new() } + } + + /// inserts list prefix at given position + /// TODO: optimise it further? + fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { + let mut res = vec![]; + match len { + 0...55 => res.push(0xc0u8 + len as u8), + _ => { + res.push(0xf7u8 + len.to_bytes_len() as u8); + res.extend(len.to_bytes()); + } + }; + + self.bytes.insert_slice(pos, &res); + } + + /// get encoded value + fn out(self) -> ElasticArray1024 { + self.bytes + } +} + +impl Encoder for BasicEncoder { + fn emit_value(&mut self, bytes: &[u8]) -> () { + match bytes.len() { + // just 0 + 0 => self.bytes.push(0x80u8), + // byte is its own encoding + 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), + // (prefix + length), followed by the string + len @ 1 ... 55 => { + self.bytes.push(0x80u8 + len as u8); + self.bytes.append_slice(bytes); + } + // (prefix + length of length), followed by the length, followd by the string + len => { + self.bytes.push(0xb7 + len.to_bytes_len() as u8); + self.bytes.append_slice(&len.to_bytes()); + self.bytes.append_slice(bytes); + } + } + } + + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () + { + // get len before inserting a list + let before_len = self.bytes.len(); + + // insert all list elements + f(self); + + // get len after inserting a list + let after_len = self.bytes.len(); + + // diff is list len + let list_len = after_len - before_len; + self.insert_list_len_at_pos(list_len, before_len); + } +} + +impl Encodable for T where T: ToBytes { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(&self.to_bytes()) + } +} + +impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_list(|e| { + // insert all list elements + for el in self.iter() { + el.encode(e); + } + }) + } +} + +impl Encodable for Vec where T: Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + let r: &[T] = self.as_ref(); + r.encode(encoder) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl<'a> Encodable for &'a [u8] { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl Encodable for Vec { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index fd385b99d..9e2e2f522 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -266,7 +266,7 @@ impl<'a> BasicDecoder<'a> { PayloadInfo::new(header_len, value_len) }, // we cant reach this place, but rust requires _ to be implemented - _ => { panic!(); } + _ => { unreachable!(); } }; match item.header_len + item.value_len <= bytes.len() { @@ -297,7 +297,7 @@ impl<'a> Decoder for BasicDecoder<'a> { Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } // we are reading value, not a list! - _ => { unreachable!(); } + _ => Err(DecoderError::RlpExpectedToBeData) } } diff --git a/src/triehash.rs b/src/triehash.rs index eea79e9d4..67ffcc666 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -7,7 +7,7 @@ use std::cmp; use hash::*; use sha3::*; use rlp; -use rlp::RlpStream; +use rlp::{RlpStream, Stream}; use vector::SharedPrefix; /// Generates a trie root hash for a vector of values From db762c534eae97ddd199f3080d0e7382ce11fd2a Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:40:11 +0100 Subject: [PATCH 21/47] docs tests are passing --- src/rlp/faces.rs | 4 ++-- src/rlp/old.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index f55b5584e..078620de2 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -116,7 +116,7 @@ pub trait View<'a, 'view>: Sized { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let dog = String::decode(&rlp.at(1)); + /// let dog: String = rlp.at(1).as_val(); /// assert_eq!(dog, "dog".to_string()); /// } fn at(&'view self, index: usize) -> Self::Item; @@ -201,7 +201,7 @@ pub trait View<'a, 'view>: Sized { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// let strings: Vec = rlp.iter().map(| i | i.as_val()).collect(); /// } /// ``` fn iter(&'view self) -> Self::Iter; diff --git a/src/rlp/old.rs b/src/rlp/old.rs index dd09ea548..1c4147463 100644 --- a/src/rlp/old.rs +++ b/src/rlp/old.rs @@ -1119,7 +1119,7 @@ mod tests { use std::{fmt, cmp}; use std::str::FromStr; use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable, View}; + use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; use uint::U256; #[test] From 36c586eb217f36dd77524c2ebd18da8742e86b15 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:59:19 +0100 Subject: [PATCH 22/47] final changes in rlp refactor --- src/rlp/errors.rs | 29 + src/rlp/mod.rs | 13 +- src/rlp/old.rs | 1461 ------------------------------- src/rlp/rlp.rs | 3 +- src/rlp/rlpstream.rs | 2 +- src/rlp/tests.rs | 345 ++++++++ src/rlp/{faces.rs => traits.rs} | 30 +- src/rlp/untrusted_rlp.rs | 2 +- 8 files changed, 385 insertions(+), 1500 deletions(-) create mode 100644 src/rlp/errors.rs delete mode 100644 src/rlp/old.rs create mode 100644 src/rlp/tests.rs rename src/rlp/{faces.rs => traits.rs} (92%) diff --git a/src/rlp/errors.rs b/src/rlp/errors.rs new file mode 100644 index 000000000..ada3c2a47 --- /dev/null +++ b/src/rlp/errors.rs @@ -0,0 +1,29 @@ +use std::fmt; +use std::error::Error as StdError; +use bytes::FromBytesError; + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} + +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index c7d657fa0..49043d65f 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -30,20 +30,21 @@ //! * You want to get view onto rlp-slice. //! * You don't want to decode whole rlp at once. -pub mod old; - -pub mod faces; +pub mod errors; +pub mod traits; pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::faces::{DecoderError, Decoder, Decodable, View, Stream, Encodable, Encoder}; +#[cfg(test)] +mod tests; + +pub use self::errors::DecoderError; +pub use self::traits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::rlp::{Rlp, RlpIterator}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::rlpstream::{RlpStream}; -//pub use self::old::{encode, RlpStream, Encodable}; - /// Shortcut function to decode trusted rlp /// /// ```rust diff --git a/src/rlp/old.rs b/src/rlp/old.rs deleted file mode 100644 index 1c4147463..000000000 --- a/src/rlp/old.rs +++ /dev/null @@ -1,1461 +0,0 @@ -//! Rlp serialization module -//! -//! Allows encoding, decoding, and view onto rlp-slice -//! -//!# What should you use when? -//! -//!### Use `encode` function when: -//! * You want to encode something inline. -//! * You do not work on big set of data. -//! * You want to encode whole data structure at once. -//! -//!### Use `decode` function when: -//! * You want to decode something inline. -//! * You do not work on big set of data. -//! * You want to decode whole rlp at once. -//! -//!### Use `RlpStream` when: -//! * You want to encode something in portions. -//! * You encode a big set of data. -//! -//!### Use `Rlp` when: -//! * You are working on trusted data (not corrupted). -//! * You want to get view onto rlp-slice. -//! * You don't want to decode whole rlp at once. -//! -//!### Use `UntrustedRlp` when: -//! * You are working on untrusted data (~corrupted). -//! * You need to handle data corruption errors. -//! * You are working on input data. -//! * You want to get view onto rlp-slice. -//! * You don't want to decode whole rlp at once. - -use std::fmt; -use std::cell::Cell; -use std::ops::Deref; -use std::error::Error as StdError; -use elastic_array::*; -use bytes::{ToBytes, FromBytes, FromBytesError}; - -/// Data-oriented view onto rlp-slice. -/// -/// This is immutable structere. No operations change it. -/// -/// Should be used in places where, error handling is required, -/// eg. on input -#[derive(Debug)] -pub struct UntrustedRlp<'a> { - bytes: &'a [u8], - cache: Cell, -} - -/// rlp offset -#[derive(Copy, Clone, Debug)] -struct OffsetCache { - index: usize, - offset: usize, -} - -impl OffsetCache { - fn new(index: usize, offset: usize) -> OffsetCache { - OffsetCache { - index: index, - offset: offset, - } - } -} - -/// Stores basic information about item -pub struct PayloadInfo { - pub header_len: usize, - pub value_len: usize, -} - -impl PayloadInfo { - fn new(header_len: usize, value_len: usize) -> PayloadInfo { - PayloadInfo { - header_len: header_len, - value_len: value_len, - } - } -} - -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} - -/// Data-oriented view onto trusted rlp-slice. -/// -/// Unlikely to `UntrustedRlp` doesn't bother you with error -/// handling. It assumes that you know what you are doing. -pub struct Rlp<'a> { - rlp: UntrustedRlp<'a> -} - -//impl<'a> Deref for Rlp<'a> { - //type Target = UntrustedRlp<'a>; - - //fn deref(&self) -> &Self::Target { - //&self.rlp - //} -//} - -impl<'a> From> for Rlp<'a> { - fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { - Rlp { rlp: rlp } - } -} - -impl<'a> From> for UntrustedRlp<'a> { - fn from(unsafe_rlp: Rlp<'a>) -> UntrustedRlp<'a> { - unsafe_rlp.rlp - } -} - -#[derive(Debug)] -pub enum Prototype { - Null, - Data(usize), - List(usize), -} - -impl<'a, 'view> Rlp<'a> where 'a: 'view { - /// Create a new instance of `Rlp` - pub fn new(bytes: &'a [u8]) -> Rlp<'a> { - Rlp { - rlp: UntrustedRlp::new(bytes) - } - } - - /// Get the prototype of the RLP. - pub fn prototype(&self) -> Prototype { - if self.is_data() { - Prototype::Data(self.size()) - } - else if self.is_list() { - Prototype::List(self.item_count()) - } - else { - Prototype::Null - } - } - - /// The raw data of the RLP. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let dog = rlp.at(1).raw(); - /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn raw(&'view self) -> &'a [u8] { - self.rlp.raw() - } - - pub fn payload_info(&self) -> PayloadInfo { - self.rlp.payload_info().unwrap() - } - - pub fn data(&'view self) -> &'a [u8] { - self.rlp.data() - } - - /// Returns number of RLP items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.item_count(), 2); - /// let view = rlp.at(1); - /// assert_eq!(view.item_count(), 0); - /// } - /// ``` - pub fn item_count(&self) -> usize { - self.rlp.item_count() - } - - /// Returns the number of bytes in the data, or zero if it isn't data. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.size(), 0); - /// let view = rlp.at(1); - /// assert_eq!(view.size(), 3); - /// } - /// ``` - pub fn size(&self) -> usize { - self.rlp.size() - } - - /// Get view onto RLP-slice at index. - /// - /// Caches offset to given index, so access to successive - /// slices is faster. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let dog = String::decode(&rlp.at(1)); - /// assert_eq!(dog, "dog".to_string()); - /// } - /// ``` - pub fn at(&'view self, index: usize) -> Rlp<'a> { - From::from(self.rlp.at(index).unwrap()) - } - - /// No value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![]; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_null()); - /// } - /// ``` - pub fn is_null(&self) -> bool { - self.rlp.is_null() - } - - /// Contains a zero-length string or zero-length list. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc0]; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_empty()); - /// } - /// ``` - pub fn is_empty(&self) -> bool { - self.rlp.is_empty() - } - - /// List value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_list()); - /// } - /// ``` - pub fn is_list(&self) -> bool { - self.rlp.is_list() - } - - /// String value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.at(1).is_data()); - /// } - /// ``` - pub fn is_data(&self) -> bool { - self.rlp.is_data() - } - - /// Int value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc1, 0x10]; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.is_int(), false); - /// assert_eq!(rlp.at(0).is_int(), true); - /// } - /// ``` - pub fn is_int(&self) -> bool { - self.rlp.is_int() - } - - /// Get iterator over rlp-slices - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); - /// } - /// ``` - pub fn iter(&'a self) -> RlpIterator<'a> { - self.into_iter() - } -} - -impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { - /// returns new instance of `UntrustedRlp` - pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { - UntrustedRlp { - bytes: bytes, - cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), - } - } - - /// The bare data of the RLP. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let dog = rlp.at(1).unwrap().raw(); - /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn raw(&'view self) -> &'a [u8] { - self.bytes - } - - pub fn payload_info(&self) -> Result { - BasicDecoder::payload_info(self.bytes) - } - - pub fn data(&'view self) -> &'a [u8] { - let ii = BasicDecoder::payload_info(self.bytes).unwrap(); - &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] - } - - /// Returns number of rlp items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.item_count(), 2); - /// let view = rlp.at(1).unwrap(); - /// assert_eq!(view.item_count(), 0); - /// } - /// ``` - pub fn item_count(&self) -> usize { - match self.is_list() { - true => self.iter().count(), - false => 0 - } - } - - /// Returns the number of bytes in the data, or zero if it isn't data. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.size(), 0); - /// let view = rlp.at(1).unwrap(); - /// assert_eq!(view.size(), 3); - /// } - /// ``` - pub fn size(&self) -> usize { - match self.is_data() { - true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, - false => 0 - } - } - - /// Get view onto rlp-slice at index - /// - /// Caches offset to given index, so access to successive - /// slices is faster. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let dog = String::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); - /// assert_eq!(dog, "dog".to_string()); - /// } - /// ``` - pub fn at(&'view self, index: usize) -> Result, DecoderError> { - if !self.is_list() { - return Err(DecoderError::RlpExpectedToBeList); - } - - // move to cached position if it's index is less or equal to - // current search index, otherwise move to beginning of list - let c = self.cache.get(); - let (mut bytes, to_skip) = match c.index <= index { - true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), - false => (try!(self.consume_list_prefix()), index), - }; - - // skip up to x items - bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); - - // update the cache - self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); - - // construct new rlp - let found = try!(BasicDecoder::payload_info(bytes)); - Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) - } - - /// No value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![]; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_null()); - /// } - /// ``` - pub fn is_null(&self) -> bool { - self.bytes.len() == 0 - } - - /// Contains a zero-length string or zero-length list. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc0]; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_empty()); - /// } - /// ``` - pub fn is_empty(&self) -> bool { - !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) - } - - /// List value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_list()); - /// } - /// ``` - pub fn is_list(&self) -> bool { - !self.is_null() && self.bytes[0] >= 0xc0 - } - - /// String value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.at(1).unwrap().is_data()); - /// } - /// ``` - pub fn is_data(&self) -> bool { - !self.is_null() && self.bytes[0] < 0xc0 - } - - /// Int value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc1, 0x10]; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.is_int(), false); - /// assert_eq!(rlp.at(0).unwrap().is_int(), true); - /// } - /// ``` - pub fn is_int(&self) -> bool { - if self.is_null() { - return false; - } - - match self.bytes[0] { - 0...0x80 => true, - 0x81...0xb7 => self.bytes[1] != 0, - b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, - _ => false - } - } - - /// Get iterator over rlp-slices - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let strings: Vec = rlp.iter() - /// .map(| i | String::decode_untrusted(&i)) - /// .map(| s | s.unwrap()) - /// .collect(); - /// } - /// ``` - pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { - self.into_iter() - } - - /// consumes first found prefix - fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(BasicDecoder::payload_info(self.bytes)); - let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); - Ok(bytes) - } - - /// consumes fixed number of items - fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { - let mut result = bytes; - for _ in 0..items { - let i = try!(BasicDecoder::payload_info(result)); - result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); - } - Ok(result) - } - - - /// consumes slice prefix of length `len` - fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { - match bytes.len() >= len { - true => Ok(&bytes[len..]), - false => Err(DecoderError::RlpIsTooShort), - } - } -} - -/// Iterator over rlp-slice list elements. -pub struct UntrustedRlpIterator<'a> { - rlp: &'a UntrustedRlp<'a>, - index: usize, -} - -impl<'a> IntoIterator for &'a UntrustedRlp<'a> { - type Item = UntrustedRlp<'a>; - type IntoIter = UntrustedRlpIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - UntrustedRlpIterator { - rlp: self, - index: 0, - } - } -} - -impl<'a> Iterator for UntrustedRlpIterator<'a> { - type Item = UntrustedRlp<'a>; - - fn next(&mut self) -> Option> { - let index = self.index; - let result = self.rlp.at(index).ok(); - self.index += 1; - result - } -} - -/// Iterator over trusted rlp-slice list elements. -pub struct RlpIterator<'a> { - rlp: &'a Rlp<'a>, - index: usize -} - -impl<'a> IntoIterator for &'a Rlp<'a> { - type Item = Rlp<'a>; - type IntoIter = RlpIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - RlpIterator { - rlp: self, - index: 0, - } - } -} - -impl<'a> Iterator for RlpIterator<'a> { - type Item = Rlp<'a>; - - fn next(&mut self) -> Option> { - let index = self.index; - let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); - self.index += 1; - result - } -} - -/// Shortcut function to decode trusted rlp -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::rlp::*; -/// -/// fn main () { -/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -/// let animals: Vec = decode(&data); -/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); -/// } -/// ``` -pub fn decode(bytes: &[u8]) -> T where T: Decodable { - let rlp = Rlp::new(bytes); - T::decode(&rlp) -} - -pub trait Decodable: Sized { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result; - fn decode(rlp: &Rlp) -> Self { - Self::decode_untrusted(&rlp.rlp).unwrap() - } -} - -impl Decodable for T where T: FromBytes { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_data() { - true => BasicDecoder::read_value(rlp.bytes, | bytes | { - Ok(try!(T::from_bytes(bytes))) - }), - false => Err(DecoderError::RlpExpectedToBeData), - } - } -} - -impl Decodable for Vec where T: Decodable { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_list() { - true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), - false => Err(DecoderError::RlpExpectedToBeList), - } - } -} - -impl Decodable for Vec { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_data() { - true => BasicDecoder::read_value(rlp.bytes, | bytes | { - let mut res = vec![]; - res.extend(bytes); - Ok(res) - }), - false => Err(DecoderError::RlpExpectedToBeData), - } - } -} - -pub trait Decoder { - fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result; -} - -struct BasicDecoder; - -impl BasicDecoder { - /// Return first item info - fn payload_info(bytes: &[u8]) -> Result { - let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => PayloadInfo::new(0, 1), - Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - } - Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - }, - // we cant reach this place, but rust requires _ to be implemented - _ => { panic!(); } - }; - - match item.header_len + item.value_len <= bytes.len() { - true => Ok(item), - false => Err(DecoderError::RlpIsTooShort), - } - } -} - -impl Decoder for BasicDecoder { - fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { - match bytes.first().map(|&x| x) { - // rlp is too short - None => Err(DecoderError::RlpIsTooShort), - // single byt value - Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), - // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), - // longer than 55 bytes - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let begin_of_value = 1 as usize + len_of_len; - let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); - Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) - } - // we are reading value, not a list! - _ => { panic!(); } - } - } -} - -#[derive(Debug, Copy, Clone)] -struct ListInfo { - position: usize, - current: usize, - max: usize, -} - -impl ListInfo { - fn new(position: usize, max: usize) -> ListInfo { - ListInfo { - position: position, - current: 0, - max: max, - } - } -} - -/// Appendable rlp encoder. -pub struct RlpStream { - unfinished_lists: ElasticArray16, - encoder: BasicEncoder, -} - -impl RlpStream { - /// Initializes instance of empty `RlpStream`. - pub fn new() -> RlpStream { - RlpStream { - unfinished_lists: ElasticArray16::new(), - encoder: BasicEncoder::new(), - } - } - - /// Initializes the `RLPStream` as a list. - pub fn new_list(len: usize) -> RlpStream { - let mut stream = RlpStream::new(); - stream.append_list(len); - stream - } - - /// Apends value to the end of stream, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat").append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { - // encode given value and add it at the end of the stream - object.encode(&mut self.encoder); - - // if list is finished, prepend the length - self.note_appended(1); - - // return chainable self - self - } - - /// Declare appending the list of given size, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_list(2).append(&"cat").append(&"dog"); - /// stream.append(&""); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); - /// } - /// ``` - pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - match len { - 0 => { - // we may finish, if the appended list len is equal 0 - self.encoder.bytes.push(0xc0u8); - self.note_appended(1); - }, - _ => { - let position = self.encoder.bytes.len(); - self.unfinished_lists.push(ListInfo::new(position, len)); - }, - } - - // return chainable self - self - } - - /// Apends null to the end of stream, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_empty_data().append_empty_data(); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); - /// } - /// ``` - pub fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { - // self push raw item - self.encoder.bytes.push(0x80); - - // try to finish and prepend the length - self.note_appended(1); - - // return chainable self - self - } - - /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - // push raw items - self.encoder.bytes.append_slice(bytes); - - // try to finish and prepend the length - self.note_appended(item_count); - - // return chainable self - self - } - - /// Clear the output stream so far. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(3); - /// stream.append(&"cat"); - /// stream.clear(); - /// stream.append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); - /// } - pub fn clear(&mut self) { - // clear bytes - self.encoder.bytes.clear(); - - // clear lists - self.unfinished_lists.clear(); - } - - /// Returns true if stream doesnt expect any more items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat"); - /// assert_eq!(stream.is_finished(), false); - /// stream.append(&"dog"); - /// assert_eq!(stream.is_finished(), true); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - pub fn is_finished(&self) -> bool { - self.unfinished_lists.len() == 0 - } - - /// Streams out encoded bytes. - /// - /// panic! if stream is not finished. - pub fn out(self) -> Vec { - match self.is_finished() { - true => self.encoder.out().to_vec(), - false => panic!() - } - } - - /// Try to finish lists - fn note_appended(&mut self, inserted_items: usize) -> () { - if self.unfinished_lists.len() == 0 { - return; - } - - let back = self.unfinished_lists.len() - 1; - let should_finish = match self.unfinished_lists.get_mut(back) { - None => false, - Some(ref mut x) => { - x.current += inserted_items; - if x.current > x.max { - panic!("You cannot append more items then you expect!"); - } - x.current == x.max - } - }; - - if should_finish { - let x = self.unfinished_lists.pop().unwrap(); - let len = self.encoder.bytes.len() - x.position; - self.encoder.insert_list_len_at_pos(len, x.position); - self.note_appended(1); - } - } -} - -/// Shortcut function to encode structure into rlp. -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::rlp::*; -/// -/// fn main () { -/// let animals = vec!["cat", "dog"]; -/// let out = encode(&animals); -/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); -/// } -/// ``` -pub fn encode(object: &E) -> Vec where E: Encodable -{ - let mut encoder = BasicEncoder::new(); - object.encode(&mut encoder); - encoder.out().to_vec() -} - -pub trait Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder; -} - -pub trait Encoder { - fn emit_value(&mut self, bytes: &[u8]) -> (); - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); -} - -impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(&self.to_bytes()) - } -} - -impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_list(|e| { - // insert all list elements - for el in self.iter() { - el.encode(e); - } - }) - } -} - -impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - let r: &[T] = self.as_ref(); - r.encode(encoder) - } -} - -/// lets treat bytes differently than other lists -/// they are a single value -impl<'a> Encodable for &'a [u8] { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(self) - } -} - -/// lets treat bytes differently than other lists -/// they are a single value -impl Encodable for Vec { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(self) - } -} - -struct BasicEncoder { - bytes: ElasticArray1024, -} - -impl BasicEncoder { - fn new() -> BasicEncoder { - BasicEncoder { bytes: ElasticArray1024::new() } - } - - /// inserts list prefix at given position - /// TODO: optimise it further? - fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { - let mut res = vec![]; - match len { - 0...55 => res.push(0xc0u8 + len as u8), - _ => { - res.push(0xf7u8 + len.to_bytes_len() as u8); - res.extend(len.to_bytes()); - } - }; - - self.bytes.insert_slice(pos, &res); - } - - /// get encoded value - fn out(self) -> ElasticArray1024 { - self.bytes - } -} - -impl Encoder for BasicEncoder { - fn emit_value(&mut self, bytes: &[u8]) -> () { - match bytes.len() { - // just 0 - 0 => self.bytes.push(0x80u8), - // byte is its own encoding - 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), - // (prefix + length), followed by the string - len @ 1 ... 55 => { - self.bytes.push(0x80u8 + len as u8); - self.bytes.append_slice(bytes); - } - // (prefix + length of length), followed by the length, followd by the string - len => { - self.bytes.push(0xb7 + len.to_bytes_len() as u8); - self.bytes.append_slice(&len.to_bytes()); - self.bytes.append_slice(bytes); - } - } - } - - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () - { - // get len before inserting a list - let before_len = self.bytes.len(); - - // insert all list elements - f(self); - - // get len after inserting a list - let after_len = self.bytes.len(); - - // diff is list len - let list_len = after_len - before_len; - self.insert_list_len_at_pos(list_len, before_len); - } -} - -#[cfg(test)] -mod tests { - extern crate json_tests; - use self::json_tests::execute_tests_from_directory; - use self::json_tests::rlp as rlptest; - use std::{fmt, cmp}; - use std::str::FromStr; - use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; - use uint::U256; - - #[test] - fn rlp_at() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = UntrustedRlp::new(&data); - assert!(rlp.is_list()); - //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); - let animals: Vec = rlp.as_val().unwrap(); - assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); - - let cat = rlp.at(0).unwrap(); - assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); - //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); - assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); - - let dog = rlp.at(1).unwrap(); - assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); - //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); - assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); - - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); - //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); - assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); - } - } - - #[test] - fn rlp_at_err() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; - { - let rlp = UntrustedRlp::new(&data); - assert!(rlp.is_list()); - - let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); - - let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); - } - } - - #[test] - fn rlp_iter() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = UntrustedRlp::new(&data); - let mut iter = rlp.iter(); - - let cat = iter.next().unwrap(); - assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); - - let dog = iter.next().unwrap(); - assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); - - let none = iter.next(); - assert!(none.is_none()); - - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); - } - } - - struct ETestPair(T, Vec) where T: rlp::Encodable; - - fn run_encode_tests(tests: Vec>) - where T: rlp::Encodable - { - for t in &tests { - let res = rlp::encode(&t.0); - assert_eq!(res, &t.1[..]); - } - } - - #[test] - fn encode_u16() { - let tests = vec![ - ETestPair(0u16, vec![0x80u8]), - ETestPair(0x100, vec![0x82, 0x01, 0x00]), - ETestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u32() { - let tests = vec![ - ETestPair(0u32, vec![0x80u8]), - ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u64() { - let tests = vec![ - ETestPair(0u64, vec![0x80u8]), - ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u256() { - let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), - ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_encode_tests(tests); - } - - #[test] - fn encode_str() { - let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), - ETestPair("dog", vec![0x83, b'd', b'o', b'g']), - ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), - ETestPair("", vec![0x80]), - ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; - run_encode_tests(tests); - } - - #[test] - fn encode_address() { - use hash::*; - - let tests = vec![ - ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), - vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) - ]; - run_encode_tests(tests); - } - - /// Vec (Bytes) is treated as a single value - #[test] - fn encode_vector_u8() { - let tests = vec![ - ETestPair(vec![], vec![0x80]), - ETestPair(vec![0u8], vec![0]), - ETestPair(vec![0x15], vec![0x15]), - ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_u64() { - let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u64], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_str() { - let tests = vec![ETestPair(vec!["cat", "dog"], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_of_vectors_str() { - let tests = vec![ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_encode_tests(tests); - } - - struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - - fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { - for t in &tests { - let res: T = rlp::decode(&t.1); - assert_eq!(res, t.0); - } - } - - /// Vec (Bytes) is treated as a single value - #[test] - fn decode_vector_u8() { - let tests = vec![ - DTestPair(vec![], vec![0x80]), - DTestPair(vec![0u8], vec![0]), - DTestPair(vec![0x15], vec![0x15]), - DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u16() { - let tests = vec![ - DTestPair(0u16, vec![0u8]), - DTestPair(0x100, vec![0x82, 0x01, 0x00]), - DTestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u32() { - let tests = vec![ - DTestPair(0u32, vec![0u8]), - DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u64() { - let tests = vec![ - DTestPair(0u64, vec![0u8]), - DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u256() { - let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), - DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_str() { - let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), - DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), - DTestPair("Marek".to_string(), - vec![0x85, b'M', b'a', b'r', b'e', b'k']), - DTestPair("".to_string(), vec![0x80]), - DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" - .to_string(), - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_address() { - use hash::*; - - let tests = vec![ - DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), - vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_u64() { - let tests = vec![ - DTestPair(vec![], vec![0xc0]), - DTestPair(vec![15u64], vec![0xc1, 0x0f]), - DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_str() { - let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_of_vectors_str() { - let tests = vec![DTestPair(vec![vec!["cat".to_string()]], - vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_decode_tests(tests); - } - - #[test] - fn test_rlp_json() { - println!("Json rlp test: "); - execute_tests_from_directory::("json-tests/json/rlp/stream/*.json", &mut | file, input, output | { - println!("file: {}", file); - - let mut stream = RlpStream::new(); - for operation in input.into_iter() { - match operation { - rlptest::Operation::Append(ref v) => stream.append(v), - rlptest::Operation::AppendList(len) => stream.append_list(len), - rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len), - rlptest::Operation::AppendEmpty => stream.append_empty_data() - }; - } - - assert_eq!(stream.out(), output); - }); - } - -} diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 2effb7297..ce43c10a5 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,5 +1,4 @@ -use super::faces::{View, Decodable, DecoderError}; -use super::untrusted_rlp::*; +use rlp::{View, Decodable, DecoderError, UntrustedRlp, PayloadInfo, Prototype}; impl<'a> From> for Rlp<'a> { fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 379852083..557c4af25 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -1,6 +1,6 @@ use elastic_array::*; use bytes::ToBytes; -use super::faces::{Stream, Encoder, Encodable}; +use rlp::{Stream, Encoder, Encodable}; #[derive(Debug, Copy, Clone)] struct ListInfo { diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs new file mode 100644 index 000000000..e44953e07 --- /dev/null +++ b/src/rlp/tests.rs @@ -0,0 +1,345 @@ +extern crate json_tests; +use self::json_tests::execute_tests_from_directory; +use self::json_tests::rlp as rlptest; +use std::{fmt, cmp}; +use std::str::FromStr; +use rlp; +use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; +use uint::U256; + +#[test] +fn rlp_at() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = UntrustedRlp::new(&data); + assert!(rlp.is_list()); + //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + let animals: Vec = rlp.as_val().unwrap(); + assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); + + let cat = rlp.at(0).unwrap(); + assert!(cat.is_data()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); + + let dog = rlp.at(1).unwrap(); + assert!(dog.is_data()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_data()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); + } +} + +#[test] +fn rlp_at_err() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; + { + let rlp = UntrustedRlp::new(&data); + assert!(rlp.is_list()); + + let cat_err = rlp.at(0).unwrap_err(); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); + + let dog_err = rlp.at(1).unwrap_err(); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); + } +} + +#[test] +fn rlp_iter() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = UntrustedRlp::new(&data); + let mut iter = rlp.iter(); + + let cat = iter.next().unwrap(); + assert!(cat.is_data()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + + let dog = iter.next().unwrap(); + assert!(dog.is_data()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + + let none = iter.next(); + assert!(none.is_none()); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_data()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + } +} + +struct ETestPair(T, Vec) where T: rlp::Encodable; + +fn run_encode_tests(tests: Vec>) + where T: rlp::Encodable +{ + for t in &tests { + let res = rlp::encode(&t.0); + assert_eq!(res, &t.1[..]); + } +} + +#[test] +fn encode_u16() { + let tests = vec![ + ETestPair(0u16, vec![0x80u8]), + ETestPair(0x100, vec![0x82, 0x01, 0x00]), + ETestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u32() { + let tests = vec![ + ETestPair(0u32, vec![0x80u8]), + ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u64() { + let tests = vec![ + ETestPair(0u64, vec![0x80u8]), + ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u256() { + let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_encode_tests(tests); +} + +#[test] +fn encode_str() { + let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_encode_tests(tests); +} + +#[test] +fn encode_address() { + use hash::*; + + let tests = vec![ + ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_encode_tests(tests); +} + +/// Vec (Bytes) is treated as a single value +#[test] +fn encode_vector_u8() { + let tests = vec![ + ETestPair(vec![], vec![0x80]), + ETestPair(vec![0u8], vec![0]), + ETestPair(vec![0x15], vec![0x15]), + ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_u64() { + let tests = vec![ + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u64], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_str() { + let tests = vec![ETestPair(vec!["cat", "dog"], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_of_vectors_str() { + let tests = vec![ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_encode_tests(tests); +} + +struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; + +fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { + for t in &tests { + let res: T = rlp::decode(&t.1); + assert_eq!(res, t.0); + } +} + +/// Vec (Bytes) is treated as a single value +#[test] +fn decode_vector_u8() { + let tests = vec![ + DTestPair(vec![], vec![0x80]), + DTestPair(vec![0u8], vec![0]), + DTestPair(vec![0x15], vec![0x15]), + DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u16() { + let tests = vec![ + DTestPair(0u16, vec![0u8]), + DTestPair(0x100, vec![0x82, 0x01, 0x00]), + DTestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u32() { + let tests = vec![ + DTestPair(0u32, vec![0u8]), + DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u64() { + let tests = vec![ + DTestPair(0u64, vec![0u8]), + DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u256() { + let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_str() { + let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), + vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" + .to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_address() { + use hash::*; + + let tests = vec![ + DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_u64() { + let tests = vec![ + DTestPair(vec![], vec![0xc0]), + DTestPair(vec![15u64], vec![0xc1, 0x0f]), + DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_str() { + let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_of_vectors_str() { + let tests = vec![DTestPair(vec![vec!["cat".to_string()]], + vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_decode_tests(tests); +} + +#[test] +fn test_rlp_json() { + println!("Json rlp test: "); + execute_tests_from_directory::("json-tests/json/rlp/stream/*.json", &mut | file, input, output | { + println!("file: {}", file); + + let mut stream = RlpStream::new(); + for operation in input.into_iter() { + match operation { + rlptest::Operation::Append(ref v) => stream.append(v), + rlptest::Operation::AppendList(len) => stream.append_list(len), + rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len), + rlptest::Operation::AppendEmpty => stream.append_empty_data() + }; + } + + assert_eq!(stream.out(), output); + }); +} + diff --git a/src/rlp/faces.rs b/src/rlp/traits.rs similarity index 92% rename from src/rlp/faces.rs rename to src/rlp/traits.rs index 078620de2..966078b08 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/traits.rs @@ -1,32 +1,4 @@ -use std::fmt; -use std::error::Error as StdError; -use bytes::FromBytesError; - -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} - -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} +use rlp::DecoderError; pub trait Decoder { fn read_value(&self, f: F) -> Result diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 9e2e2f522..19dd5e99c 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,6 +1,6 @@ use std::cell::Cell; use bytes::{FromBytes}; -use super::faces::{View, Decoder, Decodable, DecoderError}; +use rlp::{View, Decoder, Decodable, DecoderError}; /// rlp offset #[derive(Copy, Clone, Debug)] From d5ab429e777b86d141daf61ec280d36cd39973f5 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 13:02:32 +0100 Subject: [PATCH 23/47] changed order of reexports --- src/rlp/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 49043d65f..cb36426c9 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -41,8 +41,8 @@ mod tests; pub use self::errors::DecoderError; pub use self::traits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; +pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; pub use self::rlp::{Rlp, RlpIterator}; -pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::rlpstream::{RlpStream}; /// Shortcut function to decode trusted rlp From 1c58ba47642c0cf0397380e83275625e34000213 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 13:22:24 +0100 Subject: [PATCH 24/47] . --- benches/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 62f6c6112..1160e311f 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -91,6 +91,6 @@ fn bench_stream_1000_empty_lists(b: &mut Bencher) { for _ in 0..1000 { stream.append_list(0); } - //let _ = stream.out(); + let _ = stream.out(); }); } From b657a15100ef2bc0f93b6e15cc6fda55e1338e28 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 14:44:22 +0100 Subject: [PATCH 25/47] val_at rlp method --- src/rlp/rlp.rs | 12 ++++++++++-- src/rlp/traits.rs | 2 ++ src/rlp/untrusted_rlp.rs | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index ce43c10a5..ac830cc9c 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -84,16 +84,24 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { fn as_val(&self) -> Result where T: Decodable { self.rlp.as_val() } + + fn val_at(&self, index: usize) -> Result where T: Decodable { + self.at(index).rlp.as_val() + } } impl <'a, 'view> Rlp<'a> where 'a: 'view { - fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { + fn view_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { let res: Result = r.as_val(); res.unwrap_or_else(|_| panic!()) } pub fn as_val(&self) -> T where T: Decodable { - Self::reader_as_val(self) + Self::view_as_val(self) + } + + pub fn val_at(&self, index: usize) -> T where T: Decodable { + Self::view_as_val(&self.at(index)) } } diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 966078b08..1127ca3db 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -179,6 +179,8 @@ pub trait View<'a, 'view>: Sized { fn iter(&'view self) -> Self::Iter; fn as_val(&self) -> Result where T: Decodable; + + fn val_at(&self, index: usize) -> Result where T: Decodable; } pub trait Encoder { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 19dd5e99c..eae253eca 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -176,6 +176,10 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { // optimize, so it doesn't use clone (although This clone is cheap) T::decode(&BasicDecoder::new(self.clone())) } + + fn val_at(&self, index: usize) -> Result where T: Decodable { + self.at(index).unwrap().as_val() + } } impl<'a> UntrustedRlp<'a> { From 0e343d078cc9a64afc8b6ea65fe2f0f9dbd0c6f1 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 19:01:37 +0100 Subject: [PATCH 26/47] rlp encodable and decodable for options --- src/rlp/mod.rs | 3 +-- src/rlp/rlpstream.rs | 19 ++++++++++++++----- src/rlp/untrusted_rlp.rs | 12 ++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index cb36426c9..981e5f973 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -74,8 +74,7 @@ pub fn decode(bytes: &[u8]) -> T where T: Decodable { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` -pub fn encode(object: &E) -> Vec where E: Encodable -{ +pub fn encode(object: &E) -> Vec where E: Encodable { let mut stream = RlpStream::new(); stream.append(object); stream.out() diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 557c4af25..301a39053 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -207,13 +207,13 @@ impl Encoder for BasicEncoder { } impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(&self.to_bytes()) } } impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_list(|e| { // insert all list elements for el in self.iter() { @@ -224,7 +224,7 @@ impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { } impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { let r: &[T] = self.as_ref(); r.encode(encoder) } @@ -233,7 +233,7 @@ impl Encodable for Vec where T: Encodable { /// lets treat bytes differently than other lists /// they are a single value impl<'a> Encodable for &'a [u8] { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(self) } } @@ -241,7 +241,16 @@ impl<'a> Encodable for &'a [u8] { /// lets treat bytes differently than other lists /// they are a single value impl Encodable for Vec { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(self) } } + +impl Encodable for Option where T: Encodable { + fn encode(&self, encoder: &mut E) where E: Encoder { + match *self { + Some(ref x) => x.encode(encoder), + None => encoder.emit_value(&[]) + } + } +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index eae253eca..ec68fce99 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -340,3 +340,15 @@ impl Decodable for Vec { }) } } + +impl Decodable for Option where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + let res = match bytes.len() { + 0 => None, + _ => Some(try!(T::decode(decoder))) + }; + Ok(res) + }) + } +} From f10e72caeb3f8fac76610f1e697184a06a891341 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 02:29:34 +0100 Subject: [PATCH 27/47] fixed rlp benchmarks, added triehash benchmarks --- benches/rlp.rs | 14 +++++----- benches/trie.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 1160e311f..8bc7599e4 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -11,7 +11,7 @@ extern crate ethcore_util; use test::Bencher; use std::str::FromStr; -use ethcore_util::rlp::{RlpStream, Rlp, Decodable}; +use ethcore_util::rlp::*; use ethcore_util::uint::U256; #[bench] @@ -30,7 +30,7 @@ fn bench_decode_u64_value(b: &mut Bencher) { // u64 let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; let rlp = Rlp::new(&data); - let _ = u64::decode(&rlp); + let _: u64 = rlp.as_val(); }); } @@ -54,7 +54,7 @@ fn bench_decode_u256_value(b: &mut Bencher) { 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; let rlp = Rlp::new(&data); - let _ = U256::decode(&rlp); + let _ : U256 = rlp.as_val(); }); } @@ -76,11 +76,11 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let _v0: Vec = Decodable::decode(&rlp.at(0)); - let _v1: Vec> = Decodable::decode(&rlp.at(1)); + let _v0: Vec = rlp.val_at(0); + let _v1: Vec> = rlp.val_at(1); let nested_rlp = rlp.at(2); - let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); - let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); + let _v2a: Vec = nested_rlp.val_at(0); + let _v2b: Vec> = nested_rlp.val_at(1); }); } diff --git a/benches/trie.rs b/benches/trie.rs index 2865c5778..a8cd407cf 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -10,6 +10,7 @@ use test::Bencher; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; +use ethcore_util::triehash::*; use ethcore_util::sha3::*; @@ -40,7 +41,7 @@ fn random_value(seed: &mut H256) -> Bytes { } #[bench] -fn insertions_six_high(b: &mut Bencher) { +fn trie_insertions_six_high(b: &mut Bencher) { let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); for _ in 0..1000 { @@ -58,7 +59,22 @@ fn insertions_six_high(b: &mut Bencher) { } #[bench] -fn insertions_six_mid(b: &mut Bencher) { +fn triehash_insertions_six_high(b: &mut Bencher) { + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_bytes(6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(&||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_six_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -77,7 +93,22 @@ fn insertions_six_mid(b: &mut Bencher) { } #[bench] -fn insertions_random_mid(b: &mut Bencher) { +fn triehash_insertions_six_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_random_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -96,7 +127,23 @@ fn insertions_random_mid(b: &mut Bencher) { } #[bench] -fn insertions_six_low(b: &mut Bencher) { +fn triehash_insertions_random_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 1, 5, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_six_low(b: &mut Bencher) { let alphabet = b"abcdef"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -114,6 +161,22 @@ fn insertions_six_low(b: &mut Bencher) { }) } +#[bench] +fn triehash_insertions_six_low(b: &mut Bencher) { + let alphabet = b"abcdef"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} + #[bench] fn sha3x1000(b: &mut Bencher) { b.iter(||{ From d2b1b92f9eca2a5721e43e2abfa2d3bc931937b8 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 11:05:27 +0100 Subject: [PATCH 28/47] removed unused macros.rs --- src/lib.rs | 1 - src/macros.rs | 11 ----------- 2 files changed, 12 deletions(-) delete mode 100644 src/macros.rs diff --git a/src/lib.rs b/src/lib.rs index 883cac2f2..12998529a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,7 +44,6 @@ extern crate secp256k1; extern crate arrayvec; extern crate elastic_array; -pub mod macros; pub mod error; pub mod hash; pub mod uint; diff --git a/src/macros.rs b/src/macros.rs deleted file mode 100644 index 69286a340..000000000 --- a/src/macros.rs +++ /dev/null @@ -1,11 +0,0 @@ -macro_rules! map( - { $($key:expr => $value:expr),+ } => { - { - let mut m = ::std::collections::HashMap::new(); - $( - m.insert($key, $value); - )+ - m - } - }; -); From 6db12408dcf9ac01c9d91dea9b02136f97a87a50 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 14:50:01 +0100 Subject: [PATCH 29/47] fixed uint multiplication --- src/uint.rs | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index e017de385..234867ea9 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -82,6 +82,15 @@ macro_rules! construct_uint { bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; } } + + #[inline] + pub fn exp10(n: usize) -> $name { + match n { + 0 => $name::from(1u64), + _ => $name::exp10(n - 1) * $name::from(10u64) + } + } + /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -167,12 +176,12 @@ macro_rules! construct_uint { type Output = $name; fn mul(self, other: $name) -> $name { - let mut me = self; + let mut res = $name::from(0u64); // TODO: be more efficient about this for i in 0..(2 * $n_words) { - me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); + res = res + (self.mul_u32((other >> (32 * i)).low_u32()) << (32 * i)); } - me + res } } @@ -516,5 +525,37 @@ mod tests { assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); } + + #[test] + pub fn uint256_exp10() { + assert_eq!(U256::exp10(0), U256::from(1u64)); + println!("\none: {:?}", U256::from(1u64)); + println!("ten: {:?}", U256::from(10u64)); + assert_eq!(U256::from(2u64) * U256::from(10u64), U256::from(20u64)); + assert_eq!(U256::exp10(1), U256::from(10u64)); + assert_eq!(U256::exp10(2), U256::from(100u64)); + assert_eq!(U256::exp10(5), U256::from(100000u64)); + } + + #[test] + pub fn uint256_mul32() { + assert_eq!(U256::from(0u64).mul_u32(2), U256::from(0u64)); + assert_eq!(U256::from(1u64).mul_u32(2), U256::from(2u64)); + assert_eq!(U256::from(10u64).mul_u32(2), U256::from(20u64)); + assert_eq!(U256::from(10u64).mul_u32(5), U256::from(50u64)); + assert_eq!(U256::from(1000u64).mul_u32(50), U256::from(50000u64)); + } + + #[test] + pub fn uint256_mul() { + assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); + } + + #[test] + fn uint256_div() { + assert_eq!(U256::from(10u64) / U256::from(1u64), U256::from(10u64)); + assert_eq!(U256::from(10u64) / U256::from(2u64), U256::from(5u64)); + assert_eq!(U256::from(10u64) / U256::from(3u64), U256::from(3u64)); + } } From 1a404167249bcc7e1db801f40d0db4843e3341ff Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 01:41:44 +0100 Subject: [PATCH 30/47] Makefile for cross-compilation --- Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ad3c5230f --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +# Makefile for cross-compilation +IOS_ARCHS = i386-apple-ios x86_64-apple-ios armv7-apple-ios armv7s-apple-ios aarch64-apple-ios +IOS_LIB = libethcore_util.a + +ios: $(IOS_LIB) + +.PHONY: $(IOS_ARCHS) +$(IOS_ARCHS): %: + multirust run ios cargo build --target $@ + +$(IOS_LIB): $(IOS_ARCHS) + lipo -create -output $@ $(foreach arch,$(IOS_ARCHS),$(wildcard target/$(arch)/debug/$(IOS_LIB))) From 4210b957ac2ab0558271651dd83dcb3d2db14fff Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 15:48:25 +0100 Subject: [PATCH 31/47] triedb db is borrowed instead of being owned --- src/trie.rs | 110 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 3c7e8bfb2..745dc9c48 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -2,7 +2,6 @@ extern crate rand; use std::fmt; -use memorydb::*; use sha3::*; use hashdb::*; use hash::*; @@ -279,10 +278,14 @@ impl <'a>Node<'a> { /// /// # Example /// ``` -/// extern crate ethcore_util; -/// use ethcore_util::trie::*; +/// extern crate ethcore_util as util; +/// use util::trie::*; +/// use util::hashdb::*; +/// use util::memorydb::*; +/// /// fn main() { -/// let mut t = TrieDB::new_memory(); +/// let mut memdb = MemoryDB::new(); +/// let mut t = TrieDB::new(&mut memdb); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), SHA3_NULL_RLP); /// t.insert(b"foo", b"bar"); @@ -294,8 +297,8 @@ impl <'a>Node<'a> { /// assert!(t.db_items_remaining().is_empty()); /// } /// ``` -pub struct TrieDB { - db: Box, +pub struct TrieDB<'db, T> where T: 'db + HashDB { + db: &'db mut T, root: H256, pub hash_count: usize, } @@ -306,18 +309,23 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl TrieDB { - /// Create a new trie with the boxed backing database `box_db`. - pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } +impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { + /// Create a new trie with the backing database `db`. + pub fn new(db: &'db mut T) -> Self { + let mut r = TrieDB{ + db: db, + root: H256::new(), + hash_count: 0 + }; - /// Convenience function to create a new trie with the backing database `db`. - pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } - - /// Convenience function to create a new trie with a new `MemoryDB` based backing database. - pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } + r.root = r.db.insert(&NULL_RLP); + r + } /// Get the backing database. - pub fn db(&self) -> &HashDB { self.db.as_ref() } + pub fn db(&'db self) -> &'db T { + self.db + } /// Determine all the keys in the backing database that belong to the trie. pub fn keys(&self) -> Vec { @@ -868,7 +876,7 @@ impl TrieDB { } } -impl Trie for TrieDB { +impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { @@ -891,7 +899,7 @@ impl Trie for TrieDB { } } -impl fmt::Debug for TrieDB { +impl<'db, T> fmt::Debug for TrieDB<'db, T> where T: 'db + HashDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); @@ -906,6 +914,8 @@ mod tests { use self::json_tests::{trie, execute_tests_from_directory}; use triehash::*; use hash::*; + use hashdb::*; + use memorydb::*; use super::*; use nibbleslice::*; use rlp; @@ -934,8 +944,8 @@ mod tests { } } - fn populate_trie(v: &Vec<(Vec, Vec)>) -> TrieDB { - let mut t = TrieDB::new_memory(); + fn populate_trie<'db, T>(db: &'db mut T, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + let mut t = TrieDB::new(db); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -944,7 +954,7 @@ mod tests { t } - fn unpopulate_trie(t: &mut TrieDB, v: &Vec<(Vec, Vec)>) { + fn unpopulate_trie<'a, 'db, T>(t: &mut TrieDB<'db, T>, v: &Vec<(Vec, Vec)>) where T: 'db + HashDB { for i in v.iter() { let key: &[u8]= &i.0; t.remove(&key); @@ -994,7 +1004,8 @@ mod tests { } let real = trie_root(x.clone()); - let mut memtrie = populate_trie(&x); + let mut memdb = MemoryDB::new(); + let mut memtrie = populate_trie(&mut memdb, &x); if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { println!("TRIE MISMATCH"); println!(""); @@ -1023,14 +1034,16 @@ mod tests { #[test] fn init() { - let t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let t = TrieDB::new(&mut memdb); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } #[test] fn insert_on_empty() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1039,12 +1052,14 @@ mod tests { fn remove_to_empty() { let big_value = b"00000000000000000000000000000000"; - let mut t1 = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t1 = TrieDB::new(&mut memdb); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); trace!("keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); - let mut t2 = TrieDB::new_memory(); + let mut memdb2 = MemoryDB::new(); + let mut t2 = TrieDB::new(&mut memdb2); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); @@ -1058,7 +1073,8 @@ mod tests { #[test] fn insert_replace_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1066,7 +1082,8 @@ mod tests { #[test] fn insert_make_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ @@ -1077,7 +1094,8 @@ mod tests { #[test] fn insert_into_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1090,7 +1108,8 @@ mod tests { #[test] fn insert_value_into_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ @@ -1101,7 +1120,8 @@ mod tests { #[test] fn insert_split_leaf() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); assert_eq!(*t.root(), trie_root(vec![ @@ -1112,7 +1132,8 @@ mod tests { #[test] fn insert_split_extenstion() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01, 0x23, 0x45], &[0x01]); t.insert(&[0x01, 0xf3, 0x45], &[0x02]); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); @@ -1128,7 +1149,8 @@ mod tests { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], big_value0); t.insert(&[0x11u8, 0x23], big_value1); assert_eq!(*t.root(), trie_root(vec![ @@ -1141,7 +1163,8 @@ mod tests { fn insert_duplicate_value() { let big_value = b"00000000000000000000000000000000"; - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], big_value); t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ @@ -1199,20 +1222,23 @@ mod tests { #[test] fn test_at_empty() { - let t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let t = TrieDB::new(&mut memdb); assert_eq!(t.at(&[0x5]), None); } #[test] fn test_at_one() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } #[test] fn test_at_three() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1224,7 +1250,8 @@ mod tests { #[test] fn test_print_trie() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); @@ -1244,10 +1271,12 @@ mod tests { x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); - let memtrie = populate_trie(&x); + let mut memdb = MemoryDB::new(); + let memtrie = populate_trie(&mut memdb, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let memtrie_sorted = populate_trie(&y); + let mut memdb2 = MemoryDB::new(); + let memtrie_sorted = populate_trie(&mut memdb2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1273,7 +1302,8 @@ mod tests { execute_tests_from_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { println!("file: {}", file); - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for operation in input.into_iter() { match operation { trie::Operation::Insert(key, value) => t.insert(&key, &value), From 8c9fe2fbe5725275953195a51c2242309906c11c Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 15:57:47 +0100 Subject: [PATCH 32/47] fixed benches --- benches/trie.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index a8cd407cf..c12a8f752 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -10,6 +10,8 @@ use test::Bencher; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; +use ethcore_util::hashdb::*; +use ethcore_util::memorydb::*; use ethcore_util::triehash::*; use ethcore_util::sha3::*; @@ -51,7 +53,8 @@ fn trie_insertions_six_high(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -84,7 +87,8 @@ fn trie_insertions_six_mid(b: &mut Bencher) { d.push((k, v)) } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -119,7 +123,8 @@ fn trie_insertions_random_mid(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -154,7 +159,8 @@ fn trie_insertions_six_low(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } From b487f8f20e6c8a5a4ef6b125576ca618edd076f6 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 03:00:39 +0100 Subject: [PATCH 33/47] TrieDB root is borrowed --- src/trie.rs | 108 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 745dc9c48..2909e1314 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -282,10 +282,12 @@ impl <'a>Node<'a> { /// use util::trie::*; /// use util::hashdb::*; /// use util::memorydb::*; +/// use util::hash::*; /// /// fn main() { /// let mut memdb = MemoryDB::new(); -/// let mut t = TrieDB::new(&mut memdb); +/// let mut root = H256::new(); +/// let mut t = TrieDB::new(&mut memdb, &mut root); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), SHA3_NULL_RLP); /// t.insert(b"foo", b"bar"); @@ -299,7 +301,7 @@ impl <'a>Node<'a> { /// ``` pub struct TrieDB<'db, T> where T: 'db + HashDB { db: &'db mut T, - root: H256, + root: &'db mut H256, pub hash_count: usize, } @@ -310,18 +312,32 @@ enum MaybeChanged<'a> { } impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { - /// Create a new trie with the backing database `db`. - pub fn new(db: &'db mut T) -> Self { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db mut T, root: &'db mut H256) -> Self { let mut r = TrieDB{ db: db, - root: H256::new(), + root: root, hash_count: 0 }; - r.root = r.db.insert(&NULL_RLP); + // set root rlp + *r.root = r.db.insert(&NULL_RLP); r } + /// Create a new trie with the backing database `db` and `root` + /// Panics, if `root` does not exist + pub fn new_pre_existing(db: &'db mut T, root: &'db mut H256) -> Self { + assert!(db.exists(root)); + TrieDB { + db: db, + root: root, + hash_count: 0 + } + } + /// Get the backing database. pub fn db(&'db self) -> &'db T { self.db @@ -363,7 +379,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// and removing the old. fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); - self.root = self.db.insert(root_data); + *self.root = self.db.insert(root_data); self.hash_count += 1; trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } @@ -944,8 +960,8 @@ mod tests { } } - fn populate_trie<'db, T>(db: &'db mut T, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { - let mut t = TrieDB::new(db); + fn populate_trie<'db, T>(db: &'db mut T, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + let mut t = TrieDB::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -1005,7 +1021,8 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); - let mut memtrie = populate_trie(&mut memdb, &x); + let mut root = H256::new(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { println!("TRIE MISMATCH"); println!(""); @@ -1035,7 +1052,8 @@ mod tests { #[test] fn init() { let mut memdb = MemoryDB::new(); - let t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let t = TrieDB::new(&mut memdb, &mut root); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } @@ -1043,7 +1061,8 @@ mod tests { #[test] fn insert_on_empty() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1053,13 +1072,15 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::new(); - let mut t1 = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t1 = TrieDB::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); trace!("keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); let mut memdb2 = MemoryDB::new(); - let mut t2 = TrieDB::new(&mut memdb2); + let mut root2 = H256::new(); + let mut t2 = TrieDB::new(&mut memdb2, &mut root2); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); @@ -1074,7 +1095,8 @@ mod tests { #[test] fn insert_replace_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1083,7 +1105,8 @@ mod tests { #[test] fn insert_make_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ @@ -1095,7 +1118,8 @@ mod tests { #[test] fn insert_into_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1109,7 +1133,8 @@ mod tests { #[test] fn insert_value_into_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ @@ -1121,7 +1146,8 @@ mod tests { #[test] fn insert_split_leaf() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); assert_eq!(*t.root(), trie_root(vec![ @@ -1133,7 +1159,8 @@ mod tests { #[test] fn insert_split_extenstion() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]); t.insert(&[0x01, 0xf3, 0x45], &[0x02]); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); @@ -1150,7 +1177,8 @@ mod tests { let big_value1 = b"11111111111111111111111111111111"; let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0); t.insert(&[0x11u8, 0x23], big_value1); assert_eq!(*t.root(), trie_root(vec![ @@ -1164,7 +1192,8 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value); t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ @@ -1223,14 +1252,16 @@ mod tests { #[test] fn test_at_empty() { let mut memdb = MemoryDB::new(); - let t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let t = TrieDB::new(&mut memdb, &mut root); assert_eq!(t.at(&[0x5]), None); } #[test] fn test_at_one() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } @@ -1238,7 +1269,8 @@ mod tests { #[test] fn test_at_three() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1251,7 +1283,8 @@ mod tests { #[test] fn test_print_trie() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); @@ -1272,11 +1305,13 @@ mod tests { } let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); - let memtrie = populate_trie(&mut memdb, &x); + let mut root = H256::new(); + let memtrie = populate_trie(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); let mut memdb2 = MemoryDB::new(); - let memtrie_sorted = populate_trie(&mut memdb2, &y); + let mut root2 = H256::new(); + let memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1303,7 +1338,8 @@ mod tests { println!("file: {}", file); let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for operation in input.into_iter() { match operation { trie::Operation::Insert(key, value) => t.insert(&key, &value), @@ -1314,4 +1350,18 @@ mod tests { assert_eq!(*t.root(), H256::from_slice(&output)); }); } + + #[test] + fn test_trie_existing() { + let mut root = H256::new(); + let mut db = MemoryDB::new(); + { + let mut t = TrieDB::new(&mut db, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } + + { + let _ = TrieDB::new_pre_existing(&mut db, &mut root); + } + } } From f19d044e7ce5ef286f9d722004132067bd038103 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 03:49:15 +0100 Subject: [PATCH 34/47] renamed new_pre_existing to new_existing --- src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 2909e1314..94ca8c7d7 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -329,7 +329,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_pre_existing(db: &'db mut T, root: &'db mut H256) -> Self { + pub fn new_existing(db: &'db mut T, root: &'db mut H256) -> Self { assert!(db.exists(root)); TrieDB { db: db, @@ -1361,7 +1361,7 @@ mod tests { } { - let _ = TrieDB::new_pre_existing(&mut db, &mut root); + let _ = TrieDB::new_existing(&mut db, &mut root); } } } From 45ee74868a10840513ac78dcd01c843324b35e06 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 11:53:44 +0100 Subject: [PATCH 35/47] fixed trie benches --- benches/trie.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index c12a8f752..65df58309 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -54,7 +54,8 @@ fn trie_insertions_six_high(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -88,7 +89,8 @@ fn trie_insertions_six_mid(b: &mut Bencher) { } b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -124,7 +126,8 @@ fn trie_insertions_random_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -160,7 +163,8 @@ fn trie_insertions_six_low(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } From 16cadf1a64abf36d43aefa0ce5ee6ad35e15dbab Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 14:40:28 +0100 Subject: [PATCH 36/47] hash for uint --- src/uint.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/uint.rs b/src/uint.rs index 234867ea9..389894c68 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -25,6 +25,7 @@ use std::fmt; use std::cmp::*; use std::ops::*; use std::str::FromStr; +use std::hash::{Hash, Hasher}; use rustc_serialize::hex::{FromHex, FromHexError}; macro_rules! impl_map_from { @@ -350,6 +351,13 @@ macro_rules! construct_uint { Ok(()) } } + + impl Hash for $name { + fn hash(&self, state: &mut H) where H: Hasher { + unsafe { state.write(::std::slice::from_raw_parts(self.0.as_ptr() as *mut u8, self.0.len() * 8)); } + state.finish(); + } + } ); } From 862702b9c731df55b1dc0dd334830936badc30f7 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 06:04:24 +0100 Subject: [PATCH 37/47] implemented rem, lenient from_str, from_dec_str for uint --- src/uint.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 389894c68..d0bb55608 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -28,6 +28,11 @@ use std::str::FromStr; use std::hash::{Hash, Hasher}; use rustc_serialize::hex::{FromHex, FromHexError}; +pub trait FromDecStr: Sized { + type Err; + fn from_dec_str(value: &str) -> Result; +} + macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { impl From<$from> for $thing { @@ -139,8 +144,13 @@ macro_rules! construct_uint { type Err = FromHexError; fn from_str(value: &str) -> Result<$name, Self::Err> { - let bytes: &[u8] = &try!(value.from_hex()); - Ok(From::from(bytes)) + let bytes: Vec = match value.len() % 2 == 0 { + true => try!(value.from_hex()), + false => try!(("0".to_string() + value).from_hex()) + }; + + let bytes_ref: &[u8] = &bytes; + Ok(From::from(bytes_ref)) } } @@ -222,6 +232,15 @@ macro_rules! construct_uint { } } + impl Rem<$name> for $name { + type Output = $name; + + fn rem(self, other: $name) -> $name { + let times = self / other; + self - (times * other) + } + } + impl BitAnd<$name> for $name { type Output = $name; @@ -358,9 +377,21 @@ macro_rules! construct_uint { state.finish(); } } + + impl FromDecStr for $name { + type Err = FromHexError; + + /// TODO: optimize, throw appropriate err + fn from_dec_str(value: &str) -> Result { + let ten = $name::from(10u64); + Ok(value.bytes().map(|b| b - 48).fold($name::from(0u64), | acc, c | acc * ten + $name::from(c) )) + } + } + ); } +construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); @@ -374,6 +405,10 @@ impl From for U256 { } } + + + + pub const ZERO_U256: U256 = U256([0x00u64; 4]); pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); @@ -381,6 +416,7 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { use uint::U256; + use uint::FromDecStr; use std::str::FromStr; #[test] @@ -565,5 +601,17 @@ mod tests { assert_eq!(U256::from(10u64) / U256::from(2u64), U256::from(5u64)); assert_eq!(U256::from(10u64) / U256::from(3u64), U256::from(3u64)); } + + #[test] + fn uint256_rem() { + assert_eq!(U256::from(10u64) % U256::from(1u64), U256::from(0u64)); + assert_eq!(U256::from(10u64) % U256::from(3u64), U256::from(1u64)); + } + + #[test] + fn uint256_from_dec_str() { + assert_eq!(U256::from_dec_str("10").unwrap(), U256::from(10u64)); + assert_eq!(U256::from_dec_str("1024").unwrap(), U256::from(1024u64)); + } } From 38f813a5e641c0183de1e0d420794bacf956473c Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 13:32:42 +0100 Subject: [PATCH 38/47] fast mul by 10 --- src/uint.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index d0bb55608..09b654270 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -383,8 +383,13 @@ macro_rules! construct_uint { /// TODO: optimize, throw appropriate err fn from_dec_str(value: &str) -> Result { - let ten = $name::from(10u64); - Ok(value.bytes().map(|b| b - 48).fold($name::from(0u64), | acc, c | acc * ten + $name::from(c) )) + Ok(value.bytes() + .map(|b| b - 48) + .fold($name::from(0u64), | acc, c | + // fast multiplication by 10 + // (acc << 3) + (acc << 1) => acc * 10 + (acc << 3) + (acc << 1) + $name::from(c) + )) } } From 6bc56ad004d4a982f27cf8a74c38c84726482dd2 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 15:57:08 +0100 Subject: [PATCH 39/47] added raw method to RlpStream. usefull for quicklookup --- src/rlp/rlpstream.rs | 4 ++++ src/rlp/traits.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 301a39053..43ae83a9a 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -101,6 +101,10 @@ impl Stream for RlpStream { self.unfinished_lists.len() == 0 } + fn raw(&self) -> &[u8] { + &self.encoder.bytes + } + fn out(self) -> Vec { match self.is_finished() { true => self.encoder.out().to_vec(), diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 1127ca3db..4f9ad09e5 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -282,6 +282,8 @@ pub trait Stream: Sized { /// } fn is_finished(&self) -> bool; + fn raw(&self) -> &[u8]; + /// Streams out encoded bytes. /// /// panic! if stream is not finished. From d94c55133bfd2fae0c78e64c396833baeb2816b7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 13 Dec 2015 19:36:49 +0100 Subject: [PATCH 40/47] Trie takes a reference to HashDB. --- src/trie.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 94ca8c7d7..ad70a30a1 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -299,8 +299,8 @@ impl <'a>Node<'a> { /// assert!(t.db_items_remaining().is_empty()); /// } /// ``` -pub struct TrieDB<'db, T> where T: 'db + HashDB { - db: &'db mut T, +pub struct TrieDB<'db> { + db: &'db mut HashDB, root: &'db mut H256, pub hash_count: usize, } @@ -311,12 +311,12 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> TrieDB<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut T, root: &'db mut H256) -> Self { - let mut r = TrieDB{ + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + let mut r = TrieDB{ db: db, root: root, hash_count: 0 @@ -329,7 +329,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_existing(db: &'db mut T, root: &'db mut H256) -> Self { + pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { assert!(db.exists(root)); TrieDB { db: db, @@ -339,7 +339,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { } /// Get the backing database. - pub fn db(&'db self) -> &'db T { + pub fn db(&'db self) -> &'db HashDB { self.db } @@ -364,7 +364,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Determine occurances of items in the backing database which are not related to this /// trie. pub fn db_items_remaining(&self) -> HashMap { - let mut ret = self.db().keys(); + let mut ret = self.db.keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { let keycount = *ret.get(&k).unwrap_or(&0); match keycount == v as i32 { @@ -892,7 +892,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { } } -impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> Trie for TrieDB<'db> { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { @@ -915,7 +915,7 @@ impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { } } -impl<'db, T> fmt::Debug for TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> fmt::Debug for TrieDB<'db> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); @@ -960,7 +960,7 @@ mod tests { } } - fn populate_trie<'db, T>(db: &'db mut T, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db> { let mut t = TrieDB::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; @@ -970,7 +970,7 @@ mod tests { t } - fn unpopulate_trie<'a, 'db, T>(t: &mut TrieDB<'db, T>, v: &Vec<(Vec, Vec)>) where T: 'db + HashDB { + fn unpopulate_trie<'a, 'db>(t: &mut TrieDB<'db>, v: &Vec<(Vec, Vec)>) { for i in v.iter() { let key: &[u8]= &i.0; t.remove(&key); From 83af9402234576bad6bd9300560218a027c0bb78 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 13 Dec 2015 22:44:28 +0100 Subject: [PATCH 41/47] h264 --- src/hash.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index b51bbdacf..e86f8e834 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -43,19 +43,17 @@ macro_rules! impl_hash { impl Deref for $from { type Target = [u8]; + #[inline] fn deref(&self) -> &[u8] { - unsafe { - ::std::slice::from_raw_parts(self.0.as_ptr(), $size) - } + &self.0 } } impl DerefMut for $from { + #[inline] fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - ::std::slice::from_raw_parts_mut(self.0.as_mut_ptr(), $size) - } + &mut self.0 } } @@ -327,6 +325,7 @@ impl_hash!(H64, 8); impl_hash!(H128, 16); impl_hash!(Address, 20); impl_hash!(H256, 32); +impl_hash!(H264, 33); impl_hash!(H512, 64); impl_hash!(H520, 65); impl_hash!(H1024, 128); From de4a227a477216bc6a51c7c59a59951791b6282c Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 14 Dec 2015 11:56:11 +0100 Subject: [PATCH 42/47] decoding fixed sized arrays --- src/rlp/errors.rs | 1 + src/rlp/tests.rs | 8 ++++++++ src/rlp/untrusted_rlp.rs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/rlp/errors.rs b/src/rlp/errors.rs index ada3c2a47..4ee41a2ce 100644 --- a/src/rlp/errors.rs +++ b/src/rlp/errors.rs @@ -8,6 +8,7 @@ pub enum DecoderError { RlpIsTooShort, RlpExpectedToBeList, RlpExpectedToBeData, + RlpIncorrectListLen } impl StdError for DecoderError { diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs index e44953e07..b4a60a3de 100644 --- a/src/rlp/tests.rs +++ b/src/rlp/tests.rs @@ -343,3 +343,11 @@ fn test_rlp_json() { }); } +#[test] +fn test_decoding_array() { + let v = vec![5u16, 2u16]; + let res = rlp::encode(&v); + let arr: [u16; 2] = rlp::decode(&res); + assert_eq!(arr[0], 5); + assert_eq!(arr[1], 2); +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index ec68fce99..90bfef96c 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -352,3 +352,38 @@ impl Decodable for Option where T: Decodable { }) } } + +macro_rules! impl_array_decodable { + ($index_type:ty, $len:expr ) => ( + impl Decodable for [T; $len] where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_list(| decoders | { + let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; + if decoders.len() != $len { + return Err(DecoderError::RlpIncorrectListLen); + } + + for i in 0..decoders.len() { + result[i] = try!(T::decode(&decoders[i])); + } + + Ok(result) + }) + } + } + ) +} + +macro_rules! impl_array_decodable_recursive { + ($index_type:ty, ) => (); + ($index_type:ty, $len:expr, $($more:expr,)*) => ( + impl_array_decodable!($index_type, $len); + impl_array_decodable_recursive!($index_type, $($more,)*); + ); +} + +impl_array_decodable_recursive!( + u8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 40, 48, 56, 64, 72, 96, 128, 160, 192, 224, +); From 9231bc66221626c387b019659c84e1905e3f4f5a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 14 Dec 2015 12:03:48 +0100 Subject: [PATCH 43/47] read_list -> as_list --- src/rlp/traits.rs | 5 ++--- src/rlp/untrusted_rlp.rs | 33 +++++++++++++++------------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 4f9ad09e5..067c438bf 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -1,11 +1,10 @@ use rlp::DecoderError; -pub trait Decoder { +pub trait Decoder: Sized { fn read_value(&self, f: F) -> Result where F: FnOnce(&[u8]) -> Result; - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result; + fn as_list(&self) -> Result, DecoderError>; } pub trait Decodable: Sized { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 90bfef96c..a8cecf09f 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -305,13 +305,11 @@ impl<'a> Decoder for BasicDecoder<'a> { } } - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result { - + fn as_list(&self) -> Result, DecoderError> { let v: Vec> = self.rlp.iter() .map(| i | BasicDecoder::new(i)) .collect(); - f(&v) + Ok(v) } } @@ -325,9 +323,8 @@ impl Decodable for T where T: FromBytes { impl Decodable for Vec where T: Decodable { fn decode(decoder: &D) -> Result where D: Decoder { - decoder.read_list(| decoders | { - decoders.iter().map(|d| T::decode(d)).collect() - }) + let decoders = try!(decoder.as_list()); + decoders.iter().map(|d| T::decode(d)).collect() } } @@ -357,18 +354,18 @@ macro_rules! impl_array_decodable { ($index_type:ty, $len:expr ) => ( impl Decodable for [T; $len] where T: Decodable { fn decode(decoder: &D) -> Result where D: Decoder { - decoder.read_list(| decoders | { - let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; - if decoders.len() != $len { - return Err(DecoderError::RlpIncorrectListLen); - } - - for i in 0..decoders.len() { - result[i] = try!(T::decode(&decoders[i])); - } + let decoders = try!(decoder.as_list()); - Ok(result) - }) + let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; + if decoders.len() != $len { + return Err(DecoderError::RlpIncorrectListLen); + } + + for i in 0..decoders.len() { + result[i] = try!(T::decode(&decoders[i])); + } + + Ok(result) } } ) From 1e6694ec7f48e4ba5688516a3a767fd6b15536af Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 15 Dec 2015 16:36:38 +0100 Subject: [PATCH 44/47] removed unused stuff --- src/db.rs | 0 src/lib.rs | 1 - src/uint.rs | 5 ----- 3 files changed, 6 deletions(-) delete mode 100644 src/db.rs diff --git a/src/db.rs b/src/db.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib.rs b/src/lib.rs index 12998529a..24b404a88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,6 @@ pub mod uint; pub mod bytes; pub mod rlp; pub mod vector; -pub mod db; pub mod sha3; pub mod hashdb; pub mod memorydb; diff --git a/src/uint.rs b/src/uint.rs index 09b654270..88ed49712 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -396,7 +396,6 @@ macro_rules! construct_uint { ); } -construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); @@ -410,10 +409,6 @@ impl From for U256 { } } - - - - pub const ZERO_U256: U256 = U256([0x00u64; 4]); pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); From ca1a6bd7912e77901c74abc9ebf45e875dd53d31 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:23:08 +0100 Subject: [PATCH 45/47] heapsize && squeeze --- Cargo.toml | 1 + src/heapsizeof.rs | 5 +++++ src/lib.rs | 6 ++++++ src/squeeze.rs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 src/heapsizeof.rs create mode 100644 src/squeeze.rs diff --git a/Cargo.toml b/Cargo.toml index 13295f766..4d23f49f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" elastic-array = "0.4" +heapsize = "0.2" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/heapsizeof.rs b/src/heapsizeof.rs new file mode 100644 index 000000000..c6d4cace4 --- /dev/null +++ b/src/heapsizeof.rs @@ -0,0 +1,5 @@ +use uint::*; +use hash::*; + +known_heap_size!(0, H32, H64, H128, Address, H256, H264, H512, H520, H1024, H2048); +known_heap_size!(0, U128, U256); diff --git a/src/lib.rs b/src/lib.rs index 24b404a88..9120f0977 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,10 +32,14 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; + +#[macro_use] +extern crate heapsize; #[macro_use] extern crate log; #[macro_use] extern crate lazy_static; + extern crate env_logger; extern crate time; @@ -60,5 +64,7 @@ pub mod crypto; pub mod triehash; pub mod trie; pub mod nibbleslice; +pub mod heapsizeof; +pub mod squeeze; //pub mod network; diff --git a/src/squeeze.rs b/src/squeeze.rs new file mode 100644 index 000000000..220c61800 --- /dev/null +++ b/src/squeeze.rs @@ -0,0 +1,33 @@ +use std::collections::HashMap; +use std::hash::Hash; +use heapsize::HeapSizeOf; + +/// Should be used to squeeze collections to certain size in bytes +trait Squeeze { + fn squeeze(&mut self, size: usize); +} + +impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: HeapSizeOf { + fn squeeze(&mut self, size: usize) { + if self.len() == 0 { + return + } + + let size_of_entry = self.heap_size_of_children() / self.capacity(); + let mut shrinked_size = size_of_entry * self.len(); + + while self.len() > 0 || shrinked_size > size { + // could be optimized + let key = self.keys().next().unwrap().clone(); + self.remove(&key); + shrinked_size -= size_of_entry; + } + + self.shrink_to_fit(); + + // if we havent shrinked enough, squeeze again + if self.heap_size_of_children() > size { + self.squeeze(size); + } + } +} From 56de081381f18e35bcc10c38858f13cd0ffc9768 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:27:27 +0100 Subject: [PATCH 46/47] squeeze is pub --- src/squeeze.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index 220c61800..ddec2a3a9 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use heapsize::HeapSizeOf; /// Should be used to squeeze collections to certain size in bytes -trait Squeeze { +pub trait Squeeze { fn squeeze(&mut self, size: usize); } From 926efb6fc8c5eff915201bb01fe7f8435e4efc2f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 17:12:20 +0100 Subject: [PATCH 47/47] docs, tests and bugfixes for squeeze --- src/squeeze.rs | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index ddec2a3a9..e81a13793 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -1,3 +1,35 @@ +//! Helper module that should be used to randomly squeeze +//! caches to a given size in bytes +//! +//! ``` +//! extern crate heapsize; +//! extern crate ethcore_util as util; +//! use std::collections::HashMap; +//! use std::mem::size_of; +//! use heapsize::HeapSizeOf; +//! use util::squeeze::Squeeze; +//! +//! fn main() { +//! let initial_size = 60; +//! let mut map: HashMap = HashMap::with_capacity(initial_size); +//! assert!(map.capacity() >= initial_size); +//! for i in 0..initial_size { +//! map.insert(i as u8, i as u8); +//! } +//! +//! assert_eq!(map.heap_size_of_children(), map.capacity() * 2 * size_of::()); +//! assert_eq!(map.len(), initial_size); +//! let initial_heap_size = map.heap_size_of_children(); +//! +//! // squeeze it to size of key and value +//! map.squeeze(2 * size_of::()); +//! assert_eq!(map.len(), 1); +//! +//! // its likely that heap size was reduced, but we can't be 100% sure +//! assert!(initial_heap_size >= map.heap_size_of_children()); +//! } +//! ``` + use std::collections::HashMap; use std::hash::Hash; use heapsize::HeapSizeOf; @@ -14,9 +46,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: } let size_of_entry = self.heap_size_of_children() / self.capacity(); - let mut shrinked_size = size_of_entry * self.len(); + let all_entries = size_of_entry * self.len(); + let mut shrinked_size = all_entries; - while self.len() > 0 || shrinked_size > size { + while self.len() > 0 && shrinked_size > size { // could be optimized let key = self.keys().next().unwrap().clone(); self.remove(&key); @@ -25,9 +58,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: self.shrink_to_fit(); - // if we havent shrinked enough, squeeze again - if self.heap_size_of_children() > size { + // if we squeezed something, but not enough, squeeze again + if all_entries != shrinked_size && self.heap_size_of_children() > size { self.squeeze(size); } } } +