diff --git a/.travis.yml b/.travis.yml index 81a39492f..eedbcaaad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: rust rust: - - 1.4.0 + - beta os: #- linux diff --git a/src/bytes.rs b/src/bytes.rs index c08c4ee2a..e17ccca9a 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -23,7 +23,7 @@ //! use util::bytes::FromBytes; //! //! let a = String::from_bytes(&[b'd', b'o', b'g']); -//! let b = u8::from_bytes(&[0xfa]); +//! let b = u16::from_bytes(&[0xfa]); //! let c = u64::from_bytes(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); //! } //! @@ -104,28 +104,6 @@ impl ToBytes for String { fn to_bytes_len(&self) -> usize { self.len() } } -impl ToBytes for u8 { - fn to_bytes(&self) -> Vec { - match *self { - 0 => vec![], - _ => vec![*self] - } - } - - fn to_bytes_len(&self) -> usize { - match *self { - 0 => 0, - _ => 1 - } - } - fn first_byte(&self) -> Option { - match *self { - 0 => None, - _ => Some(*self) - } - } -} - impl ToBytes for u64 { fn to_bytes(&self) -> Vec { let mut res= vec![]; @@ -223,15 +201,6 @@ impl FromBytes for String { } } -impl FromBytes for u8 { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - match bytes.len() { - 0 => Ok(0), - _ => Ok(bytes[0]) - } - } -} - impl FromBytes for u64 { fn from_bytes(bytes: &[u8]) -> FromBytesResult { match bytes.len() { @@ -293,3 +262,4 @@ impl FromBytes for T where T: FixedHash { } } } + diff --git a/src/hashdb.rs b/src/hashdb.rs index 48f5182bc..dd0973bb3 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,5 +1,5 @@ use hash::*; -use bytes::Bytes; +use bytes::*; pub trait HashDB { /// Look up a given hash into the bytes that hash to it, returning None if the @@ -17,7 +17,7 @@ pub trait HashDB { /// assert_eq!(m.lookup(&hash).unwrap(), hello_bytes); /// } /// ``` - fn lookup(&self, key: &H256) -> Option; + fn lookup(&self, key: &H256) -> Option<&[u8]>; /// Check for the existance of a hash-key. /// @@ -57,6 +57,9 @@ pub trait HashDB { /// ``` fn insert(&mut self, value: &[u8]) -> H256; + /// Like `insert()` , except you provide the key and the data is all moved. + fn emplace(&mut self, key: H256, value: Bytes); + /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may /// happen without the data being eventually being inserted into the DB. /// diff --git a/src/lib.rs b/src/lib.rs index b39558d1b..abc8017df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,9 @@ pub mod overlaydb; pub mod math; pub mod chainfilter; pub mod crypto; +pub mod triehash; +pub mod trie; +pub mod nibbleslice; pub mod network; diff --git a/src/memorydb.rs b/src/memorydb.rs index cbb9e426b..14fae12d0 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -37,6 +37,12 @@ use std::collections::HashMap; /// m.kill(&k); /// assert!(!m.exists(&k)); /// +/// m.kill(&k); +/// assert!(!m.exists(&k)); +/// +/// m.insert(d); +/// assert!(!m.exists(&k)); + /// m.insert(d); /// assert!(m.exists(&k)); /// assert_eq!(m.lookup(&k).unwrap(), d); @@ -100,12 +106,22 @@ impl MemoryDB { mem::swap(&mut self.data, &mut data); data } + + pub fn denote(&self, key: &H256, value: Bytes) -> &(Bytes, i32) { + if self.data.get(&key) == None { + unsafe { + let p = &self.data as *const HashMap as *mut HashMap; + (*p).insert(key.clone(), (value, 0)); + } + } + self.data.get(key).unwrap() + } } impl HashDB for MemoryDB { - fn lookup(&self, key: &H256) -> Option { + fn lookup(&self, key: &H256) -> Option<&[u8]> { match self.data.get(key) { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + Some(&(ref d, rc)) if rc > 0 => Some(d), _ => None } } @@ -120,9 +136,9 @@ impl HashDB for MemoryDB { fn insert(&mut self, value: &[u8]) -> H256 { let key = value.sha3(); if match self.data.get_mut(&key) { - Some(&mut (ref mut old_value, ref mut rc @ 0)) => { + Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { *old_value = From::from(value.bytes()); - *rc = 1; + *rc += 1; false }, Some(&mut (_, ref mut x)) => { *x += 1; false } , @@ -133,6 +149,20 @@ impl HashDB for MemoryDB { key } + fn emplace(&mut self, key: H256, value: Bytes) { + match self.data.get_mut(&key) { + Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { + *old_value = value; + *rc += 1; + return; + }, + Some(&mut (_, ref mut x)) => { *x += 1; return; } , + None => {}, + } + // ... None falls through into... + self.data.insert(key, (value, 1)); + } + fn kill(&mut self, key: &H256) { if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } @@ -143,3 +173,20 @@ impl HashDB for MemoryDB { } } +#[test] +fn memorydb_denote() { + let mut m = MemoryDB::new(); + let hello_bytes = b"Hello world!"; + let hash = m.insert(hello_bytes); + assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); + + for _ in 0..1000 { + let r = H256::random(); + let k = r.sha3(); + let &(ref v, ref rc) = m.denote(&k, r.bytes().to_vec()); + assert_eq!(v, &r.bytes()); + assert_eq!(*rc, 0); + } + + assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); +} \ No newline at end of file diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs new file mode 100644 index 000000000..6ccf695c0 --- /dev/null +++ b/src/nibbleslice.rs @@ -0,0 +1,195 @@ +//! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. +use std::cmp::*; +use bytes::*; + +/// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. +/// +/// This is an immutable struct. No operations actually change it. +/// +/// # Example +/// ```rust +/// extern crate ethcore_util; +/// use ethcore_util::nibbleslice::*; +/// fn main() { +/// let d1 = &[0x01u8, 0x23, 0x45]; +/// let d2 = &[0x34u8, 0x50, 0x12]; +/// let d3 = &[0x00u8, 0x12]; +/// let n1 = NibbleSlice::new(d1); // 0,1,2,3,4,5 +/// let n2 = NibbleSlice::new(d2); // 3,4,5,0,1,2 +/// let n3 = NibbleSlice::new_offset(d3, 1); // 0,1,2 +/// assert!(n1 > n3); // 0,1,2,... > 0,1,2 +/// assert!(n1 < n2); // 0,... < 3,... +/// assert!(n2.mid(3) == n3); // 0,1,2 == 0,1,2 +/// assert!(n1.starts_with(&n3)); +/// assert_eq!(n1.common_prefix(&n3), 3); +/// assert_eq!(n2.mid(3).common_prefix(&n1), 3); +/// } +/// ``` +#[derive(Debug, Copy, Clone, Eq, Ord)] +pub struct NibbleSlice<'a> { + data: &'a [u8], + offset: usize, +} + +impl<'a> NibbleSlice<'a> { + /// Create a new nibble slice with the given byte-slice. + pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) } + + /// Create a new nibble slice with the given byte-slice with a nibble offset. + pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset} } + + /// Create a new nibble slice from the given HPE encoded data (e.g. output of `encoded()`). + pub fn from_encoded(data: &'a [u8]) -> (NibbleSlice, bool) { + (Self::new_offset(data, if data[0] & 16 == 16 {1} else {2}), data[0] & 32 == 32) + } + + /// Is this an empty slice? + pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Get the length (in nibbles, naturally) of this slice. + pub fn len(&self) -> usize { self.data.len() * 2 - self.offset } + + /// Get the nibble at position `i`. + pub fn at(&self, i: usize) -> u8 { + if (self.offset + i) & 1 == 1 { + self.data[(self.offset + i) / 2] & 15u8 + } + else { + self.data[(self.offset + i) / 2] >> 4 + } + } + + /// Return object which represents a view on to this slice (further) offset by `i` nibbles. + pub fn mid(&self, i: usize) -> Self { NibbleSlice{ data: self.data, offset: self.offset + i} } + + /// Do we start with the same nibbles as the whole of `them`? + pub fn starts_with(&self, them: &Self) -> bool { self.common_prefix(them) == them.len() } + + /// How many of the same nibbles at the beginning do we match with `them`? + pub fn common_prefix(&self, them: &Self) -> usize { + let s = min(self.len(), them.len()); + let mut i = 0usize; + while i < s { + if self.at(i) != them.at(i) { break; } + i += 1; + } + i + } + + pub fn encoded(&self, is_leaf: bool) -> Bytes { + let l = self.len(); + let mut r = Bytes::with_capacity(l / 2 + 1); + let mut i = l % 2; + r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0}); + while i < l { + r.push(self.at(i) * 16 + self.at(i + 1)); + i += 2; + } + r + } +} + +impl<'a> PartialEq for NibbleSlice<'a> { + fn eq(&self, them: &Self) -> bool { + self.len() == them.len() && self.starts_with(them) + } +} + +impl<'a> PartialOrd for NibbleSlice<'a> { + fn partial_cmp(&self, them: &Self) -> Option { + let s = min(self.len(), them.len()); + let mut i = 0usize; + while i < s { + match self.at(i).partial_cmp(&them.at(i)).unwrap() { + Ordering::Less => return Some(Ordering::Less), + Ordering::Greater => return Some(Ordering::Greater), + _ => i += 1, + } + } + self.len().partial_cmp(&them.len()) + } +} + +#[cfg(test)] +mod tests { + use super::NibbleSlice; + static D: &'static [u8;3] = &[0x01u8, 0x23, 0x45]; + + #[test] + fn basics() { + let n = NibbleSlice::new(D); + assert_eq!(n.len(), 6); + assert!(!n.is_empty()); + + let n = NibbleSlice::new_offset(D, 6); + assert!(n.is_empty()); + + let n = NibbleSlice::new_offset(D, 3); + assert_eq!(n.len(), 3); + for i in 0..3 { + assert_eq!(n.at(i), i as u8 + 3); + } + } + + #[test] + fn mid() { + let n = NibbleSlice::new(D); + let m = n.mid(2); + for i in 0..4 { + assert_eq!(m.at(i), i as u8 + 2); + } + let m = n.mid(3); + for i in 0..3 { + assert_eq!(m.at(i), i as u8 + 3); + } + } + + #[test] + fn encoded() { + let n = NibbleSlice::new(D); + assert_eq!(n.encoded(false), &[0x00, 0x01, 0x23, 0x45]); + assert_eq!(n.encoded(true), &[0x20, 0x01, 0x23, 0x45]); + assert_eq!(n.mid(1).encoded(false), &[0x11, 0x23, 0x45]); + assert_eq!(n.mid(1).encoded(true), &[0x31, 0x23, 0x45]); + } + + #[test] + fn from_encoded() { + let n = NibbleSlice::new(D); + assert_eq!((n, false), NibbleSlice::from_encoded(&[0x00, 0x01, 0x23, 0x45])); + assert_eq!((n, true), NibbleSlice::from_encoded(&[0x20, 0x01, 0x23, 0x45])); + assert_eq!((n.mid(1), false), NibbleSlice::from_encoded(&[0x11, 0x23, 0x45])); + assert_eq!((n.mid(1), true), NibbleSlice::from_encoded(&[0x31, 0x23, 0x45])); + } + + #[test] + fn shared() { + let n = NibbleSlice::new(D); + + let other = &[0x01u8, 0x23, 0x01, 0x23, 0x45, 0x67]; + let m = NibbleSlice::new(other); + + assert_eq!(n.common_prefix(&m), 4); + assert_eq!(m.common_prefix(&n), 4); + assert_eq!(n.mid(1).common_prefix(&m.mid(1)), 3); + assert_eq!(n.mid(1).common_prefix(&m.mid(2)), 0); + assert_eq!(n.common_prefix(&m.mid(4)), 6); + assert!(!n.starts_with(&m.mid(4))); + assert!(m.mid(4).starts_with(&n)); + } + + #[test] + fn compare() { + let other = &[0x01u8, 0x23, 0x01, 0x23, 0x45]; + let n = NibbleSlice::new(D); + let m = NibbleSlice::new(other); + + assert!(n != m); + assert!(n > m); + assert!(m < n); + + assert!(n == m.mid(4)); + assert!(n >= m.mid(4)); + assert!(n <= m.mid(4)); + } +} \ No newline at end of file diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 0f6ae3d77..d69afa0f1 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -130,24 +130,24 @@ impl OverlayDB { let mut s = RlpStream::new_list(2); s.append(&payload.1); s.append(&payload.0); - self.backing.put(&key.bytes(), &s.out().unwrap()).expect("Low-level database error. Some issue with your hard disk?"); + self.backing.put(&key.bytes(), &s.out()).expect("Low-level database error. Some issue with your hard disk?"); } } impl HashDB for OverlayDB { - fn lookup(&self, key: &H256) -> Option { + fn lookup(&self, key: &H256) -> Option<&[u8]> { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. let k = self.overlay.raw(key); match k { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + Some(&(ref d, rc)) if rc > 0 => Some(d), _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { Some(x) => { let (d, rc) = x; if rc as i32 + memrc > 0 { - Some(d) + Some(&self.overlay.denote(key, d).0) } else { None @@ -186,6 +186,7 @@ impl HashDB for OverlayDB { } } fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } + fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); } fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } @@ -193,7 +194,7 @@ impl HashDB for OverlayDB { fn overlaydb_overlay_insert_and_kill() { let mut trie = OverlayDB::new_temp(); let h = trie.insert(b"hello world"); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.kill(&h); assert_eq!(trie.lookup(&h), None); } @@ -202,11 +203,11 @@ fn overlaydb_overlay_insert_and_kill() { fn overlaydb_backing_insert_revert() { let mut trie = OverlayDB::new_temp(); let h = trie.insert(b"hello world"); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.revert(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); } #[test] @@ -230,7 +231,7 @@ fn overlaydb_backing_kill_revert() { trie.kill(&h); assert_eq!(trie.lookup(&h), None); trie.revert(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); } #[test] @@ -248,29 +249,29 @@ fn overlaydb_negative() { fn overlaydb_complex() { let mut trie = OverlayDB::new_temp(); let hfoo = trie.insert(b"foo"); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); let hbar = trie.insert(b"bar"); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.insert(b"foo"); // two refs - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.kill(&hbar); // zero refs - delete assert_eq!(trie.lookup(&hbar), None); trie.kill(&hfoo); // one ref - keep - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.kill(&hfoo); // zero ref - would delete, but... assert_eq!(trie.lookup(&hfoo), None); trie.insert(b"foo"); // one ref - keep after all. - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.kill(&hfoo); // zero ref - delete assert_eq!(trie.lookup(&hfoo), None); trie.commit().unwrap(); // diff --git a/src/rlp.rs b/src/rlp.rs index ef1232c55..e8562828a 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1,85 +1,34 @@ -//! Rlp serialization module -//! -//! Types implementing `Endocable` and `Decodable` traits -//! can be easily coverted to and from rlp. -//! Trusted rlp should be decoded with `Rlp`, untrusted with `UntrustedRlp`. -//! -//! # Examples: -//! -//! ```rust -//! extern crate ethcore_util; -//! use ethcore_util::rlp::{Rlp, UntrustedRlp, RlpStream, Decodable}; -//! -//! fn encode_value() { -//! // 1029 -//! let mut stream = RlpStream::new(); -//! stream.append(&1029u32); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0x82, 0x04, 0x05]); -//! } -//! -//! fn encode_list() { -//! // [ "cat", "dog" ] -//! let mut stream = RlpStream::new_list(2); -//! stream.append(&"cat").append(&"dog"); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); -//! } -//! -//! fn encode_list2() { -//! // [ [], [[]], [ [], [[]] ] ] -//! let mut stream = RlpStream::new_list(3); -//! stream.append_list(0); -//! stream.append_list(1).append_list(0); -//! stream.append_list(2).append_list(0).append_list(1).append_list(0); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); -//! } -//! -//! fn decode_value() { -//! // 0x102456789abcdef -//! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; -//! let rlp = Rlp::new(&data); -//! let _ = u64::decode(&rlp); -//! } +//! Rlp serialization module //! -//! fn decode_untrusted_string() { -//! // "cat" -//! let data = vec![0x83, b'c', b'a', b't']; -//! let rlp = UntrustedRlp::new(&data); -//! let _ = String::decode_untrusted(&rlp).unwrap(); -//! } -//! -//! fn decode_list() { -//! // ["cat", "dog"] -//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = Rlp::new(&data); -//! let _ : Vec = Decodable::decode(&rlp); -//! } -//! -//! fn decode_list2() { -//! // [ [], [[]], [ [], [[]] ] ] -//! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; -//! let rlp = Rlp::new(&data); -//! let _v0: Vec = Decodable::decode(&rlp.at(0)); -//! let _v1: Vec> = Decodable::decode(&rlp.at(1)); -//! let nested_rlp = rlp.at(2); -//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); -//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); -//! } +//! Allows encoding, decoding, and view onto rlp-slice //! -//! fn main() { -//! encode_value(); -//! encode_list(); -//! encode_list2(); +//!# What should you use when? //! -//! decode_value(); -//! decode_untrusted_string(); -//! decode_list(); -//! decode_list2(); -//! } -//! ``` +//!### Use `encode` function when: +//! * You want to encode something inline. +//! * You do not work on big set of data. +//! * You want to encode whole data structure at once. //! +//!### Use `decode` function when: +//! * You want to decode something inline. +//! * You do not work on big set of data. +//! * You want to decode whole rlp at once. +//! +//!### Use `RlpStream` when: +//! * You want to encode something in portions. +//! * You encode a big set of data. +//! +//!### Use `Rlp` when: +//! * You are working on trusted data (not corrupted). +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. +//! +//!### Use `UntrustedRlp` when: +//! * You are working on untrusted data (~corrupted). +//! * You need to handle data corruption errors. +//! * You are working on input data. +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. use std::fmt; use std::cell::Cell; @@ -88,7 +37,12 @@ use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; -/// rlp container +/// Data-oriented view onto rlp-slice. +/// +/// This is immutable structere. No operations change it. +/// +/// Should be used in places where, error handling is required, +/// eg. on input #[derive(Debug)] pub struct UntrustedRlp<'a> { bytes: &'a [u8], @@ -129,10 +83,10 @@ impl ItemInfo { #[derive(Debug, PartialEq, Eq)] pub enum DecoderError { FromBytesError(FromBytesError), - UntrustedRlpIsTooShort, - UntrustedRlpExpectedToBeList, - UntrustedRlpExpectedToBeValue, - BadUntrustedRlp, + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, + BadRlp, } impl StdError for DecoderError { fn description(&self) -> &str { @@ -152,10 +106,10 @@ impl From for DecoderError { } } -/// Unsafe wrapper for rlp decode_untrustedr. +/// Data-oriented view onto trusted rlp-slice. /// -/// It assumes that you know what you are doing. Doesn't bother -/// you with error handling. +/// Unlikely to `UntrustedRlp` doesn't bother you with error +/// handling. It assumes that you know what you are doing. pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } @@ -172,31 +126,202 @@ impl<'a> From> for UntrustedRlp<'a> { } } +pub enum Prototype { + Null, + Data(usize), + List(usize), +} + impl<'a> Rlp<'a> { - /// returns new instance of `Rlp` + /// Create a new instance of `Rlp` pub fn new(bytes: &'a [u8]) -> Rlp<'a> { Rlp { rlp: UntrustedRlp::new(bytes) } } + /// Get the prototype of the RLP. + pub fn prototype(&self) -> Prototype { + if self.is_data() { + Prototype::Data(self.size()) + } + else if self.is_list() { + Prototype::List(self.item_count()) + } + else { + Prototype::Null + } + } + + /// The bare data of the rlp. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let view = rlp.at(1); + /// let dog = view.data(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn data(&self) -> &[u8] { + self.rlp.data() + } + + /// Returns number of rlp items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.item_count(), 2); + /// let view = rlp.at(1); + /// assert_eq!(view.item_count(), 0); + /// } + /// ``` + pub fn item_count(&self) -> usize { + self.rlp.item_count() + } + + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1); + /// assert_eq!(view.size(), 3); + /// } + /// ``` + pub fn size(&self) -> usize { + self.rlp.size() + } + + /// Get view onto rlp-slice at index. + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = String::decode(&rlp.at(1)); + /// assert_eq!(dog, "dog".to_string()); + /// } + /// ``` pub fn at(&self, index: usize) -> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } - /// returns true if rlp is a list + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` + pub fn is_null(&self) -> bool { + self.rlp.is_null() + } + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + self.rlp.is_empty() + } + + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` pub fn is_list(&self) -> bool { self.rlp.is_list() } - /// returns true if rlp is a value - pub fn is_value(&self) -> bool { - self.rlp.is_value() + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.at(1).is_data()); + /// } + /// ``` + pub fn is_data(&self) -> bool { + self.rlp.is_data() } - /// returns rlp iterator - pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { - self.rlp.into_iter() + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).is_int(), true); + /// } + /// ``` + pub fn is_int(&self) -> bool { + self.rlp.is_int() + } + + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// } + /// ``` + pub fn iter(&'a self) -> RlpIterator<'a> { + self.into_iter() } } @@ -209,12 +334,85 @@ impl<'a> UntrustedRlp<'a> { } } - /// get container subset at given index - /// - /// paren container caches searched position + /// The bare data of the rlp. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let view = rlp.at(1).unwrap(); + /// let dog = view.data(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn data(&self) -> &[u8] { + self.bytes + } + + /// Returns number of rlp items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.item_count(), 2); + /// let view = rlp.at(1).unwrap(); + /// assert_eq!(view.item_count(), 0); + /// } + /// ``` + pub fn item_count(&self) -> usize { + match self.is_list() { + true => self.iter().count(), + false => 0 + } + } + + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1).unwrap(); + /// assert_eq!(view.size(), 3); + /// } + /// ``` + pub fn size(&self) -> usize { + match self.is_data() { + true => Self::item_info(self.bytes).unwrap().value_len, + false => 0 + } + } + + /// Get view onto rlp-slice at index + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let dog = String::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); + /// assert_eq!(dog, "dog".to_string()); + /// } + /// ``` pub fn at(&self, index: usize) -> Result, DecoderError> { if !self.is_list() { - return Err(DecoderError::UntrustedRlpExpectedToBeList); + return Err(DecoderError::RlpExpectedToBeList); } // move to cached position if it's index is less or equal to @@ -236,17 +434,111 @@ impl<'a> UntrustedRlp<'a> { Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) } - /// returns true if rlp is a list + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` + pub fn is_null(&self) -> bool { + self.bytes.len() == 0 + } + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) + } + + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` pub fn is_list(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] >= 0xc0 + !self.is_null() && self.bytes[0] >= 0xc0 } - /// returns true if rlp is a value - pub fn is_value(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] <= 0xbf + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.at(1).unwrap().is_data()); + /// } + /// ``` + pub fn is_data(&self) -> bool { + !self.is_null() && self.bytes[0] < 0xc0 } - /// returns rlp iterator + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).unwrap().is_int(), true); + /// } + /// ``` + pub fn is_int(&self) -> bool { + if self.is_null() { + return false; + } + + match self.bytes[0] { + 0...0x80 => true, + 0x81...0xb7 => self.bytes[1] != 0, + b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, + _ => false + } + } + + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let strings: Vec = rlp.iter() + /// .map(| i | String::decode_untrusted(&i)) + /// .map(| s | s.unwrap()) + /// .collect(); + /// } + /// ``` pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { self.into_iter() } @@ -270,10 +562,10 @@ impl<'a> UntrustedRlp<'a> { /// return first item info /// - /// TODO: move this to decode_untrustedr? + /// TODO: move this to decoder (?) fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::UntrustedRlpIsTooShort), + None => return Err(DecoderError::RlpIsTooShort), Some(0...0x7f) => ItemInfo::new(0, 1), Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), Some(l @ 0xb8...0xbf) => { @@ -289,12 +581,12 @@ impl<'a> UntrustedRlp<'a> { let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); ItemInfo::new(prefix_len, value_len) } - _ => return Err(DecoderError::BadUntrustedRlp), + _ => return Err(DecoderError::BadRlp), }; match item.prefix_len + item.value_len <= bytes.len() { true => Ok(item), - false => Err(DecoderError::UntrustedRlpIsTooShort), + false => Err(DecoderError::RlpIsTooShort), } } @@ -302,12 +594,12 @@ impl<'a> UntrustedRlp<'a> { fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { match bytes.len() >= len { true => Ok(&bytes[len..]), - false => Err(DecoderError::UntrustedRlpIsTooShort), + false => Err(DecoderError::RlpIsTooShort), } } } -/// non-consuming rlp iterator +/// Iterator over rlp-slice list elements. pub struct UntrustedRlpIterator<'a> { rlp: &'a UntrustedRlp<'a>, index: usize, @@ -315,7 +607,7 @@ pub struct UntrustedRlpIterator<'a> { impl<'a> IntoIterator for &'a UntrustedRlp<'a> { type Item = UntrustedRlp<'a>; - type IntoIter = UntrustedRlpIterator<'a>; + type IntoIter = UntrustedRlpIterator<'a>; fn into_iter(self) -> Self::IntoIter { UntrustedRlpIterator { @@ -336,20 +628,52 @@ impl<'a> Iterator for UntrustedRlpIterator<'a> { } } -/// shortcut function to decoded Trusted Rlp `&[u8]` into an object +/// Iterator over trusted rlp-slice list elements. +pub struct RlpIterator<'a> { + rlp: &'a Rlp<'a>, + index: usize +} + +impl<'a> IntoIterator for &'a Rlp<'a> { + type Item = Rlp<'a>; + type IntoIter = RlpIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + RlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a> Iterator for RlpIterator<'a> { + type Item = Rlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); + self.index += 1; + result + } +} + +/// Shortcut function to decode trusted rlp +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +/// let animals: Vec = decode(&data); +/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); +/// } +/// ``` pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); T::decode(&rlp) } -/// shortcut function to decode UntrustedRlp `&[u8]` into an object -pub fn decode_untrusted(bytes: &[u8]) -> Result - where T: Decodable -{ - let rlp = UntrustedRlp::new(bytes); - T::decode_untrusted(&rlp) -} - pub trait Decodable: Sized { fn decode_untrusted(rlp: &UntrustedRlp) -> Result; fn decode(rlp: &Rlp) -> Self { @@ -357,51 +681,62 @@ pub trait Decodable: Sized { } } -impl Decodable for T where T: FromBytes -{ +impl Decodable for T where T: FromBytes { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_value() { - true => BasicDecoder::read_value(rlp.bytes), - false => Err(DecoderError::UntrustedRlpExpectedToBeValue), + match rlp.is_data() { + true => BasicDecoder::read_value(rlp.bytes, | bytes | { + Ok(try!(T::from_bytes(bytes))) + }), + false => Err(DecoderError::RlpExpectedToBeData), } } } -impl Decodable for Vec where T: Decodable -{ +impl Decodable for Vec where T: Decodable { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_list() { true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), - false => Err(DecoderError::UntrustedRlpExpectedToBeList), + false => Err(DecoderError::RlpExpectedToBeList), + } + } +} + +impl Decodable for Vec { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { + match rlp.is_data() { + true => BasicDecoder::read_value(rlp.bytes, | bytes | { + let mut res = vec![]; + res.extend(bytes); + Ok(res) + }), + false => Err(DecoderError::RlpExpectedToBeData), } } } pub trait Decoder { - fn read_value(bytes: &[u8]) -> Result where T: FromBytes; + fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result; } struct BasicDecoder; impl Decoder for BasicDecoder { - fn read_value(bytes: &[u8]) -> Result - where T: FromBytes - { + fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { match bytes.first().map(|&x| x) { // rlp is too short - None => Err(DecoderError::UntrustedRlpIsTooShort), + None => Err(DecoderError::RlpIsTooShort), // single byt value - Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), + Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(T::from_bytes(&bytes[1..(1 + l as usize - 0x80)]))), + Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), // longer than 55 bytes Some(l @ 0xb8...0xbf) => { let len_of_len = l as usize - 0xb7; let begin_of_value = 1 as usize + len_of_len; let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); - Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) + Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } - _ => Err(DecoderError::BadUntrustedRlp), + _ => Err(DecoderError::BadRlp), } } } @@ -423,15 +758,14 @@ impl ListInfo { } } -/// container that should be used to encode rlp +/// Appendable rlp encoder. pub struct RlpStream { unfinished_lists: LinkedList, encoder: BasicEncoder, } impl RlpStream { - /// create new container for values appended one after another, - /// but not being part of the same list + /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { unfinished_lists: LinkedList::new(), @@ -439,28 +773,51 @@ impl RlpStream { } } - /// create new container for list of size `max_len` + /// Initializes the `RLPStream` as a list. pub fn new_list(len: usize) -> RlpStream { let mut stream = RlpStream::new(); stream.append_list(len); stream } - /// apends value to the end of stream, chainable - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream - where E: Encodable - { + /// Apends value to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { // encode given value and add it at the end of the stream object.encode(&mut self.encoder); // if list is finished, prepend the length - self.try_to_finish(); + self.note_appended(1); // return chainable self self } - /// declare appending the list of given size + /// Declare appending the list of given size, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); + /// } + /// ``` pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { // push new list let position = self.encoder.bytes.len(); @@ -468,7 +825,7 @@ impl RlpStream { 0 => { // we may finish, if the appended list len is equal 0 self.encoder.bytes.push(0xc0u8); - self.try_to_finish(); + self.note_appended(1); } _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), } @@ -477,25 +834,102 @@ impl RlpStream { self } - /// return true if stream is ready + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` + pub fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { + // self push raw item + self.encoder.bytes.push(0x80); + + // try to finish and prepend the length + self.note_appended(1); + + // return chainable self + self + } + + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. + pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + // push raw items + self.encoder.bytes.extend(bytes); + + // try to finish and prepend the length + self.note_appended(item_count); + + // return chainable self + self + } + + /// Clear the output stream so far. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// } + pub fn clear(&mut self) { + // clear bytes + self.encoder.bytes.clear(); + + // clear lists + self.unfinished_lists.clear(); + } + + /// Returns true if stream doesnt expect any more items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } pub fn is_finished(&self) -> bool { self.unfinished_lists.back().is_none() } - /// streams out encoded bytes - pub fn out(self) -> Result, EncoderError> { + /// Streams out encoded bytes. + /// + /// panic! if stream is not finished. + pub fn out(self) -> Vec { match self.is_finished() { - true => Ok(self.encoder.out()), - false => Err(EncoderError::StreamIsUnfinished), + true => self.encoder.out(), + false => panic!() } } - /// try to finish lists - fn try_to_finish(&mut self) -> () { + /// Try to finish lists + fn note_appended(&mut self, inserted_items: usize) -> () { let should_finish = match self.unfinished_lists.back_mut() { None => false, Some(ref mut x) => { - x.current += 1; + x.current += inserted_items; + if x.current > x.max { + panic!("You cannot append more items then you expect!"); + } x.current == x.max } }; @@ -504,37 +938,30 @@ impl RlpStream { let x = self.unfinished_lists.pop_back().unwrap(); let len = self.encoder.bytes.len() - x.position; self.encoder.insert_list_len_at_pos(len, x.position); - self.try_to_finish(); + self.note_appended(1); } } } -/// shortcut function to encode a `T: Encodable` into a UntrustedRlp `Vec` -pub fn encode(object: &E) -> Vec - where E: Encodable +/// Shortcut function to encode structure into rlp. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let animals = vec!["cat", "dog"]; +/// let out = encode(&animals); +/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +/// } +/// ``` +pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); object.encode(&mut encoder); encoder.out() } -#[derive(Debug)] -pub enum EncoderError { - StreamIsUnfinished, -} - -impl StdError for EncoderError { - fn description(&self) -> &str { - "encoder error" - } -} - -impl fmt::Display for EncoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - pub trait Encodable { fn encode(&self, encoder: &mut E) -> () where E: Encoder; } @@ -544,20 +971,14 @@ pub trait Encoder { fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); } -impl Encodable for T where T: ToBytes -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl Encodable for T where T: ToBytes { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_value(&self.to_bytes()) } } -impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_list(|e| { // insert all list elements for el in self.iter() { @@ -567,16 +988,29 @@ impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a } } -impl Encodable for Vec where T: Encodable -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl Encodable for Vec where T: Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { let r: &[T] = self.as_ref(); r.encode(encoder) } } +/// lets treat bytes differently than other lists +/// they are a single value +impl<'a> Encodable for &'a [u8] { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl Encodable for Vec { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + struct BasicEncoder { bytes: Vec, } @@ -593,7 +1027,7 @@ impl BasicEncoder { match len { 0...55 => res.push(0xc0u8 + len as u8), _ => { - res.push(0x7fu8 + len.to_bytes_len() as u8); + res.push(0xf7u8 + len.to_bytes_len() as u8); res.extend(len.to_bytes()); } }; @@ -628,8 +1062,7 @@ impl Encoder for BasicEncoder { } } - fn emit_list(&mut self, f: F) -> () - where F: FnOnce(&mut Self) -> () + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { // get len before inserting a list let before_len = self.bytes.len(); @@ -664,17 +1097,17 @@ mod tests { assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); - assert!(cat.is_value()); + assert!(cat.is_data()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); - assert!(dog.is_value()); + assert!(dog.is_data()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); + assert!(cat_again.is_data()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); } @@ -688,10 +1121,10 @@ mod tests { assert!(rlp.is_list()); let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::UntrustedRlpIsTooShort); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::UntrustedRlpIsTooShort); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); } } @@ -703,18 +1136,18 @@ mod tests { let mut iter = rlp.iter(); let cat = iter.next().unwrap(); - assert!(cat.is_value()); + assert!(cat.is_data()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); let dog = iter.next().unwrap(); - assert!(dog.is_value()); + assert!(dog.is_data()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); let none = iter.next(); assert!(none.is_none()); let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); + assert!(cat_again.is_data()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); } } @@ -729,81 +1162,67 @@ mod tests { assert_eq!(res, &t.1[..]); } } - - #[test] - fn encode_u8() { - let tests = vec![ - ETestPair(0u8, vec![0x80u8]), - ETestPair(15, vec![15]), - ETestPair(55, vec![55]), - ETestPair(56, vec![56]), - ETestPair(0x7f, vec![0x7f]), - ETestPair(0x80, vec![0x81, 0x80]), - ETestPair(0xff, vec![0x81, 0xff]), - ]; - run_encode_tests(tests); - } - + #[test] fn encode_u16() { let tests = vec![ - ETestPair(0u16, vec![0x80u8]), - ETestPair(0x100, vec![0x82, 0x01, 0x00]), - ETestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; + ETestPair(0u16, vec![0x80u8]), + ETestPair(0x100, vec![0x82, 0x01, 0x00]), + ETestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u32() { let tests = vec![ - ETestPair(0u32, vec![0x80u8]), - ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; + ETestPair(0u32, vec![0x80u8]), + ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u64() { let tests = vec![ - ETestPair(0u64, vec![0x80u8]), - ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; + ETestPair(0u64, vec![0x80u8]), + ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u256() { let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), - ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; run_encode_tests(tests); } #[test] fn encode_str() { let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), - ETestPair("dog", vec![0x83, b'd', b'o', b'g']), - ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), - ETestPair("", vec![0x80]), - ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; run_encode_tests(tests); } @@ -814,37 +1233,39 @@ mod tests { let tests = vec![ ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) ]; run_encode_tests(tests); } + /// Vec is treated as a single value #[test] fn encode_vector_u8() { let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u8], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ]; + ETestPair(vec![], vec![0x80]), + ETestPair(vec![0u8], vec![0]), + ETestPair(vec![0x15], vec![0x15]), + ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; run_encode_tests(tests); } #[test] fn encode_vector_u64() { let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u64], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u64], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_vector_str() { let tests = vec![ETestPair(vec!["cat", "dog"], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; run_encode_tests(tests); } @@ -854,13 +1275,21 @@ mod tests { run_encode_tests(tests); } + #[test] + fn encode_bytes() { + let vec = vec![0u8]; + let slice: &[u8] = &vec; + let res = rlp::encode(&slice); + assert_eq!(res, vec![0u8]); + } + #[test] fn rlp_stream() { let mut stream = RlpStream::new_list(2); stream.append(&"cat").append(&"dog"); - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); } #[test] @@ -869,98 +1298,121 @@ mod tests { stream.append_list(0); stream.append_list(1).append_list(0); stream.append_list(2).append_list(0).append_list(1).append_list(0); - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); } + #[test] + fn rlp_stream_list2() { + let mut stream = RlpStream::new(); + stream.append_list(17); + for _ in 0..17 { + stream.append(&""); + } + let out = stream.out(); + assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); + } + + #[test] + fn rlp_stream_list3() { + let mut stream = RlpStream::new(); + stream.append_list(17); + + let mut res = vec![0xf8, 0x44]; + for _ in 0..17 { + stream.append(&"aaa"); + res.extend(vec![0x83, b'a', b'a', b'a']); + } + let out = stream.out(); + assert_eq!(out, res); + } + struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - fn run_decode_untrusted_tests(tests: Vec>) - where T: rlp::Decodable + fmt::Debug + cmp::Eq - { + fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { for t in &tests { - let res: T = rlp::decode_untrusted(&t.1).unwrap(); + let res: T = rlp::decode(&t.1); assert_eq!(res, t.0); } } + /// Vec is treated as a single value #[test] - fn decode_untrusted_u8() { + fn decode_vector_u8() { let tests = vec![ - DTestPair(0u8, vec![0u8]), - DTestPair(15, vec![15]), - DTestPair(55, vec![55]), - DTestPair(56, vec![56]), - DTestPair(0x7f, vec![0x7f]), - DTestPair(0x80, vec![0x81, 0x80]), - DTestPair(0xff, vec![0x81, 0xff]), - ]; - run_decode_untrusted_tests(tests); + DTestPair(vec![], vec![0x80]), + DTestPair(vec![0u8], vec![0]), + DTestPair(vec![0x15], vec![0x15]), + DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_decode_tests(tests); } #[test] fn decode_untrusted_u16() { let tests = vec![ - DTestPair(0u16, vec![0u8]), - DTestPair(0x100, vec![0x82, 0x01, 0x00]), - DTestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; - run_decode_untrusted_tests(tests); + DTestPair(0u16, vec![0u8]), + DTestPair(0x100, vec![0x82, 0x01, 0x00]), + DTestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_decode_tests(tests); } #[test] fn decode_untrusted_u32() { let tests = vec![ - DTestPair(0u32, vec![0u8]), - DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; - run_decode_untrusted_tests(tests); + DTestPair(0u32, vec![0u8]), + DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); } #[test] fn decode_untrusted_u64() { let tests = vec![ - DTestPair(0u64, vec![0u8]), - DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; - run_decode_untrusted_tests(tests); + DTestPair(0u64, vec![0u8]), + DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); } #[test] fn decode_untrusted_u256() { let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), - DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_decode_untrusted_tests(tests); + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_decode_tests(tests); } #[test] fn decode_untrusted_str() { let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), - DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), - DTestPair("Marek".to_string(), - vec![0x85, b'M', b'a', b'r', b'e', b'k']), - DTestPair("".to_string(), vec![0x80]), - DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" - .to_string(), - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; - run_decode_untrusted_tests(tests); + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), + vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" + .to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_decode_tests(tests); } #[test] @@ -970,44 +1422,34 @@ mod tests { let tests = vec![ DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) ]; - run_decode_untrusted_tests(tests); - } - - #[test] - fn decode_untrusted_vector_u8() { - let tests = vec![ - DTestPair(vec![] as Vec, vec![0xc0]), - DTestPair(vec![15u8], vec![0xc1, 0x0f]), - DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] fn decode_untrusted_vector_u64() { let tests = vec![ - DTestPair(vec![], vec![0xc0]), - DTestPair(vec![15u64], vec![0xc1, 0x0f]), - DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_decode_untrusted_tests(tests); + DTestPair(vec![], vec![0xc0]), + DTestPair(vec![15u64], vec![0xc1, 0x0f]), + DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_decode_tests(tests); } #[test] fn decode_untrusted_vector_str() { let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_decode_untrusted_tests(tests); + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_decode_tests(tests); } #[test] fn decode_untrusted_vector_of_vectors_str() { let tests = vec![DTestPair(vec![vec!["cat".to_string()]], - vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_decode_untrusted_tests(tests); + vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_decode_tests(tests); } } diff --git a/src/trie.rs b/src/trie.rs new file mode 100644 index 000000000..4aebc688f --- /dev/null +++ b/src/trie.rs @@ -0,0 +1,160 @@ +use memorydb::*; +use hashdb::*; +use hash::*; +use nibbleslice::*; +use bytes::*; +use rlp::*; + +pub const NULL_RLP: [u8; 1] = [0x80; 1]; +pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + +/*lazy_static! { + pub static ref NULL_RLP: Bytes = { let mut r = RlpStream::new(); r.append(&""); r.out().unwrap() }; + pub static ref SHA3_NULL_RLP: H256 = { use sha3::Hashable; NULL_RLP.sha3() }; +}*/ + +pub trait Trie { + fn root(&self) -> &H256; + fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } + + // TODO: consider returning &[u8]... + fn contains(&self, key: &[u8]) -> bool; + fn at(&self, key: &[u8]) -> Option<&[u8]>; + fn insert(&mut self, key: &[u8], value: &[u8]); + fn remove(&mut self, key: &[u8]); +} + +pub struct TrieDB { + db: Box, + root: H256, +} + +struct Diff { + new: Vec<(H256, Bytes)>, + old: Vec, +} + +impl Diff { + pub fn new() -> Diff { Diff { new: vec![], old: vec![] }} +} + +impl TrieDB { + pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } + + pub fn new_boxed(db_box: Box) -> Self { TrieDB{ db: db_box, root: H256::new() } } + + pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } } + + pub fn init(&mut self) { self.set_root_rlp(&NULL_RLP); } + + pub fn db(&self) -> &HashDB { self.db.as_ref() } + + fn set_root_rlp(&mut self, root_data: &[u8]) { + self.db.kill(&self.root); + self.root = self.db.insert(root_data); + println!("set_root_rlp {:?} {:?}", root_data, self.root); + } + + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { + // determine what the new root is, insert new nodes and remove old as necessary. + let todo = { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.merge(root_rlp, key, value) + }; + self.apply(todo.1); + self.set_root_rlp(&todo.0); + } + + fn apply(&mut self, diff: Diff) { + for d in diff.old.iter() { + self.db.kill(&d); + } + for d in diff.new.into_iter() { + self.db.emplace(d.0, d.1); + } + } + + /// Determine the RLP of the node, assuming we're inserting `partial_key` into the + /// node at `old`. This will *not* delete the old mode; it will just return the new RLP + /// that includes the new node. + /// + /// The database will be updated so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { + let o = Rlp::new(old); + match o.prototype() { + Prototype::List(17) => { + // already have a branch. route and merge. + unimplemented!(); + }, + Prototype::List(2) => { + let their_key_rlp = o.at(0); + let (them, _) = NibbleSlice::from_encoded(their_key_rlp.data()); + match partial_key.common_prefix(&them) { + 0 => { + // transmute to branch here + }, + cp if cp == them.len() => { + // fast-forward + }, + _ => { + // cleve into two + branch in the middle + }, + } + // already have an extension. either fast_forward, cleve or transmute_to_branch. + unimplemented!(); + }, + Prototype::Data(0) => { + (Self::compose_extension(partial_key, value, true), Diff::new()) + }, + _ => panic!("Invalid RLP for node."), + } + } + + fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { + println!("compose_extension {:?} {:?} {:?} ({:?})", partial_key, value, is_leaf, partial_key.encoded(is_leaf)); + let mut s = RlpStream::new_list(2); + s.append(&partial_key.encoded(is_leaf)); + s.append(&value.to_vec()); // WTF?!?! + //s.append(value); // <-- should be. + let r = s.out(); + println!("output: -> {:?}", &r); + r + } +} + +impl Trie for TrieDB { + fn root(&self) -> &H256 { &self.root } + + fn contains(&self, _key: &[u8]) -> bool { + unimplemented!(); + } + + fn at(&self, _key: &[u8]) -> Option<&[u8]> { + unimplemented!(); + } + + fn insert(&mut self, key: &[u8], value: &[u8]) { + (self as &mut TrieDB).add(&NibbleSlice::new(key), value); + } + + fn remove(&mut self, _key: &[u8]) { + unimplemented!(); + } +} + +#[test] +fn playpen() { + use overlaydb::*; + use triehash::*; + + (&[1, 2, 3]).starts_with(&[1, 2]); + + let mut t = TrieDB::new(OverlayDB::new_temp()); + t.init(); + assert_eq!(*t.root(), SHA3_NULL_RLP); + assert!(t.is_empty()); + + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); +} diff --git a/src/triehash.rs b/src/triehash.rs new file mode 100644 index 000000000..bc8dedf76 --- /dev/null +++ b/src/triehash.rs @@ -0,0 +1,354 @@ +//! Generete trie root. +//! +//! This module should be used to generate trie root hash. + +use std::collections::BTreeMap; +use std::cmp; +use hash::*; +use sha3::*; +use rlp; +use rlp::RlpStream; +use vector::SharedPrefix; + +// todo: verify if example for ordered_trie_root is valid +/// Generates a trie root hash for a vector of values +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::triehash::*; +/// use util::hash::*; +/// +/// fn main() { +/// let v = vec![From::from("doe"), From::from("reindeer")]; +/// let root = "e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3"; +/// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap()); +/// } +/// ``` +pub fn ordered_trie_root(input: Vec>) -> H256 { + let gen_input = input + // first put elements into btree to sort them by nibbles + // optimize it later + .into_iter() + .fold(BTreeMap::new(), | mut acc, vec | { + let len = acc.len(); + acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc + }) + // then move them to a vector + .into_iter() + .map(|p| p ) + .collect(); + + gen_trie_root(gen_input) +} + +/// Generates a trie root hash for a vector of key-values +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::triehash::*; +/// use util::hash::*; +/// +/// fn main() { +/// let v = vec![ +/// (From::from("doe"), From::from("reindeer")), +/// (From::from("dog"), From::from("puppy")), +/// (From::from("dogglesworth"), From::from("cat")), +/// ]; +/// +/// let root = "8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"; +/// assert_eq!(trie_root(v), H256::from_str(root).unwrap()); +/// } +/// ``` +pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let gen_input = input + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v)) + .collect(); + + gen_trie_root(gen_input) +} + +fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let mut stream = RlpStream::new(); + hash256rlp(&input, 0, &mut stream); + stream.out().sha3() +} + +/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. +/// +/// The "termination marker" and "leaf-node" specifier are completely equivalent. +/// +/// Input values are in range `[0, 0xf]`. +/// +/// ```markdown +/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +/// [1,2,3,4,5] 0x112345 // 5 > 3 +/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +/// [0,1,2,3,4] 0x101234 // 5 > 3 +/// [1,2,3,4] 0x001234 // 4 > 3 +/// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 +/// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 +/// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 +/// [1,2,3,4,5,T] 0x312345 // 5 > 3 +/// [1,2,3,4,T] 0x201234 // 4 > 3 +/// ``` +fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { + let inlen = nibbles.len(); + let oddness_factor = inlen % 2; + // next even number divided by two + let reslen = (inlen + 2) >> 1; + let mut res = vec![]; + res.reserve(reslen); + + let first_byte = { + let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; + if oddness_factor == 1 { + bits += nibbles[0]; + } + bits + }; + + res.push(first_byte); + + let mut offset = oddness_factor; + while offset < inlen { + let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; + res.push(byte); + offset += 2; + } + + res +} + +/// Converts slice of bytes to nibbles. +fn as_nibbles(bytes: &[u8]) -> Vec { + let mut res = vec![]; + res.reserve(bytes.len() * 2); + for i in 0..bytes.len() { + res.push(bytes[i] >> 4); + res.push((bytes[i] << 4) >> 4); + } + res +} + +fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { + let inlen = input.len(); + + // in case of empty slice, just append null + if inlen == 0 { + stream.append(&""); + return; + } + + // take slices + let key: &Vec = &input[0].0; + let value: &[u8] = &input[0].1; + + // if the slice contains just one item, append the suffix of the key + // and then append value + if inlen == 1 { + stream.append_list(2); + stream.append(&hex_prefix_encode(&key[pre_len..], true)); + stream.append(&value); + return; + } + + // get length of the longest shared prefix in slice keys + let shared_prefix = input.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(key.len(), | acc, &(ref k, _) | { + cmp::min(key.shared_prefix_len(&k), acc) + }); + + // if shared prefix is higher than current prefix append its + // new part of the key to the stream + // then recursively append suffixes of all items who had this key + if shared_prefix > pre_len { + stream.append_list(2); + stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix], false)); + hash256aux(input, shared_prefix, stream); + return; + } + + // an item for every possible nibble/suffix + // + 1 for data + stream.append_list(17); + + // if first key len is equal to prefix_len, move to next element + let mut begin = match pre_len == key.len() { + true => 1, + false => 0 + }; + + // iterate over all possible nibbles + for i in 0..16 { + // cout how many successive elements have same next nibble + let len = input[begin..].iter() + .map(| pair | pair.0[pre_len] ) + .take_while(|&q| q == i).count(); + + // if at least 1 successive element has the same nibble + // append their suffixes + match len { + 0 => { stream.append(&""); }, + _ => hash256aux(&input[begin..(begin + len)], pre_len + 1, stream) + } + begin += len; + } + + // if fist key len is equal prefix, append it's value + match pre_len == key.len() { + true => { stream.append(&value); }, + false => { stream.append(&""); } + }; +} + +fn hash256aux(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { + let mut s = RlpStream::new(); + hash256rlp(input, pre_len, &mut s); + let out = s.out(); + match out.len() { + 0...31 => stream.append_raw(&out, 1), + _ => stream.append(&out.sha3()) + }; +} + + +#[test] +fn test_nibbles() { + let v = vec![0x31, 0x23, 0x45]; + let e = vec![3, 1, 2, 3, 4, 5]; + assert_eq!(as_nibbles(&v), e); + + // A => 65 => 0x41 => [4, 1] + let v: Vec = From::from("A"); + let e = vec![4, 1]; + assert_eq!(as_nibbles(&v), e); +} + +#[test] +fn test_hex_prefix_encode() { + let v = vec![0, 0, 1, 2, 3, 4, 5]; + let e = vec![0x10, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x00, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x20, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4, 5]; + let e = vec![0x31, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4]; + let e = vec![0x00, 0x12, 0x34]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![4, 1]; + let e = vec![0x20, 0x41]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use rustc_serialize::hex::FromHex; + use hash::*; + use triehash::*; + + #[test] + fn empty_trie_root() { + assert_eq!(trie_root(vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); + } + + #[test] + fn single_trie_item() { + let v = vec![(From::from("A"), From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))]; + assert_eq!(trie_root(v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); + } + + #[test] + fn foo_trie_item() { + + let v = vec![ + (From::from("foo"), From::from("bar")), + (From::from("food"), From::from("bass")) + ]; + + assert_eq!(trie_root(v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); + } + + #[test] + fn dogs_trie_item() { + + let v = vec![ + (From::from("doe"), From::from("reindeer")), + (From::from("dog"), From::from("puppy")), + (From::from("dogglesworth"), From::from("cat")), + ]; + + assert_eq!(trie_root(v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); + } + + #[test] + fn puppy_trie_items() { + + let v = vec![ + (From::from("do"), From::from("verb")), + (From::from("dog"), From::from("puppy")), + (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), + ]; + + assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); + } + + #[test] + fn test_trie_root() { + let v = vec![ + + ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + + ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + + ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + + ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + + ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) + + ]; + + assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); + } +} diff --git a/src/vector.rs b/src/vector.rs index ce891beba..552e285cf 100644 --- a/src/vector.rs +++ b/src/vector.rs @@ -1,3 +1,4 @@ +//! vector util functions use std::ptr; @@ -32,3 +33,53 @@ impl InsertSlice for Vec { } } } + +/// Returns len of prefix shared with elem +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::vector::SharedPrefix; +/// +/// fn main () { +/// let a = vec![1,2,3,3,5]; +/// let b = vec![1,2,3]; +/// assert_eq!(a.shared_prefix_len(&b), 3); +/// } +/// ``` +pub trait SharedPrefix { + fn shared_prefix_len(&self, elem: &[T]) -> usize; +} + +impl SharedPrefix for Vec where T: Eq { + fn shared_prefix_len(&self, elem: &[T]) -> usize { + use std::cmp; + let len = cmp::min(self.len(), elem.len()); + (0..len).take_while(|&i| self[i] == elem[i]).count() + } +} + +#[cfg(test)] +mod test { + use vector::SharedPrefix; + + #[test] + fn test_shared_prefix() { + let a = vec![1,2,3,4,5,6]; + let b = vec![4,2,3,4,5,6]; + assert_eq!(a.shared_prefix_len(&b), 0); + } + + #[test] + fn test_shared_prefix2() { + let a = vec![1,2,3,3,5]; + let b = vec![1,2,3]; + assert_eq!(a.shared_prefix_len(&b), 3); + } + + #[test] + fn test_shared_prefix3() { + let a = vec![1,2,3,4,5,6]; + let b = vec![1,2,3,4,5,6]; + assert_eq!(a.shared_prefix_len(&b), 6); + } +}