diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..2571e36c6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests"] + path = tests + url = https://github.com/ethereum/tests diff --git a/src/bytes.rs b/src/bytes.rs index e17ccca9a..e81cd8572 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -40,6 +40,41 @@ use std::error::Error as StdError; use uint::{U128, U256}; use hash::FixedHash; +pub struct PrettySlice<'a> (&'a [u8]); + +impl<'a> fmt::Debug for PrettySlice<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.0.len() { + match i > 0 { + true => { try!(write!(f, "ยท{:02x}", self.0[i])); }, + false => { try!(write!(f, "{:02x}", self.0[i])); }, + } + } + Ok(()) + } +} + +pub trait ToPretty { + fn pretty(&self) -> PrettySlice; +} + +impl<'a> ToPretty for &'a [u8] { + fn pretty(&self) -> PrettySlice { + PrettySlice(self) + } +} + +impl<'a> ToPretty for &'a Bytes { + fn pretty(&self) -> PrettySlice { + PrettySlice(self.bytes()) + } +} +impl ToPretty for Bytes { + fn pretty(&self) -> PrettySlice { + PrettySlice(self.bytes()) + } +} + /// Vector of bytes pub type Bytes = Vec; diff --git a/src/memorydb.rs b/src/memorydb.rs index 14fae12d0..85fdc7c19 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -189,4 +189,4 @@ fn memorydb_denote() { } assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); -} \ No newline at end of file +} diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 7569046ad..5f2615991 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -1,5 +1,6 @@ //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. use std::cmp::*; +use std::fmt; use bytes::*; /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. @@ -25,7 +26,7 @@ use bytes::*; /// assert_eq!(n2.mid(3).common_prefix(&n1), 3); /// } /// ``` -#[derive(Debug, Copy, Clone, Eq, Ord)] +#[derive(Copy, Clone, Eq, Ord)] pub struct NibbleSlice<'a> { data: &'a [u8], offset: usize, @@ -122,6 +123,18 @@ impl<'a> PartialOrd for NibbleSlice<'a> { } } +impl<'a> fmt::Debug for NibbleSlice<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.len() { + match i { + 0 => try!(write!(f, "{:01x}", self.at(i))), + _ => try!(write!(f, "'{:01x}", self.at(i))), + } + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::NibbleSlice; diff --git a/src/rlp.rs b/src/rlp.rs index cb3fa378c..802147500 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -65,16 +65,16 @@ impl OffsetCache { } } -/// stores basic information about item -struct ItemInfo { - prefix_len: usize, - value_len: usize, +/// Stores basic information about item +pub struct PayloadInfo { + pub header_len: usize, + pub value_len: usize, } -impl ItemInfo { - fn new(prefix_len: usize, value_len: usize) -> ItemInfo { - ItemInfo { - prefix_len: prefix_len, +impl PayloadInfo { + fn new(header_len: usize, value_len: usize) -> PayloadInfo { + PayloadInfo { + header_len: header_len, value_len: value_len, } } @@ -86,7 +86,6 @@ pub enum DecoderError { RlpIsTooShort, RlpExpectedToBeList, RlpExpectedToBeData, - BadRlp, } impl StdError for DecoderError { fn description(&self) -> &str { @@ -170,6 +169,10 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.raw() } + pub fn payload_info(&self) -> PayloadInfo { + self.rlp.payload_info().unwrap() + } + pub fn data(&'view self) -> &'a [u8] { self.rlp.data() } @@ -354,9 +357,13 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { self.bytes } + pub fn payload_info(&self) -> Result { + BasicDecoder::payload_info(self.bytes) + } + pub fn data(&'view self) -> &'a [u8] { - let ii = Self::item_info(self.bytes).unwrap(); - &self.bytes[ii.prefix_len..(ii.prefix_len + ii.value_len)] + let ii = BasicDecoder::payload_info(self.bytes).unwrap(); + &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] } /// Returns number of rlp items. @@ -396,7 +403,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// ``` pub fn size(&self) -> usize { match self.is_data() { - true => Self::item_info(self.bytes).unwrap().value_len, + true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, false => 0 } } @@ -437,8 +444,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); // construct new rlp - let found = try!(UntrustedRlp::item_info(bytes)); - Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) + let found = try!(BasicDecoder::payload_info(bytes)); + Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) } /// No value @@ -552,8 +559,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// consumes first found prefix fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(UntrustedRlp::item_info(self.bytes)); - let bytes = try!(UntrustedRlp::consume(self.bytes, item.prefix_len)); + let item = try!(BasicDecoder::payload_info(self.bytes)); + let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); Ok(bytes) } @@ -561,41 +568,12 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { let mut result = bytes; for _ in 0..items { - let i = try!(UntrustedRlp::item_info(result)); - result = try!(UntrustedRlp::consume(result, (i.prefix_len + i.value_len))); + let i = try!(BasicDecoder::payload_info(result)); + result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); } Ok(result) } - /// return first item info - /// - /// TODO: move this to decoder (?) - fn item_info(bytes: &[u8]) -> Result { - let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => ItemInfo::new(0, 1), - Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) - } - Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) - } - _ => return Err(DecoderError::BadRlp), - }; - - match item.prefix_len + item.value_len <= bytes.len() { - true => Ok(item), - false => Err(DecoderError::RlpIsTooShort), - } - } /// consumes slice prefix of length `len` fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { @@ -727,6 +705,37 @@ pub trait Decoder { pub struct BasicDecoder; +impl BasicDecoder { + /// Return first item info + fn payload_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + }, + // we cant reach this place, but rust requires _ to be implemented + _ => { panic!(); } + }; + + match item.header_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + impl Decoder for BasicDecoder { fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { match bytes.first().map(|&x| x) { @@ -743,7 +752,8 @@ impl Decoder for BasicDecoder { let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } - _ => Err(DecoderError::BadRlp), + // we are reading value, not a list! + _ => { panic!(); } } } } @@ -800,7 +810,7 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { // encode given value and add it at the end of the stream object.encode(&mut self.encoder); diff --git a/src/trie.rs b/src/trie.rs index 9f0b209be..2fb05ccfb 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,3 +1,6 @@ +extern crate rand; + +use std::fmt; use memorydb::*; use sha3::*; use hashdb::*; @@ -5,6 +8,7 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; + //use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; @@ -16,11 +20,90 @@ pub trait Trie { // TODO: consider returning &[u8]... fn contains(&self, key: &[u8]) -> bool; - fn at(&self, key: &[u8]) -> Option<&[u8]>; + fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; fn insert(&mut self, key: &[u8], value: &[u8]); fn remove(&mut self, key: &[u8]); } +#[derive(Eq, PartialEq, Debug)] +pub enum Node<'a> { + NullRoot, + Leaf(NibbleSlice<'a>, &'a[u8]), + Extension(NibbleSlice<'a>, &'a[u8]), + Branch([Option<&'a[u8]>; 16], Option<&'a [u8]>) +} + +impl <'a>Node<'a> { + pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> { + let r = Rlp::new(node_rlp); + match r.prototype() { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { + (slice, true) => Node::Leaf(slice, r.at(1).data()), + (slice, false) => Node::Extension(slice, r.at(1).raw()), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes: [Option<&'a [u8]>; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { + nodes[i] = if r.at(i).is_empty() { None } else { Some(r.at(i).raw()) } + } + Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) + }, + // an empty branch index. + Prototype::Data(0) => Node::NullRoot, + // something went wrong. + _ => panic!("Rlp is not valid.") + } + } + + // todo: should check length before encoding, cause it may just be sha3 of data + pub fn encoded(&self) -> Bytes { + match *self { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + match nodes[i] { + Some(n) => { stream.append_raw(n, 1); }, + None => { stream.append_empty_data(); }, + } + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::NullRoot => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } +} + +//enum ValidationResult<'a> { + //Valid, + //Invalid { node: Node<'a>, depth: usize } +//} + enum Operation { New(H256, Bytes), Delete(H256), @@ -35,11 +118,13 @@ impl Diff { /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { + trace!("new_node: reference node {:?}", rlp.pretty()); let rlp_sha3 = rlp.sha3(); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); } else { + trace!("new_node: inline node {:?}", rlp.pretty()); out.append_raw(&rlp, 1); } } @@ -66,24 +151,32 @@ pub struct TrieDB { root: H256, } +impl fmt::Debug for TrieDB { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "[")); + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + try!(self.fmt_all(root_rlp, f, 0)); + writeln!(f, "]") + } +} + impl TrieDB { - pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } + pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } - pub fn new_boxed(db_box: Box) -> Self { TrieDB{ db: db_box, root: H256::new() } } + pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } - pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } } - - pub fn init(&mut self) { self.set_root_rlp(&NULL_RLP); } + pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } pub fn db(&self) -> &HashDB { self.db.as_ref() } fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); - trace!("set_root_rlp {:?} {:?}", root_data, self.root); + trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } fn apply(&mut self, diff: Diff) { + trace!("applying {:?} changes", diff.0.len()); for d in diff.0.into_iter() { match d { Operation::Delete(h) => { @@ -91,60 +184,154 @@ impl TrieDB { self.db.kill(&h); }, Operation::New(h, d) => { - trace!("TrieDB::apply +++ {:?} -> {:?}", &h, &d); + trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty()); self.db.emplace(h, d); } } } } + fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { + for _ in 0..size { + try!(write!(f, " ")); + } + Ok(()) + } + + fn fmt_all(&self, node: &[u8], f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { + let node = Node::decoded(node); + match node { + Node::Leaf(slice, value) => try!(writeln!(f, "-{:?}: {:?}.", slice, value.pretty())), + Node::Extension(ref slice, ref item) => { + try!(write!(f, "-{:?}- ", slice)); + try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness)); + }, + Node::Branch(ref nodes, ref value) => { + try!(writeln!(f, "")); + match value { + &Some(v) => { + try!(self.fmt_indent(f, deepness + 1)); + try!(writeln!(f, "=: {:?}", v.pretty())) + }, + &None => {} + } + for i in 0..16 { + match nodes[i] { + Some(n) => { + try!(self.fmt_indent(f, deepness + 1)); + try!(write!(f, "{:x}: ", i)); + try!(self.fmt_all(self.get_raw_or_lookup(n), f, deepness + 1)); + }, + None => {}, + } + } + }, + // empty + Node::NullRoot => { + try!(writeln!(f, "")); + } + }; + Ok(()) + } + + fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.get_from_node(&root_rlp, key) + } + + fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + match Node::decoded(node) { + Node::Leaf(ref slice, ref value) if key == slice => Some(value), + Node::Extension(ref slice, ref item) if key.starts_with(slice) => { + self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) + }, + Node::Branch(ref nodes, value) => match key.is_empty() { + true => value, + false => match nodes[key.at(0) as usize] { + Some(payload) => { + self.get_from_node(self.get_raw_or_lookup(payload), &key.mid(1)) + }, + None => None + } + }, + _ => None + } + } + + fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { + // check if its sha3 + len + let r = Rlp::new(node); + match r.is_data() && r.size() == 32 { + true => self.db.lookup(&H256::decode(&r)).expect("Not found!"), + false => node + } + } + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { + trace!("ADD: {:?} {:?}", key, value.pretty()); // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); + trace!("/"); } fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { - trace!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty()); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(true)); s.append(&value); let r = s.out(); - trace!("output: -> {:?}", &r); + trace!("compose_leaf: -> {:?}", r.pretty()); r } fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { - println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload, is_leaf, partial.encoded(is_leaf)); + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); let r = s.out(); - println!("output: -> {:?}", &r); + println!("compose_raw: -> {:?}", r.pretty()); r } + fn compose_stub_branch(value: &[u8]) -> Bytes { + let mut s = RlpStream::new_list(17); + for _ in 0..16 { s.append_empty_data(); } + s.append(&value); + s.out() + } + fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } + fn create_extension(partial: &NibbleSlice, downstream_node: Bytes, diff: &mut Diff) -> Bytes { + trace!("create_extension partial: {:?}, downstream_node: {:?}", partial, downstream_node.pretty()); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(false)); + diff.new_node(downstream_node, &mut s); + s.out() + } + /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { - trace!("take_node {:?} (inline)", rlp.raw()); + trace!("take_node {:?} (inline)", rlp.raw().pretty()); rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!"); - trace!("take_node {:?} (indirect for {:?})", rlp.raw(), r); + trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); diff.delete_node_sha3(h); r } else { + trace!("take_node {:?} (???)", rlp.raw().pretty()); panic!("Empty or invalid node given?"); } } @@ -211,22 +398,16 @@ impl TrieDB { let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; for i in 0usize..17 { - if index == i { - // this is node to augment into... - if orig.at(i).is_empty() { - // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); - } else if i == 16 { - // leaf entry - just replace. - let new = Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value); - diff.replace_node(&orig.at(i), new, &mut s); - } else { - // harder - original has something there already - let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff); + match (index == i, i) { + (true, 16) => // leaf entry - just replace. + { s.append(&value); }, + (true, i) if orig.at(i).is_empty() => // easy - original had empty slot. + diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + (true, i) => { // harder - original has something there already + let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); diff.replace_node(&orig.at(i), new, &mut s); } - } else { - s.append_raw(orig.at(i).raw(), 1); + (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } } s.out() @@ -241,40 +422,42 @@ impl TrieDB { /// /// **This operation will not insert the new node now destroy the original.** fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("augmented ({:?}, {:?}, {:?})", old, partial, value); + trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { + trace!("branch: ROUTE,AUGMENT"); // already have a branch. route and augment. self.augmented_into_branch(&old_rlp, partial, value, diff) }, Prototype::List(2) => { let existing_key_rlp = old_rlp.at(0); let (existing_key, is_leaf) = NibbleSlice::from_encoded(existing_key_rlp.data()); - match partial.common_prefix(&existing_key) { - cp if partial.len() == existing_key.len() && cp == existing_key.len() && is_leaf => { + match (is_leaf, partial.common_prefix(&existing_key)) { + (true, cp) if cp == existing_key.len() && partial.len() == existing_key.len() => { // equivalent-leaf: replace trace!("equivalent-leaf: REPLACE"); Self::compose_leaf(partial, value) }, - 0 => { + (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, - cp if cp == existing_key.len() => { + (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: - // skip to the end of this extension and continue to augment there. - let n = self.take_node(&old_rlp.at(1), diff); - let downstream_node = self.augmented(n, &partial.mid(cp), value, diff); - let mut s = RlpStream::new_list(2); - s.append_raw(old_rlp.at(0).raw(), 1); - diff.new_node(downstream_node, &mut s); - s.out() + // transform to an extension + augmented version of onward node. + let downstream_node: Bytes = if is_leaf { + // no onward node because we're a leaf - create fake stub and use that. + self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff) + } else { + self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff) + }; + Self::create_extension(&existing_key, downstream_node, diff) }, - cp => { + (_, cp) => { // partially-shared prefix for this extension: // split into two extensions, high and low, pass the // low through augment with the value before inserting the result @@ -297,9 +480,10 @@ impl TrieDB { } }, Prototype::Data(0) => { + trace!("empty: COMPOSE"); Self::compose_leaf(partial, value) }, - _ => panic!("Invalid RLP for node."), + _ => panic!("Invalid RLP for node: {:?}", old.pretty()), } } } @@ -307,12 +491,12 @@ impl TrieDB { impl Trie for TrieDB { fn root(&self) -> &H256 { &self.root } - fn contains(&self, _key: &[u8]) -> bool { - unimplemented!(); + fn contains(&self, key: &[u8]) -> bool { + self.at(key).is_some() } - fn at(&self, _key: &[u8]) -> Option<&[u8]> { - unimplemented!(); + fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.get(&NibbleSlice::new(key)) } fn insert(&mut self, key: &[u8], value: &[u8]) { @@ -326,42 +510,319 @@ impl Trie for TrieDB { #[cfg(test)] mod tests { - use memorydb::*; + use rustc_serialize::hex::FromHex; use triehash::*; + use hash::*; use super::*; + use nibbleslice::*; + use rlp; use env_logger; + use rand::random; + + #[test] + fn test_node_leaf() { + let k = vec![0x20u8, 0x01, 0x23, 0x45]; + let v: Vec = From::from("cat"); + let (slice, is_leaf) = NibbleSlice::from_encoded(&k); + assert_eq!(is_leaf, true); + let leaf = Node::Leaf(slice, &v); + let rlp = leaf.encoded(); + let leaf2 = Node::decoded(&rlp); + assert_eq!(leaf, leaf2); + } + + #[test] + fn test_node_extension() { + let k = vec![0x00u8, 0x01, 0x23, 0x45]; + // in extension, value must be valid rlp + let v = rlp::encode(&"cat"); + let (slice, is_leaf) = NibbleSlice::from_encoded(&k); + assert_eq!(is_leaf, false); + let ex = Node::Extension(slice, &v); + let rlp = ex.encoded(); + let ex2 = Node::decoded(&rlp); + assert_eq!(ex, ex2); + } + + #[test] + fn test_node_empty_branch() { + let branch = Node::Branch([None; 16], None); + let rlp = branch.encoded(); + let branch2 = Node::decoded(&rlp); + assert_eq!(branch, branch2); + } + + #[test] + fn test_node_branch() { + let k = rlp::encode(&"cat"); + let mut nodes: [Option<&[u8]>; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { nodes[i] = Some(&k); } + let v: Vec = From::from("dog"); + let branch = Node::Branch(nodes, Some(&v)); + let rlp = branch.encoded(); + let branch2 = Node::decoded(&rlp); + assert_eq!(branch, branch2); + } + + #[test] + fn test_at_empty() { + let t = TrieDB::new_memory(); + assert_eq!(t.at(&[0x5]), None); + } + + #[test] + fn test_at_one() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); + } + + #[test] + fn test_at_three() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + assert_eq!(t.at(&[0x01, 0x23]).unwrap(), &[0x01u8, 0x23]); + assert_eq!(t.at(&[0xf1, 0x23]).unwrap(), &[0xf1u8, 0x23]); + assert_eq!(t.at(&[0x81, 0x23]).unwrap(), &[0x81u8, 0x23]); + assert_eq!(t.at(&[0x82, 0x23]), None); + } + + #[test] + fn test_print_trie() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + println!("trie:"); + println!("{:?}", t); + //assert!(false); + } + + fn test_all(v: Vec<(Vec, Vec)>) { + let mut t = TrieDB::new_memory(); + + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + +// trace!("{:?}", t); +// println!("{:?}", t); + + // check lifetime +// let _q = t.at(&[b'd', b'o']).unwrap(); + + assert_eq!(*t.root(), trie_root(v)); + } + + fn random_key() -> Vec { + let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + let mut ret: Vec = Vec::new(); + let r = random::() % 4 + 1; + for _ in 0..r { + ret.push(chars[random::() % chars.len()]); + } + ret + } + + #[test] + fn stress() { + for _ in 0..1000 { + let mut x: Vec<(Vec, Vec)> = Vec::new(); + for j in 0..100u32 { + let key = random_key(); + x.push((key, rlp::encode(&j))); + } + let real = trie_root(x.clone()); + + let mem = trie_root_mem(&x); + assert_eq!(mem, real); + } + } + + fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> H256 { + let mut t = TrieDB::new_memory(); + + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + + t.root().clone() + } + + #[test] + fn test_at_dog() { + env_logger::init().ok(); + let v = vec![ + (From::from("do"), From::from("verb")), + (From::from("dog"), From::from("puppy")), + (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), + ]; + + test_all(v); + } + + #[test] + fn test_more_data() { + let v = vec![ + + ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + + ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + + ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + + ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + + ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) + + ]; + + test_all(v); + } #[test] fn playpen() { - env_logger::init().unwrap(); + env_logger::init().ok(); - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + let big_value = b"00000000000000000000000000000000"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value); + t.insert(&[0x11u8, 0x23], big_value); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ])); } #[test] fn init() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let t = TrieDB::new_memory(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } #[test] fn insert_on_empty() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } #[test] fn insert_replace_root() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); } -} \ No newline at end of file + + #[test] + fn insert_make_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ])); + } + + #[test] + fn insert_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ])); + } + + #[test] + fn insert_value_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[], &[0x0]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![], vec![0x0]), + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + ])); + } + + #[test] + fn insert_split_leaf() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), + ])); + } + + #[test] + fn insert_split_extenstion() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01, 0x23, 0x45], &[0x01]); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ])); + } + + #[test] + fn insert_big_value() { + let big_value0 = b"00000000000000000000000000000000"; + let big_value1 = b"11111111111111111111111111111111"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value0); + t.insert(&[0x11u8, 0x23], big_value1); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value0.to_vec()), + (vec![0x11u8, 0x23], big_value1.to_vec()) + ])); + } + + #[test] + fn insert_duplicate_value() { + let big_value = b"00000000000000000000000000000000"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value); + t.insert(&[0x11u8, 0x23], big_value); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ])); + } +} + diff --git a/src/triehash.rs b/src/triehash.rs index e26cc255f..778dc7f17 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -32,12 +32,12 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { .into_iter() .fold(BTreeMap::new(), | mut acc, vec | { let len = acc.len(); - acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc.insert(rlp::encode(&len), vec); acc }) // then move them to a vector .into_iter() - .map(|p| p ) + .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); gen_trie_root(gen_input) @@ -64,8 +64,15 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { /// ``` pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { let gen_input = input + // first put elements into btree to sort them and to remove duplicates .into_iter() - .map(|(k, v)| (as_nibbles(&k), v)) + .fold(BTreeMap::new(), | mut acc, (k, v) | { + acc.insert(k, v); + acc + }) + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); gen_trie_root(gen_input) @@ -189,9 +196,12 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // iterate over all possible nibbles for i in 0..16 { // cout how many successive elements have same next nibble - let len = input[begin..].iter() - .map(| pair | pair.0[pre_len] ) - .take_while(|&q| q == i).count(); + let len = match begin < input.len() { + true => input[begin..].iter() + .take_while(| pair | pair.0[pre_len] == i ) + .count(), + false => 0 + }; // if at least 1 successive element has the same nibble // append their suffixes @@ -268,7 +278,10 @@ fn test_hex_prefix_encode() { #[cfg(test)] mod tests { use std::str::FromStr; + use std::collections::BTreeMap; use rustc_serialize::hex::FromHex; + use rustc_serialize::json::Json; + use bytes::*; use hash::*; use triehash::*; @@ -319,6 +332,20 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); } + #[test] + fn out_of_order() { + assert!(trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]) == + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + ])); + } + #[test] fn test_trie_root() { let v = vec![ @@ -351,4 +378,19 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); } + + #[test] + fn test_triehash_json_trietest_json() { + let data = include_bytes!("../tests/TrieTests/trietest.json"); + + let s = String::from_bytes(data).unwrap(); + let json = Json::from_str(&s).unwrap(); + let obj = json.as_object().unwrap(); + + for (key, value) in obj.iter() { + println!("running test: {}", key); + } + assert!(false); + } + } diff --git a/tests b/tests new file mode 160000 index 000000000..2e4987ad2 --- /dev/null +++ b/tests @@ -0,0 +1 @@ +Subproject commit 2e4987ad2a973e2cf85ef742a8b9bd094363cd18