Merge branch 'master' of github.com:gavofyork/ethcore-util into network

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests"]
+	path = tests
+	url = https://github.com/ethereum/tests

src/bytes.rs

@@ -40,6 +40,41 @@ use std::error::Error as StdError;
 use uint::{U128, U256};
 use hash::FixedHash;
 
+pub struct PrettySlice<'a> (&'a [u8]);
+
+impl<'a> fmt::Debug for PrettySlice<'a> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		for i in 0..self.0.len() {
+			match i > 0 {
+				true => { try!(write!(f, "·{:02x}", self.0[i])); },
+				false => { try!(write!(f, "{:02x}", self.0[i])); },
+			}
+		}
+		Ok(())
+	}
+}
+
+pub trait ToPretty {
+	fn pretty(&self) -> PrettySlice;
+}
+
+impl<'a> ToPretty for &'a [u8] {
+	fn pretty(&self) -> PrettySlice {
+		PrettySlice(self)
+	}
+}
+
+impl<'a> ToPretty for &'a Bytes {
+	fn pretty(&self) -> PrettySlice {
+		PrettySlice(self.bytes())
+	}
+}
+impl ToPretty for Bytes {
+	fn pretty(&self) -> PrettySlice {
+		PrettySlice(self.bytes())
+	}
+}
+
 /// Vector of bytes
 pub type Bytes = Vec<u8>;
 

src/nibbleslice.rs

@@ -1,5 +1,6 @@
 //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
 use std::cmp::*;
+use std::fmt;
 use bytes::*;
 
 /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
@@ -25,7 +26,7 @@ use bytes::*;
 ///   assert_eq!(n2.mid(3).common_prefix(&n1), 3);
 /// }
 /// ```
-#[derive(Debug, Copy, Clone, Eq, Ord)]
+#[derive(Copy, Clone, Eq, Ord)]
 pub struct NibbleSlice<'a> {
 	data: &'a [u8],
 	offset: usize,
@@ -122,6 +123,18 @@ impl<'a> PartialOrd for NibbleSlice<'a> {
 	}
 }
 
+impl<'a> fmt::Debug for NibbleSlice<'a> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		for i in 0..self.len() {
+			match i {
+				0 => try!(write!(f, "{:01x}", self.at(i))),
+				_ => try!(write!(f, "'{:01x}", self.at(i))),
+			}
+		}
+		Ok(())
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::NibbleSlice;

src/rlp.rs

@@ -65,16 +65,16 @@ impl OffsetCache {
 	}
 }
 
-/// stores basic information about item
+/// Stores basic information about item
-struct ItemInfo {
+pub struct PayloadInfo {
-	prefix_len: usize,
+	pub header_len: usize,
-	value_len: usize,
+	pub value_len: usize,
 }
 
-impl ItemInfo {
+impl PayloadInfo {
-	fn new(prefix_len: usize, value_len: usize) -> ItemInfo {
+	fn new(header_len: usize, value_len: usize) -> PayloadInfo {
-		ItemInfo {
+		PayloadInfo {
-			prefix_len: prefix_len,
+			header_len: header_len,
 			value_len: value_len,
 		}
 	}
@@ -86,7 +86,6 @@ pub enum DecoderError {
 	RlpIsTooShort,
 	RlpExpectedToBeList,
 	RlpExpectedToBeData,
-	BadRlp,
 }
 impl StdError for DecoderError {
 	fn description(&self) -> &str {
@@ -170,6 +169,10 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view {
 		self.rlp.raw()
 	}
 
+	pub fn payload_info(&self) -> PayloadInfo {
+		self.rlp.payload_info().unwrap()
+	}
+
 	pub fn data(&'view self) -> &'a [u8] {
 		self.rlp.data()
 	}
@@ -354,9 +357,13 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
 		self.bytes
 	}
 
+	pub fn payload_info(&self) -> Result<PayloadInfo, DecoderError> {
+		BasicDecoder::payload_info(self.bytes)
+	}
+
 	pub fn data(&'view self) -> &'a [u8] {
-		let ii = Self::item_info(self.bytes).unwrap();
+		let ii = BasicDecoder::payload_info(self.bytes).unwrap();
-		&self.bytes[ii.prefix_len..(ii.prefix_len + ii.value_len)]
+		&self.bytes[ii.header_len..(ii.header_len + ii.value_len)]
 	}
 
 	/// Returns number of rlp items.
@@ -396,7 +403,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
 	/// ```
 	pub fn size(&self) -> usize {
 		match self.is_data() {
-			true => Self::item_info(self.bytes).unwrap().value_len,
+			true => BasicDecoder::payload_info(self.bytes).unwrap().value_len,
 			false => 0
 		}
 	}
@@ -437,8 +444,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
 		self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len()));
 
 		// construct new rlp
-		let found = try!(UntrustedRlp::item_info(bytes));
+		let found = try!(BasicDecoder::payload_info(bytes));
-		Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len]))
+		Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len]))
 	}
 
 	/// No value
@@ -552,8 +559,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
 
 	/// consumes first found prefix
 	fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> {
-		let item = try!(UntrustedRlp::item_info(self.bytes));
+		let item = try!(BasicDecoder::payload_info(self.bytes));
-		let bytes = try!(UntrustedRlp::consume(self.bytes, item.prefix_len));
+		let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len));
 		Ok(bytes)
 	}
 
@@ -561,41 +568,12 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view {
 	fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> {
 		let mut result = bytes;
 		for _ in 0..items {
-			let i = try!(UntrustedRlp::item_info(result));
+			let i = try!(BasicDecoder::payload_info(result));
-			result = try!(UntrustedRlp::consume(result, (i.prefix_len + i.value_len)));
+			result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len)));
 		}
 		Ok(result)
 	}
 
-	/// return first item info
-	///
-	/// TODO: move this to decoder (?)
-	fn item_info(bytes: &[u8]) -> Result<ItemInfo, DecoderError> {
-		let item = match bytes.first().map(|&x| x) {
-			None => return Err(DecoderError::RlpIsTooShort),
-			Some(0...0x7f) => ItemInfo::new(0, 1),
-			Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80),
-			Some(l @ 0xb8...0xbf) => {
-				let len_of_len = l as usize - 0xb7;
-				let prefix_len = 1 + len_of_len;
-				let value_len = try!(usize::from_bytes(&bytes[1..prefix_len]));
-				ItemInfo::new(prefix_len, value_len)
-			}
-			Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0),
-			Some(l @ 0xf8...0xff) => {
-				let len_of_len = l as usize - 0xf7;
-				let prefix_len = 1 + len_of_len;
-				let value_len = try!(usize::from_bytes(&bytes[1..prefix_len]));
-				ItemInfo::new(prefix_len, value_len)
-			}
-			_ => return Err(DecoderError::BadRlp),
-		};
-
-		match item.prefix_len + item.value_len <= bytes.len() {
-			true => Ok(item),
-			false => Err(DecoderError::RlpIsTooShort),
-		}
-	}
 
 	/// consumes slice prefix of length `len`
 	fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> {
@@ -727,6 +705,37 @@ pub trait Decoder {
 
 pub struct BasicDecoder;
 
+impl BasicDecoder {
+	/// Return first item info
+	fn payload_info(bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
+		let item = match bytes.first().map(|&x| x) {
+			None => return Err(DecoderError::RlpIsTooShort),
+			Some(0...0x7f) => PayloadInfo::new(0, 1),
+			Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80),
+			Some(l @ 0xb8...0xbf) => {
+				let len_of_len = l as usize - 0xb7;
+				let header_len = 1 + len_of_len;
+				let value_len = try!(usize::from_bytes(&bytes[1..header_len]));
+				PayloadInfo::new(header_len, value_len)
+			}
+			Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0),
+			Some(l @ 0xf8...0xff) => {
+				let len_of_len = l as usize - 0xf7;
+				let header_len = 1 + len_of_len;
+				let value_len = try!(usize::from_bytes(&bytes[1..header_len]));
+				PayloadInfo::new(header_len, value_len)
+			},
+			// we cant reach this place, but rust requires _ to be implemented
+			_ => { panic!(); }
+		};
+
+		match item.header_len + item.value_len <= bytes.len() {
+			true => Ok(item),
+			false => Err(DecoderError::RlpIsTooShort),
+		}
+	}
+}
+
 impl Decoder for BasicDecoder {
 	fn read_value<T, F>(bytes: &[u8], f: F) -> Result<T, DecoderError> where F: FnOnce(&[u8]) -> Result<T, DecoderError> {
 		match bytes.first().map(|&x| x) {
@@ -743,7 +752,8 @@ impl Decoder for BasicDecoder {
 				let len = try!(usize::from_bytes(&bytes[1..begin_of_value]));
 				Ok(try!(f(&bytes[begin_of_value..begin_of_value + len])))
 			}
-			_ => Err(DecoderError::BadRlp),
+			// we are reading value, not a list!
+			_ => { panic!(); }
 		}
 	}
 }
@@ -800,7 +810,7 @@ impl RlpStream {
 	/// 	assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']);
 	/// }
 	/// ```
-	pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug {
+	pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable {
 		// encode given value and add it at the end of the stream
 		object.encode(&mut self.encoder);
 

src/trie.rs

@@ -1,3 +1,6 @@
+extern crate rand;
+
+use std::fmt;
 use memorydb::*;
 use sha3::*;
 use hashdb::*;
@@ -5,6 +8,7 @@ use hash::*;
 use nibbleslice::*;
 use bytes::*;
 use rlp::*;
+
 //use log::*;
 
 pub const NULL_RLP: [u8; 1] = [0x80; 1];
@@ -16,11 +20,90 @@ pub trait Trie {
 
 	// TODO: consider returning &[u8]...
 	fn contains(&self, key: &[u8]) -> bool;
-	fn at(&self, key: &[u8]) -> Option<&[u8]>;
+	fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key;
 	fn insert(&mut self, key: &[u8], value: &[u8]);
 	fn remove(&mut self, key: &[u8]);
 }
 
+#[derive(Eq, PartialEq, Debug)]
+pub enum Node<'a> {
+	NullRoot,
+	Leaf(NibbleSlice<'a>, &'a[u8]),
+	Extension(NibbleSlice<'a>, &'a[u8]),
+	Branch([Option<&'a[u8]>; 16], Option<&'a [u8]>)
+}
+
+impl <'a>Node<'a> {
+	pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> {
+		let r = Rlp::new(node_rlp);
+		match r.prototype() {
+			// either leaf or extension - decode first item with NibbleSlice::??? 
+			// and use is_leaf return to figure out which.
+			// if leaf, second item is a value (is_data())
+			// if extension, second item is a node (either SHA3 to be looked up and 
+			// fed back into this function or inline RLP which can be fed back into this function).
+			Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
+				(slice, true) => Node::Leaf(slice, r.at(1).data()),
+				(slice, false) => Node::Extension(slice, r.at(1).raw()),
+			},
+			// branch - first 16 are nodes, 17th is a value (or empty).
+			Prototype::List(17) => {
+				let mut nodes: [Option<&'a [u8]>; 16] = unsafe { ::std::mem::uninitialized() };
+				for i in 0..16 {
+					nodes[i] = if r.at(i).is_empty() { None } else { Some(r.at(i).raw()) }
+				}
+				Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
+			},
+			// an empty branch index.
+			Prototype::Data(0) => Node::NullRoot,
+			// something went wrong.
+			_ => panic!("Rlp is not valid.")
+		}
+	}
+
+	// todo: should check length before encoding, cause it may just be sha3 of data
+	pub fn encoded(&self) -> Bytes {
+		match *self {
+			Node::Leaf(ref slice, ref value) => {
+				let mut stream = RlpStream::new_list(2);
+				stream.append(&slice.encoded(true));
+				stream.append(value);
+				stream.out()
+			},
+			Node::Extension(ref slice, ref raw_rlp) => {
+				let mut stream = RlpStream::new_list(2);
+				stream.append(&slice.encoded(false));
+				stream.append_raw(raw_rlp, 1);
+				stream.out()
+			},
+			Node::Branch(ref nodes, ref value) => {
+				let mut stream = RlpStream::new_list(17);
+				for i in 0..16 { 
+					match nodes[i] {
+						Some(n) => { stream.append_raw(n, 1); },
+						None => { stream.append_empty_data(); },
+					}
+				}
+				match *value {
+					Some(n) => { stream.append(&n); },
+					None => { stream.append_empty_data(); },
+				}
+				stream.out()
+			},
+			Node::NullRoot => {
+				let mut stream = RlpStream::new();
+				stream.append_empty_data();
+				stream.out()
+			}
+		}
+	}
+}
+
+//enum ValidationResult<'a> {
+	//Valid,
+	//Invalid { node: Node<'a>, depth: usize }
+//}
+
 enum Operation {
 	New(H256, Bytes),
 	Delete(H256),
@@ -35,11 +118,13 @@ impl Diff {
 	/// such that the reference is valid, once applied.
 	fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) {
 		if rlp.len() >= 32 {
+			trace!("new_node: reference node {:?}", rlp.pretty());
 			let rlp_sha3 = rlp.sha3();
 			out.append(&rlp_sha3);
 			self.0.push(Operation::New(rlp_sha3, rlp));
 		}
 		else {
+			trace!("new_node: inline node {:?}", rlp.pretty());
 			out.append_raw(&rlp, 1);
 		}
 	}
@@ -66,24 +151,32 @@ pub struct TrieDB {
 	root: H256,
 }
 
+impl fmt::Debug for TrieDB {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		try!(writeln!(f, "["));
+		let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!");
+		try!(self.fmt_all(root_rlp, f, 0));
+		writeln!(f, "]")
+	}
+}
+
 impl TrieDB {
-	pub fn new<T>(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } }
+	pub fn new_boxed(db_box: Box<HashDB>) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r }
 
-	pub fn new_boxed(db_box: Box<HashDB>) -> Self { TrieDB{ db: db_box, root: H256::new() } }
+	pub fn new<T>(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) }
 
-	pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } }
+	pub fn new_memory() -> Self { Self::new(MemoryDB::new()) }
-
-	pub fn init(&mut self) { self.set_root_rlp(&NULL_RLP); }
 
 	pub fn db(&self) -> &HashDB { self.db.as_ref() }
 
 	fn set_root_rlp(&mut self, root_data: &[u8]) {
 		self.db.kill(&self.root);
 		self.root = self.db.insert(root_data);
-		trace!("set_root_rlp {:?} {:?}", root_data, self.root);
+		trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root);
 	}
 
 	fn apply(&mut self, diff: Diff) {
+		trace!("applying {:?} changes", diff.0.len());
 		for d in diff.0.into_iter() {
 			match d {
 				Operation::Delete(h) => {
@@ -91,60 +184,154 @@ impl TrieDB {
 					self.db.kill(&h);
 				},
 				Operation::New(h, d) => {
-					trace!("TrieDB::apply +++ {:?} -> {:?}", &h, &d);
+					trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty());
 					self.db.emplace(h, d);
 				}
 			}
 		}
 	}
 
+	fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
+		for _ in 0..size { 
+			try!(write!(f, "  "));
+		}
+		Ok(())
+	}
+
+	fn fmt_all(&self, node: &[u8], f: &mut fmt::Formatter, deepness: usize) -> fmt::Result {
+		let node = Node::decoded(node);
+		match node {
+			Node::Leaf(slice, value) => try!(writeln!(f, "-{:?}: {:?}.", slice, value.pretty())),
+			Node::Extension(ref slice, ref item) => {
+				try!(write!(f, "-{:?}- ", slice));
+				try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness));
+			},
+			Node::Branch(ref nodes, ref value) => {
+				try!(writeln!(f, ""));
+				match value {
+					&Some(v) => {
+						try!(self.fmt_indent(f, deepness + 1));
+						try!(writeln!(f, "=: {:?}", v.pretty()))
+					},
+					&None => {}
+				}
+				for i in 0..16 {
+					match nodes[i] {
+						Some(n) => {
+							try!(self.fmt_indent(f, deepness + 1));
+							try!(write!(f, "{:x}: ", i));
+							try!(self.fmt_all(self.get_raw_or_lookup(n), f, deepness + 1));
+						},
+						None => {},
+					}
+				}
+			},
+			// empty
+			Node::NullRoot => {
+				try!(writeln!(f, "<empty>"));
+			}
+		};
+		Ok(())
+	}
+
+	fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key {
+		let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!");
+		self.get_from_node(&root_rlp, key)
+	}
+
+	fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key {
+		match Node::decoded(node) {
+			Node::Leaf(ref slice, ref value) if key == slice => Some(value),
+			Node::Extension(ref slice, ref item) if key.starts_with(slice) => {
+				self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len()))
+			},
+			Node::Branch(ref nodes, value) => match key.is_empty() {
+				true => value,
+				false => match nodes[key.at(0) as usize] {
+					Some(payload) => {
+						self.get_from_node(self.get_raw_or_lookup(payload), &key.mid(1))
+					},
+					None => None
+				}
+			},
+			_ => None
+		}
+	}
+
+	fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] {
+		// check if its sha3 + len
+		let r = Rlp::new(node);
+		match r.is_data() && r.size() == 32 {
+			true => self.db.lookup(&H256::decode(&r)).expect("Not found!"),
+			false => node
+		}
+	}
+
 	fn add(&mut self, key: &NibbleSlice, value: &[u8]) {
+		trace!("ADD: {:?} {:?}", key, value.pretty());
 		// determine what the new root is, insert new nodes and remove old as necessary.
 		let mut todo: Diff = Diff::new();
 		let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo);
 		self.apply(todo);
 		self.set_root_rlp(&root_rlp);
+		trace!("/");
 	}
 
 	fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes {
-		trace!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true));
+		trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty());
 		let mut s = RlpStream::new_list(2);
 		s.append(&partial.encoded(true));
 		s.append(&value);
 		let r = s.out();
-		trace!("output: -> {:?}", &r);
+		trace!("compose_leaf: -> {:?}", r.pretty());
 		r
 	}
 
 	fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes {
-		println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload, is_leaf, partial.encoded(is_leaf));
+		println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf));
 		let mut s = RlpStream::new_list(2);
 		s.append(&partial.encoded(is_leaf));
 		s.append_raw(raw_payload, 1);
 		let r = s.out();
-		println!("output: -> {:?}", &r);
+		println!("compose_raw: -> {:?}", r.pretty());
 		r
 	}
 
+	fn compose_stub_branch(value: &[u8]) -> Bytes {
+		let mut s = RlpStream::new_list(17);
+		for _ in 0..16 { s.append_empty_data(); }
+		s.append(&value);
+		s.out()
+	}
+
 	fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes {
 		Self::compose_raw(partial, raw_payload, false)
 	}
 
+	fn create_extension(partial: &NibbleSlice, downstream_node: Bytes, diff: &mut Diff) -> Bytes {
+		trace!("create_extension partial: {:?}, downstream_node: {:?}", partial, downstream_node.pretty());
+		let mut s = RlpStream::new_list(2);
+		s.append(&partial.encoded(false));
+		diff.new_node(downstream_node, &mut s);
+		s.out()
+	}
+
 	/// Return the bytes encoding the node represented by `rlp`. It will be unlinked from
 	/// the trie.
 	fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view {
 		if rlp.is_list() {
-			trace!("take_node {:?} (inline)", rlp.raw());
+			trace!("take_node {:?} (inline)", rlp.raw().pretty());
 			rlp.raw()
 		}
 		else if rlp.is_data() && rlp.size() == 32 {
 			let h = H256::decode(rlp);
 			let r = self.db.lookup(&h).expect("Trie root not found!");
-			trace!("take_node {:?} (indirect for {:?})", rlp.raw(), r);
+			trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r);
 			diff.delete_node_sha3(h);
 			r
 		}
 		else {
+			trace!("take_node {:?} (???)", rlp.raw().pretty());
 			panic!("Empty or invalid node given?");
 		}
 	}
@@ -211,22 +398,16 @@ impl TrieDB {
 		let mut s = RlpStream::new_list(17);
 		let index = if partial.is_empty() {16} else {partial.at(0) as usize};
 		for i in 0usize..17 {
-			if index == i {
+			match (index == i, i) {
-				// this is node to augment into...
+				(true, 16) => // leaf entry - just replace.
-				if orig.at(i).is_empty() {
+					{ s.append(&value); },
-					// easy - original had empty slot.
+				(true, i) if orig.at(i).is_empty() => // easy - original had empty slot.
-					diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s);
+					diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s),
-				} else if i == 16 {
+				(true, i) => {	// harder - original has something there already
-					// leaf entry - just replace.
+					let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff);
-					let new = Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value);
-					diff.replace_node(&orig.at(i), new, &mut s); 
-				} else {
-					// harder - original has something there already
-					let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff);
 					diff.replace_node(&orig.at(i), new, &mut s);
 				}
-			} else {
+				(false, i) => { s.append_raw(orig.at(i).raw(), 1); },
-				s.append_raw(orig.at(i).raw(), 1);
 			}
 		}
 		s.out()
@@ -241,40 +422,42 @@ impl TrieDB {
 	///
 	/// **This operation will not insert the new node now destroy the original.**
 	fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes {
-		trace!("augmented ({:?}, {:?}, {:?})", old, partial, value);
+		trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty());
 		// already have an extension. either fast_forward, cleve or transmute_to_branch.
 		let old_rlp = Rlp::new(old);
 		match old_rlp.prototype() {
 			Prototype::List(17) => {
+				trace!("branch: ROUTE,AUGMENT");
 				// already have a branch. route and augment.
 				self.augmented_into_branch(&old_rlp, partial, value, diff)
 			},
 			Prototype::List(2) => {
 				let existing_key_rlp = old_rlp.at(0);
 				let (existing_key, is_leaf) = NibbleSlice::from_encoded(existing_key_rlp.data());
-				match partial.common_prefix(&existing_key) {
+				match (is_leaf, partial.common_prefix(&existing_key)) {
-					cp if partial.len() == existing_key.len() && cp == existing_key.len() && is_leaf => {
+					(true, cp) if cp == existing_key.len() && partial.len() == existing_key.len() => {
 						// equivalent-leaf: replace
 						trace!("equivalent-leaf: REPLACE");
 						Self::compose_leaf(partial, value)
 					},
-					0 => {
+					(_, 0) => {
 						// one of us isn't empty: transmute to branch here
 						trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len());
 						self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff)
 					},
-					cp if cp == existing_key.len() => {
+					(_, cp) if cp == existing_key.len() => {
 						trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp);
 						// fully-shared prefix for this extension:
-						// skip to the end of this extension and continue to augment there.
+						// transform to an extension + augmented version of onward node.
-						let n = self.take_node(&old_rlp.at(1), diff);
+						let downstream_node: Bytes = if is_leaf {
-						let downstream_node = self.augmented(n, &partial.mid(cp), value, diff);
+							// no onward node because we're a leaf - create fake stub and use that.
-						let mut s = RlpStream::new_list(2);
+							self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff)
-						s.append_raw(old_rlp.at(0).raw(), 1);
+						} else {
-						diff.new_node(downstream_node, &mut s);
+							self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff)
-						s.out()
+						};
+						Self::create_extension(&existing_key, downstream_node, diff)
 					},
-					cp => {
+					(_, cp) => {
 						// partially-shared prefix for this extension:
 						// split into two extensions, high and low, pass the
 						// low through augment with the value before inserting the result
@@ -297,9 +480,10 @@ impl TrieDB {
 				}
 			},
 			Prototype::Data(0) => {
+				trace!("empty: COMPOSE");
 				Self::compose_leaf(partial, value)
 			},
-			_ => panic!("Invalid RLP for node."),
+			_ => panic!("Invalid RLP for node: {:?}", old.pretty()),
 		}
 	}
 }
@@ -307,12 +491,12 @@ impl TrieDB {
 impl Trie for TrieDB {
 	fn root(&self) -> &H256 { &self.root }
 
-	fn contains(&self, _key: &[u8]) -> bool {
+	fn contains(&self, key: &[u8]) -> bool {
-		unimplemented!();
+		self.at(key).is_some()
 	}
 
-	fn at(&self, _key: &[u8]) -> Option<&[u8]> {
+	fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
-		unimplemented!();
+		self.get(&NibbleSlice::new(key))
 	}
 
 	fn insert(&mut self, key: &[u8], value: &[u8]) {
@@ -326,42 +510,319 @@ impl Trie for TrieDB {
 
 #[cfg(test)]
 mod tests {
-	use memorydb::*;
+	use rustc_serialize::hex::FromHex;
 	use triehash::*;
+	use hash::*;
 	use super::*;
+	use nibbleslice::*;
+	use rlp;
 	use env_logger;
+	use rand::random;
+
+	#[test]
+	fn test_node_leaf() {
+		let k = vec![0x20u8, 0x01, 0x23, 0x45];
+		let v: Vec<u8> = From::from("cat");
+		let (slice, is_leaf) = NibbleSlice::from_encoded(&k);
+		assert_eq!(is_leaf, true);
+		let leaf = Node::Leaf(slice, &v);
+		let rlp = leaf.encoded();
+		let leaf2 = Node::decoded(&rlp);
+		assert_eq!(leaf, leaf2);
+	}
+
+	#[test]
+	fn test_node_extension() {
+		let k = vec![0x00u8, 0x01, 0x23, 0x45];
+		// in extension, value must be valid rlp
+		let v = rlp::encode(&"cat");
+		let (slice, is_leaf) = NibbleSlice::from_encoded(&k);
+		assert_eq!(is_leaf, false);
+		let ex = Node::Extension(slice, &v);
+		let rlp = ex.encoded();
+		let ex2 = Node::decoded(&rlp);
+		assert_eq!(ex, ex2);
+	}
+
+	#[test]
+	fn test_node_empty_branch() {
+		let branch = Node::Branch([None; 16], None);
+		let rlp = branch.encoded();
+		let branch2 = Node::decoded(&rlp);
+		assert_eq!(branch, branch2);
+	}
+
+	#[test]
+	fn test_node_branch() {
+		let k = rlp::encode(&"cat");
+		let mut nodes: [Option<&[u8]>; 16] = unsafe { ::std::mem::uninitialized() };
+		for i in 0..16 { nodes[i] = Some(&k); }
+		let v: Vec<u8> = From::from("dog");
+		let branch = Node::Branch(nodes, Some(&v));
+		let rlp = branch.encoded();
+		let branch2 = Node::decoded(&rlp);
+		assert_eq!(branch, branch2);
+	}
+
+	#[test]
+	fn test_at_empty() {
+		let t = TrieDB::new_memory();
+		assert_eq!(t.at(&[0x5]), None);
+	}
+
+	#[test]
+	fn test_at_one() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]);
+	}
+
+	#[test]
+	fn test_at_three() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]);
+		t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]);
+		assert_eq!(t.at(&[0x01, 0x23]).unwrap(), &[0x01u8, 0x23]);
+		assert_eq!(t.at(&[0xf1, 0x23]).unwrap(), &[0xf1u8, 0x23]);
+		assert_eq!(t.at(&[0x81, 0x23]).unwrap(), &[0x81u8, 0x23]);
+		assert_eq!(t.at(&[0x82, 0x23]), None);
+	}
+
+	#[test]
+	fn test_print_trie() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]);
+		t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]);
+		println!("trie:");
+		println!("{:?}", t);
+		//assert!(false);
+	}
+
+	fn test_all(v: Vec<(Vec<u8>, Vec<u8>)>) {
+		let mut t = TrieDB::new_memory();
+		
+		for i in 0..v.len() {
+			let key: &[u8]= &v[i].0;
+			let val: &[u8] = &v[i].1;
+			t.insert(&key, &val);
+		}
+
+// 		trace!("{:?}", t);
+//		println!("{:?}", t);
+
+		// check lifetime
+//		let _q = t.at(&[b'd', b'o']).unwrap();
+
+		assert_eq!(*t.root(), trie_root(v));
+	}
+
+	fn random_key() -> Vec<u8> {
+		let chars = b"abcdefgrstuvwABCDEFGRSTUVW";
+		let mut ret: Vec<u8> = Vec::new();
+		let r = random::<u8>() % 4 + 1;
+		for _ in 0..r {
+			ret.push(chars[random::<usize>() % chars.len()]);
+		}
+		ret
+	}
+
+	#[test]
+	fn stress() {
+		for _ in 0..1000 {
+			let mut x: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
+			for j in 0..100u32 {
+				let key = random_key();
+				x.push((key, rlp::encode(&j)));
+			}
+			let real = trie_root(x.clone());
+
+			let mem = trie_root_mem(&x);
+			assert_eq!(mem, real);
+		}
+	}
+
+	fn trie_root_mem(v: &Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
+		let mut t = TrieDB::new_memory();
+		
+		for i in 0..v.len() {
+			let key: &[u8]= &v[i].0;
+			let val: &[u8] = &v[i].1;
+			t.insert(&key, &val);
+		}
+
+		t.root().clone()
+	}
+
+	#[test]
+	fn test_at_dog() {
+		env_logger::init().ok();
+		let v = vec![
+			(From::from("do"), From::from("verb")),
+			(From::from("dog"), From::from("puppy")),
+			(From::from("doge"), From::from("coin")),
+			(From::from("horse"), From::from("stallion")),
+		];
+
+		test_all(v);
+	}
+
+	#[test]
+	fn test_more_data() {
+		let v = vec![
+		
+			("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), 
+			 "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()),
+
+			("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(),
+			 "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()),
+
+			("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(),
+			 "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()),
+
+			("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(),
+			 "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()),
+
+			("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(),
+			 "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()),
+
+			("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(),
+			 "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()),
+
+			("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(),
+			 "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()),
+
+			("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(),
+			 "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap())
+
+		];
+
+		test_all(v);
+	}
 
 	#[test]
 	fn playpen() {
-		env_logger::init().unwrap();
+		env_logger::init().ok();
 
-		let mut t = TrieDB::new(MemoryDB::new());
+		let big_value = b"00000000000000000000000000000000";
-		t.init();
+
-		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], big_value);
+		t.insert(&[0x11u8, 0x23], big_value);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], big_value.to_vec()),
+			(vec![0x11u8, 0x23], big_value.to_vec())
+		]));
 	}
 
 	#[test]
 	fn init() {
-		let mut t = TrieDB::new(MemoryDB::new());
+		let t = TrieDB::new_memory();
-		t.init();
 		assert_eq!(*t.root(), SHA3_NULL_RLP);
 		assert!(t.is_empty());
 	}
 
 	#[test]
 	fn insert_on_empty() {
-		let mut t = TrieDB::new(MemoryDB::new());
+		let mut t = TrieDB::new_memory();
-		t.init();
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
 		assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ]));
 	}
 
 	#[test]
 	fn insert_replace_root() {
-		let mut t = TrieDB::new(MemoryDB::new());
+		let mut t = TrieDB::new_memory();
-		t.init();
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
 		t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]);
 		assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ]));
 	}
+
+	#[test]
+	fn insert_make_branch_root() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+			(vec![0x11u8, 0x23], vec![0x11u8, 0x23])
+		]));
 	}
+
+	#[test]
+	fn insert_into_branch_root() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]);
+		t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+			(vec![0x81u8, 0x23], vec![0x81u8, 0x23]),
+			(vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]),
+		]));
+	}
+
+	#[test]
+	fn insert_value_into_branch_root() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[], &[0x0]);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![], vec![0x0]),
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+		]));
+	}
+
+	#[test]
+	fn insert_split_leaf() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+		t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+			(vec![0x01u8, 0x34], vec![0x01u8, 0x34]),
+		]));
+	}
+
+	#[test]
+	fn insert_split_extenstion() {
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01, 0x23, 0x45], &[0x01]);
+		t.insert(&[0x01, 0xf3, 0x45], &[0x02]);
+		t.insert(&[0x01, 0xf3, 0xf5], &[0x03]);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01, 0x23, 0x45], vec![0x01]),
+			(vec![0x01, 0xf3, 0x45], vec![0x02]),
+			(vec![0x01, 0xf3, 0xf5], vec![0x03]),
+		]));
+	}
+
+	#[test]
+	fn insert_big_value() {
+		let big_value0 = b"00000000000000000000000000000000";
+		let big_value1 = b"11111111111111111111111111111111";
+
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], big_value0);
+		t.insert(&[0x11u8, 0x23], big_value1);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], big_value0.to_vec()),
+			(vec![0x11u8, 0x23], big_value1.to_vec())
+		]));
+	}
+
+	#[test]
+	fn insert_duplicate_value() {
+		let big_value = b"00000000000000000000000000000000";
+
+		let mut t = TrieDB::new_memory();
+		t.insert(&[0x01u8, 0x23], big_value);
+		t.insert(&[0x11u8, 0x23], big_value);
+		assert_eq!(*t.root(), trie_root(vec![
+			(vec![0x01u8, 0x23], big_value.to_vec()),
+			(vec![0x11u8, 0x23], big_value.to_vec())
+		]));
+	}
+}
+

src/triehash.rs

@@ -32,12 +32,12 @@ pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
 		.into_iter()
 		.fold(BTreeMap::new(), | mut acc, vec | {
 			let len = acc.len();
-			acc.insert(as_nibbles(&rlp::encode(&len)), vec);
+			acc.insert(rlp::encode(&len), vec);
 			acc
 		})
 		// then move them to a vector
 		.into_iter()
-		.map(|p| p )
+		.map(|(k, v)| (as_nibbles(&k), v) )
 		.collect();
 
 	gen_trie_root(gen_input)
@@ -64,6 +64,13 @@ pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
 /// ```
 pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
 	let gen_input = input
+		// first put elements into btree to sort them and to remove duplicates
+		.into_iter()
+		.fold(BTreeMap::new(), | mut acc, (k, v) | {
+			acc.insert(k, v);
+			acc
+		})
+		// then move them to a vector
 		.into_iter()
 		.map(|(k, v)| (as_nibbles(&k), v) )
 		.collect();
@@ -189,9 +196,12 @@ fn hash256rlp(input: &[(Vec<u8>, Vec<u8>)], pre_len: usize, stream: &mut RlpStre
 	// iterate over all possible nibbles
 	for i in 0..16 {
 		// cout how many successive elements have same next nibble
-		let len = input[begin..].iter()
+		let len = match begin < input.len() {
-			.map(| pair | pair.0[pre_len] )
+			true => input[begin..].iter()
-			.take_while(|&q| q == i).count();
+				.take_while(| pair | pair.0[pre_len] == i )
+				.count(), 
+			false => 0
+		};
 			
 		// if at least 1 successive element has the same nibble
 		// append their suffixes
@@ -268,7 +278,10 @@ fn test_hex_prefix_encode() {
 #[cfg(test)]
 mod tests {
 	use std::str::FromStr;
+	use std::collections::BTreeMap;
 	use rustc_serialize::hex::FromHex;
+	use rustc_serialize::json::Json;
+	use bytes::*;
 	use hash::*;
 	use triehash::*;
 
@@ -319,6 +332,20 @@ mod tests {
 		assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap());
 	}
 
+	#[test]
+	fn out_of_order() {
+		assert!(trie_root(vec![
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+			(vec![0x81u8, 0x23], vec![0x81u8, 0x23]),
+			(vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]),
+		]) ==
+		trie_root(vec![
+			(vec![0x01u8, 0x23], vec![0x01u8, 0x23]),
+			(vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]),
+			(vec![0x81u8, 0x23], vec![0x81u8, 0x23]),
+		]));
+	}
+
 	#[test]
 	fn test_trie_root() {
 		let v = vec![
@@ -351,4 +378,19 @@ mod tests {
 
 		assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap());
 	}
+
+	#[test]
+	fn test_triehash_json_trietest_json() {
+		let data = include_bytes!("../tests/TrieTests/trietest.json");
+
+		let s = String::from_bytes(data).unwrap();
+		let json = Json::from_str(&s).unwrap();
+		let obj = json.as_object().unwrap();
+
+		for (key, value) in obj.iter() {
+			println!("running test: {}", key);
+		}
+		assert!(false);
+	}
+
 }

tests

@@ -0,0 +1 @@
+Subproject commit 2e4987ad2a973e2cf85ef742a8b9bd094363cd18