From 2b09521b5632478039cae28311c220b48d6570cb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:12:06 +0100 Subject: [PATCH 1/7] Main logic for insertion into trie. --- src/nibbleslice.rs | 12 +++ src/rlp.rs | 6 +- src/trie.rs | 252 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 229 insertions(+), 41 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 6ccf695c0..9656306e5 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -87,6 +87,18 @@ impl<'a> NibbleSlice<'a> { } r } + + pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> Bytes { + let l = min(self.len(), n); + let mut r = Bytes::with_capacity(l / 2 + 1); + let mut i = l % 2; + r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0}); + while i < l { + r.push(self.at(i) * 16 + self.at(i + 1)); + i += 2; + } + r + } } impl<'a> PartialEq for NibbleSlice<'a> { diff --git a/src/rlp.rs b/src/rlp.rs index e8562828a..aca12281f 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -167,8 +167,8 @@ impl<'a> Rlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { - self.rlp.data() + pub fn raw(&self) -> &[u8] { + self.rlp.raw() } /// Returns number of rlp items. @@ -348,7 +348,7 @@ impl<'a> UntrustedRlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { + pub fn raw(&self) -> &[u8] { self.bytes } diff --git a/src/trie.rs b/src/trie.rs index 4aebc688f..9d004c9dd 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -4,6 +4,7 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; +use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); @@ -29,13 +30,44 @@ pub struct TrieDB { root: H256, } -struct Diff { - new: Vec<(H256, Bytes)>, - old: Vec, +enum Operation { + New(H256, Bytes), + Delete(H256), } +struct Diff (Vec) + impl Diff { - pub fn new() -> Diff { Diff { new: vec![], old: vec![] }} + fn new() -> Diff { Diff(vec![]) } + + /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` + /// such that the reference is valid, once applied. + fn new_node(&mut self, Bytes rlp, out: &mut RlpStream) { + if (rlp.len() >= 32) { + let rlp_sha3 = rlp.sha3(); + out.append(&rlp_sha3); + self.operations.push(Operation::New(rlp_sha3, rlp)); + } + else { + out.append_raw(&rlp); + } + } + + /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. + fn delete_node_sha3(&mut self, old_sha3: H256) { + self.operations.push(Operation::Delete(old_sha3)); + } + + fn delete_node(&mut self, old: &Rlp) { + if (old.is_data() && old.size() == 32) { + self.operations.push(Operation::Delete(H256::decode(old))); + } + } + + fn replace_node(&mut self, old: &Rlp, Bytes rlp, out: &mut RlpStream) { + self.delete_node(old); + self.new_node(rlp, &mut out); + } } impl TrieDB { @@ -57,70 +89,214 @@ impl TrieDB { fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let todo = { - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - self.merge(root_rlp, key, value) - }; - self.apply(todo.1); - self.set_root_rlp(&todo.0); + let todo: Diff = Diff::new(); + let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); + self.apply(todo); + self.set_root_rlp(&root_rlp); } fn apply(&mut self, diff: Diff) { - for d in diff.old.iter() { - self.db.kill(&d); - } - for d in diff.new.into_iter() { - self.db.emplace(d.0, d.1); + for d in diff.operations.into_iter() { + match d { + Operation::Delete(h) => { + trace!("TrieDB::apply --- {:?}", &h); + self.db.kill(&h); + }, + Operation::New(h, d) => { + trace!("TrieDB::apply +++ {:?} -> {:?}", &h, &d); + self.db.emplace(h, d); + } + } } } - /// Determine the RLP of the node, assuming we're inserting `partial_key` into the - /// node at `old`. This will *not* delete the old mode; it will just return the new RLP - /// that includes the new node. + /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from + /// the trie. + fn take_node(&self, rlp: &Rlp, &mut diff) -> Bytes { + if (rlp.is_data()) { + Bytes::decode(rlp) + } + else { + let h = H256::decode(rlp); + let r = self.db.lookup(&h).as_vec(); + diff.delete_node(h); + r + } + } + + fn inject_and_replace(&self, old: &[u8], old_sha3: H256, partial: &NibbleSlice, value: &[u8], diff: &mut Diff, out: &mut RlpStream) { + diff.new_node(self.inject(old, partial, value, diff), &mut out); + diff.delete_node(old, old_sha3); + } + + /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_to_branch_and_inject(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + let intermediate = match orig_is_leaf { + true => Self::transmute_leaf_to_branch(orig_partial, orig_raw_payload, &mut diff), + false => Self::transmute_extension_to_branch(orig_partial, orig_raw_payload, &mut diff), + }; + self.inject(&intermediate, partial, value, &mut diff) + // TODO: implement without having to make an intermediate representation. + } + + /// Transform an existing extension or leaf node to an invalid single-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RLPStream::new_list(17); + assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. + let index = orig_partial.at(0); + // orig is extension - orig_payload is a node itself. + for i in 0..17 { + if index == i { + if orig_partial.len() > 1 { + // still need an extension + diff.new_node(compose_extension(orig_partial.mid(1), orig_raw_payload), &mut s); + } else { + // was an extension of length 1 - just redirect the payload into here. + s.append_raw(orig_payload.raw()); + } + } else { + s.append_null_data(); + } + } + s.out() + } + + fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RLPStream::new_list(17); + let index = orig_partial.is_empty() ? 16 : orig_partial.at(0); + // orig is leaf - orig_payload is data representing the actual value. + for i in 0..17 { + if index == i { + // this is our node. + diff.new_node(compose_raw(orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), &mut s); + } else { + s.append_null_data(); + } + } + s.out() + } + + /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the + /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that + /// once applied the returned node is valid. + fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + RlpStream s; + let index = partial.is_empty() ? 16 : partial.at(0); + for i in 0..17 { + if index == i && { + // this is our node. + if (orig.at(i).is_empty()) { + // easy - original had empty slot. + diff.new_node(compose_leaf(partial.mid(if i == 16 {0} else {1}), value), &mut s); + } else if (i == 16) { + // leaf entry - just replace. + let new = compose_leaf(partial.mid(if i == 16 {0} else {1}), value); + diff.replace_node(orig.at(i).raw(), new, &mut s), + } else { + // harder - original has something there already + let new = self.inject(orig.at(i).raw(), partial.mid(1), value, &mut diff); + diff.replace_node(orig.at(i).raw(), new, &mut s) + } + } else { + s.append_raw(orig.at(i).raw()); + } + } + s + } + + /// Determine the RLP of the node, assuming we're inserting `partial` into the + /// node currently of data `old`. This will *not* delete any hash of `old` from the database; + /// it will just return the new RLP that includes the new node. /// /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. - fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { - let o = Rlp::new(old); - match o.prototype() { + /// + /// **This operation will not insert the new node now destroy the original.** + fn inject(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + // already have an extension. either fast_forward, cleve or transmute_to_branch. + let old_rlp = Rlp::new(old); + match old_rlp.prototype() { Prototype::List(17) => { - // already have a branch. route and merge. - unimplemented!(); + // already have a branch. route and inject. + self.injected_into_branch(old_rlp, partial, value, &mut diff) }, Prototype::List(2) => { - let their_key_rlp = o.at(0); - let (them, _) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial_key.common_prefix(&them) { + let their_key_rlp = old_rlp.at(0); + let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); + + match partial.common_prefix(&them) { + 0 if partial.is_empty() && them.is_empty() => { + // both empty: just replace. + compose_leaf(partial, value) + }, 0 => { - // transmute to branch here + // one of us isn't empty: transmute to branch here + transmute_to_branch_and_inject(is_leaf, them, old_rlp.at(1).raw()) }, cp if cp == them.len() => { - // fast-forward + // fully-shared prefix for this extension: + // skip to the end of this extension and continue the inject there. + let n = self.take_node(old_rlp.at(1).raw()); + let downstream_node = self.inject(&n, partial.mid(cp), value, &mut diff); + let mut s = RlpStream::new_list(2); + s.append_raw(old_rlp.at(0).raw()); + diff.new_node(downstream_node, &mut s); + s.out() }, - _ => { - // cleve into two + branch in the middle + cp => { + // partially-shared prefix for this extension: + // split into two extensions, high and low, pass the + // low through inject with the value before inserting the result + // into high to create the new. + + // TODO: optimise by doing this without creating injected_low. + + // low (farther from root) + let low = Self::compose_raw(them.mid(cp), old_rlp.at(1).raw(), is_leaf); + let injected_low = self.inject(&low, partial.mid(cp), value, &mut diff); + + // high (closer to root) + let mut s = RlpStream::new_list(2); + s.append(them.encoded_leftmost(cp, false)); + diff.new_node(injected_low, &mut s); + s.out() }, } - // already have an extension. either fast_forward, cleve or transmute_to_branch. - unimplemented!(); }, Prototype::Data(0) => { - (Self::compose_extension(partial_key, value, true), Diff::new()) + (Self::compose_leaf(partial, value, true), Diff::new()) }, _ => panic!("Invalid RLP for node."), } } - fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { - println!("compose_extension {:?} {:?} {:?} ({:?})", partial_key, value, is_leaf, partial_key.encoded(is_leaf)); + fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); - s.append(&partial_key.encoded(is_leaf)); - s.append(&value.to_vec()); // WTF?!?! - //s.append(value); // <-- should be. + s.append(&partial.encoded(is_leaf)); + s.append_raw(raw_payload); let r = s.out(); println!("output: -> {:?}", &r); r } + + fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { + println!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(true)); + s.append(value); + let r = s.out(); + println!("output: -> {:?}", &r); + r + } + + fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { + Self::compose_raw(partial, raw_payload, false) + } } impl Trie for TrieDB { From 1934cb3bdd4b02c9f975f5afa39ae9d53c54211f Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 01:35:32 +0100 Subject: [PATCH 2/7] updated tiny_keccak library --- Cargo.toml | 2 +- src/sha3.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 51e9f4470..de4dcd41b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ arrayvec = "0.3" mio = "0.4.4" rand = "0.3.12" time = "0.1.34" -tiny-keccak = "0.3" +tiny-keccak = "1.0" rocksdb = "0.2.1" num = "0.1" lazy_static = "0.1.*" diff --git a/src/sha3.rs b/src/sha3.rs index ee328913c..11d7ca274 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,5 +1,5 @@ use std::mem::uninitialized; -use tiny_keccak::keccak_256; +use tiny_keccak::Keccak; use bytes::BytesConvertable; use hash::{FixedHash, H256}; @@ -10,8 +10,10 @@ pub trait Hashable { impl Hashable for T where T: BytesConvertable { fn sha3(&self) -> H256 { unsafe { + let mut keccak = Keccak::new_keccak256(); + keccak.update(self.bytes()); let mut ret: H256 = uninitialized(); - keccak_256(self.bytes(), ret.mut_bytes()); + keccak.finalize(ret.mut_bytes()); ret } } From 26f29b2fcdbc25d489ced14478688399fdae9342 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:44:18 +0100 Subject: [PATCH 3/7] Logging. --- src/lib.rs | 1 + src/trie.rs | 51 +++++++++++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e2cb3a754..c52a1b2d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ extern crate num; extern crate log; #[macro_use] extern crate lazy_static; +extern crate env_logger; extern crate time; extern crate crypto as rcrypto; diff --git a/src/trie.rs b/src/trie.rs index b7f9bfa46..f5392f2f1 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -85,7 +85,7 @@ impl TrieDB { fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); - println!("set_root_rlp {:?} {:?}", root_data, self.root); + trace!("set_root_rlp {:?} {:?}", root_data, self.root); } fn apply(&mut self, diff: Diff) { @@ -102,15 +102,26 @@ impl TrieDB { } } } -/* + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let todo: Diff = Diff::new(); + let mut todo: Diff = Diff::new(); let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); } + fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { + trace!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(true)); + s.append(&value); + let r = s.out(); + trace!("output: -> {:?}", &r); + r + } + +/* fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); @@ -121,16 +132,6 @@ impl TrieDB { r } - fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { - println!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); - let mut s = RlpStream::new_list(2); - s.append(&partial.encoded(true)); - s.append(value); - let r = s.out(); - println!("output: -> {:?}", &r); - r - } - fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } @@ -232,6 +233,7 @@ impl TrieDB { diff.new_node(self.inject(old, partial, value, diff), &mut out); diff.delete_node(old, old_sha3); } +*/ /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; @@ -246,11 +248,13 @@ impl TrieDB { let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { + unimplemented!(); // already have a branch. route and inject. - self.injected_into_branch(old_rlp, partial, value, diff) +// self.injected_into_branch(old_rlp, partial, value, diff) }, Prototype::List(2) => { - let their_key_rlp = old_rlp.at(0); + unimplemented!(); +/* let their_key_rlp = old_rlp.at(0); let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); match partial.common_prefix(&them) { @@ -290,15 +294,14 @@ impl TrieDB { diff.new_node(injected_low, s); s.out() }, - } + }*/ }, Prototype::Data(0) => { - (Self::compose_leaf(partial, value, true), Diff::new()) + Self::compose_leaf(partial, value) }, _ => panic!("Invalid RLP for node."), } } - */ } impl Trie for TrieDB { @@ -313,8 +316,7 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - unimplemented!(); -// (self as &mut TrieDB).add(&NibbleSlice::new(key), value); + (self as &mut TrieDB).add(&NibbleSlice::new(key), value); } fn remove(&mut self, _key: &[u8]) { @@ -326,6 +328,9 @@ impl Trie for TrieDB { fn playpen() { use overlaydb::*; use triehash::*; + use env_logger; + + env_logger::init().unwrap(); (&[1, 2, 3]).starts_with(&[1, 2]); @@ -333,7 +338,9 @@ fn playpen() { t.init(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); -/* + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ]));*/ + assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); + + assert!(false); } From cceae8ecc2a465bc8572fa4730db04e268e3c8cc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:52:20 +0100 Subject: [PATCH 4/7] less commented out code in trie. --- src/trie.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index f5392f2f1..58354fcce 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -121,9 +121,8 @@ impl TrieDB { r } -/* - fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { - println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); + fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); @@ -138,18 +137,21 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. - fn take_node(&self, rlp: &Rlp, &mut diff) -> Bytes { - if (rlp.is_data()) { - Bytes::decode(rlp) + fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { + if (rlp.is_list()) { + rlp.raw().to_vec() } - else { + else if (rlp.is_data() && rlp.size() == 32) { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!").as_vec(); - diff.delete_node(h); + let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); + diff.delete_node_sha3(h); r } + else { + panic!("Empty or invalid node given?"); + } } - +/* /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** @@ -341,6 +343,4 @@ fn playpen() { t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); - - assert!(false); } From 24f9771716f7b17beaaff309b6e3bec4ecfc84ab Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 02:04:52 +0100 Subject: [PATCH 5/7] fixed take_node lifetimes --- src/trie.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 58354fcce..55cbeb6c5 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -137,13 +137,13 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. - fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { + fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if (rlp.is_list()) { - rlp.raw().to_vec() + rlp.raw() } else if (rlp.is_data() && rlp.size() == 32) { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); + let r = self.db.lookup(&h).expect("Trie root not found!"); diff.delete_node_sha3(h); r } From d6e5bbae78fab295229044424e58ca11ff18d20f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 02:23:53 +0100 Subject: [PATCH 6/7] Trie compiles. --- src/trie.rs | 156 ++++++++++++++++++++++++---------------------------- 1 file changed, 72 insertions(+), 84 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 58354fcce..9518cec30 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -5,16 +5,11 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; -use log::*; +//use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); -/*lazy_static! { - pub static ref NULL_RLP: Bytes = { let mut r = RlpStream::new(); r.append(&""); r.out().unwrap() }; - pub static ref SHA3_NULL_RLP: H256 = { use sha3::Hashable; NULL_RLP.sha3() }; -}*/ - pub trait Trie { fn root(&self) -> &H256; fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } @@ -39,7 +34,7 @@ impl Diff { /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { - if (rlp.len() >= 32) { + if rlp.len() >= 32 { let rlp_sha3 = rlp.sha3(); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); @@ -55,7 +50,7 @@ impl Diff { } fn delete_node(&mut self, old: &Rlp) { - if (old.is_data() && old.size() == 32) { + if old.is_data() && old.size() == 32 { self.0.push(Operation::Delete(H256::decode(old))); } } @@ -138,10 +133,10 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { - if (rlp.is_list()) { + if rlp.is_list() { rlp.raw().to_vec() } - else if (rlp.is_data() && rlp.size() == 32) { + else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); diff.delete_node_sha3(h); @@ -151,7 +146,46 @@ impl TrieDB { panic!("Empty or invalid node given?"); } } -/* + + /// Transform an existing extension or leaf node to an invalid single-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RlpStream::new_list(17); + assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. + let index = orig_partial.at(0); + // orig is extension - orig_raw_payload is a node itself. + for i in 0..17 { + if index == i { + if orig_partial.len() > 1 { + // still need an extension + diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s); + } else { + // was an extension of length 1 - just redirect the payload into here. + s.append_raw(orig_raw_payload, 1); + } + } else { + s.append_empty_data(); + } + } + s.out() + } + + fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RlpStream::new_list(17); + let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; + // orig is leaf - orig_raw_payload is data representing the actual value. + for i in 0..17 { + if index == i { + // this is our node. + diff.new_node(Self::compose_raw(&orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), &mut s); + } else { + s.append_empty_data(); + } + } + s.out() + } + /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** @@ -164,78 +198,35 @@ impl TrieDB { // TODO: implement without having to make an intermediate representation. } - /// Transform an existing extension or leaf node to an invalid single-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - let mut s = RLPStream::new_list(17); - assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let index = orig_partial.at(0); - // orig is extension - orig_payload is a node itself. - for i in 0..17 { - if index == i { - if orig_partial.len() > 1 { - // still need an extension - diff.new_node(compose_extension(orig_partial.mid(1), orig_raw_payload), s); - } else { - // was an extension of length 1 - just redirect the payload into here. - s.append_raw(orig_payload.raw(), 1); - } - } else { - s.append_null_data(); - } - } - s.out() - } - - fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - let mut s = RLPStream::new_list(17); - let index = orig_partial.is_empty() ? 16 : orig_partial.at(0); - // orig is leaf - orig_payload is data representing the actual value. - for i in 0..17 { - if index == i { - // this is our node. - diff.new_node(compose_raw(orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), s); - } else { - s.append_null_data(); - } - } - s.out() - } - /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that /// once applied the returned node is valid. fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - RlpStream s; - let index = partial.is_empty() ? 16 : partial.at(0); - for i in 0..17 { - if index == i && { - // this is our node. - if (orig.at(i).is_empty()) { + let mut s = RlpStream::new_list(17); + let index = if partial.is_empty() {16} else {partial.at(0) as usize}; + for i in 0usize..17 { + if index == i { + // this is node to inject into... + if orig.at(i).is_empty() { // easy - original had empty slot. - diff.new_node(compose_leaf(partial.mid(if i == 16 {0} else {1}), value), s); - } else if (i == 16) { + diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); + } else if i == 16 { // leaf entry - just replace. - let new = compose_leaf(partial.mid(if i == 16 {0} else {1}), value); - diff.replace_node(orig.at(i).raw(), new, s), + let new = Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value); + diff.replace_node(&orig.at(i), new, &mut s); } else { // harder - original has something there already - let new = self.inject(orig.at(i).raw(), partial.mid(1), value, diff); - diff.replace_node(orig.at(i).raw(), new, s) + let new = self.inject(orig.at(i).raw(), &partial.mid(1), value, diff); + diff.replace_node(&orig.at(i), new, &mut s); } } else { s.append_raw(orig.at(i).raw(), 1); } } - s + s.out() } - fn inject_and_replace(&self, old: &[u8], old_sha3: H256, partial: &NibbleSlice, value: &[u8], diff: &mut Diff, out: &mut RlpStream) { - diff.new_node(self.inject(old, partial, value, diff), &mut out); - diff.delete_node(old, old_sha3); - } -*/ + /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; @@ -250,32 +241,29 @@ impl TrieDB { let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { - unimplemented!(); // already have a branch. route and inject. -// self.injected_into_branch(old_rlp, partial, value, diff) + self.injected_into_branch(&old_rlp, partial, value, diff) }, Prototype::List(2) => { - unimplemented!(); -/* let their_key_rlp = old_rlp.at(0); + let their_key_rlp = old_rlp.at(0); let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial.common_prefix(&them) { 0 if partial.is_empty() && them.is_empty() => { // both empty: just replace. - compose_leaf(partial, value) + Self::compose_leaf(partial, value) }, 0 => { // one of us isn't empty: transmute to branch here - transmute_to_branch_and_inject(is_leaf, them, old_rlp.at(1).raw()) + self.transmute_to_branch_and_inject(is_leaf, &them, old_rlp.at(1).raw(), partial, value, diff) }, cp if cp == them.len() => { // fully-shared prefix for this extension: // skip to the end of this extension and continue the inject there. - let n = self.take_node(old_rlp.at(1).raw()); - let downstream_node = self.inject(&n, partial.mid(cp), value, diff); + let n = self.take_node(&old_rlp.at(1), diff); + let downstream_node = self.inject(&n, &partial.mid(cp), value, diff); let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); - diff.new_node(downstream_node, s); + diff.new_node(downstream_node, &mut s); s.out() }, cp => { @@ -287,16 +275,16 @@ impl TrieDB { // TODO: optimise by doing this without creating injected_low. // low (farther from root) - let low = Self::compose_raw(them.mid(cp), old_rlp.at(1).raw(), is_leaf); - let injected_low = self.inject(&low, partial.mid(cp), value, diff); + let low = Self::compose_raw(&them.mid(cp), old_rlp.at(1).raw(), is_leaf); + let injected_low = self.inject(&low, &partial.mid(cp), value, diff); // high (closer to root) let mut s = RlpStream::new_list(2); - s.append(them.encoded_leftmost(cp, false)); - diff.new_node(injected_low, s); + s.append(&them.encoded_leftmost(cp, false)); + diff.new_node(injected_low, &mut s); s.out() }, - }*/ + } }, Prototype::Data(0) => { Self::compose_leaf(partial, value) @@ -318,7 +306,7 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - (self as &mut TrieDB).add(&NibbleSlice::new(key), value); + self.add(&NibbleSlice::new(key), value); } fn remove(&mut self, _key: &[u8]) { From 4e4b754eca3cb782af291032e1b972c9e291ad6d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 03:35:55 +0100 Subject: [PATCH 7/7] First few tests working. --- src/rlp.rs | 9 ++-- src/trie.rs | 117 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 80 insertions(+), 46 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 4d792a15b..c023b1dfb 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -174,7 +174,7 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.data() } - /// Returns number of rlp items. + /// Returns number of RLP items. /// /// ```rust /// extern crate ethcore_util as util; @@ -210,7 +210,7 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.size() } - /// Get view onto rlp-slice at index. + /// Get view onto RLP-slice at index. /// /// Caches offset to given index, so access to successive /// slices is faster. @@ -337,7 +337,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } } - /// The bare data of the rlp. + /// The bare data of the RLP. /// /// ```rust /// extern crate ethcore_util as util; @@ -355,7 +355,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } pub fn data(&'view self) -> &'a [u8] { - unimplemented!(); + let ii = Self::item_info(self.bytes).unwrap(); + &self.bytes[ii.prefix_len..(ii.prefix_len + ii.value_len)] } /// Returns number of rlp items. diff --git a/src/trie.rs b/src/trie.rs index 0cf030c75..9f0b209be 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -101,7 +101,7 @@ impl TrieDB { fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); - let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); + let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); } @@ -134,11 +134,13 @@ impl TrieDB { /// the trie. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { + trace!("take_node {:?} (inline)", rlp.raw()); rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!"); + trace!("take_node {:?} (indirect for {:?})", rlp.raw(), r); diff.delete_node_sha3(h); r } @@ -150,7 +152,8 @@ impl TrieDB { /// Transform an existing extension or leaf node to an invalid single-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** - fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_extension_to_branch"); let mut s = RlpStream::new_list(17); assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. let index = orig_partial.at(0); @@ -171,7 +174,8 @@ impl TrieDB { s.out() } - fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_leaf_to_branch"); let mut s = RlpStream::new_list(17); let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; // orig is leaf - orig_raw_payload is data representing the actual value. @@ -189,24 +193,26 @@ impl TrieDB { /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** - fn transmute_to_branch_and_inject(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_to_branch_and_augmented(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_to_branch_and_augmented"); let intermediate = match orig_is_leaf { - true => Self::transmute_leaf_to_branch(orig_partial, orig_raw_payload, diff), - false => Self::transmute_extension_to_branch(orig_partial, orig_raw_payload, diff), + true => Self::transmuted_leaf_to_branch(orig_partial, orig_raw_payload, diff), + false => Self::transmuted_extension_to_branch(orig_partial, orig_raw_payload, diff), }; - self.inject(&intermediate, partial, value, diff) + self.augmented(&intermediate, partial, value, diff) // TODO: implement without having to make an intermediate representation. } /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that /// once applied the returned node is valid. - fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("augmented_into_branch"); let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; for i in 0usize..17 { if index == i { - // this is node to inject into... + // this is node to augment into... if orig.at(i).is_empty() { // easy - original had empty slot. diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); @@ -216,7 +222,7 @@ impl TrieDB { diff.replace_node(&orig.at(i), new, &mut s); } else { // harder - original has something there already - let new = self.inject(orig.at(i).raw(), &partial.mid(1), value, diff); + let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff); diff.replace_node(&orig.at(i), new, &mut s); } } else { @@ -226,8 +232,6 @@ impl TrieDB { s.out() } - - /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that includes the new node. @@ -236,31 +240,35 @@ impl TrieDB { /// and deleting nodes as necessary. /// /// **This operation will not insert the new node now destroy the original.** - fn inject(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("augmented ({:?}, {:?}, {:?})", old, partial, value); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { - // already have a branch. route and inject. - self.injected_into_branch(&old_rlp, partial, value, diff) + // already have a branch. route and augment. + self.augmented_into_branch(&old_rlp, partial, value, diff) }, Prototype::List(2) => { - let their_key_rlp = old_rlp.at(0); - let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial.common_prefix(&them) { - 0 if partial.is_empty() && them.is_empty() => { - // both empty: just replace. + let existing_key_rlp = old_rlp.at(0); + let (existing_key, is_leaf) = NibbleSlice::from_encoded(existing_key_rlp.data()); + match partial.common_prefix(&existing_key) { + cp if partial.len() == existing_key.len() && cp == existing_key.len() && is_leaf => { + // equivalent-leaf: replace + trace!("equivalent-leaf: REPLACE"); Self::compose_leaf(partial, value) }, 0 => { // one of us isn't empty: transmute to branch here - self.transmute_to_branch_and_inject(is_leaf, &them, old_rlp.at(1).raw(), partial, value, diff) + trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); + self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, - cp if cp == them.len() => { + cp if cp == existing_key.len() => { + trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: - // skip to the end of this extension and continue the inject there. + // skip to the end of this extension and continue to augment there. let n = self.take_node(&old_rlp.at(1), diff); - let downstream_node = self.inject(n, &partial.mid(cp), value, diff); + let downstream_node = self.augmented(n, &partial.mid(cp), value, diff); let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); diff.new_node(downstream_node, &mut s); @@ -269,19 +277,21 @@ impl TrieDB { cp => { // partially-shared prefix for this extension: // split into two extensions, high and low, pass the - // low through inject with the value before inserting the result + // low through augment with the value before inserting the result // into high to create the new. - // TODO: optimise by doing this without creating injected_low. + // TODO: optimise by doing this without creating augmented_low. + + trace!("partially-shared-prefix (exist={:?}; new={:?}; cp={:?}): AUGMENT-AT-END", existing_key.len(), partial.len(), cp); // low (farther from root) - let low = Self::compose_raw(&them.mid(cp), old_rlp.at(1).raw(), is_leaf); - let injected_low = self.inject(&low, &partial.mid(cp), value, diff); + let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).raw(), is_leaf); + let augmented_low = self.augmented(&low, &partial.mid(cp), value, diff); // high (closer to root) let mut s = RlpStream::new_list(2); - s.append(&them.encoded_leftmost(cp, false)); - diff.new_node(injected_low, &mut s); + s.append(&existing_key.encoded_leftmost(cp, false)); + diff.new_node(augmented_low, &mut s); s.out() }, } @@ -314,21 +324,44 @@ impl Trie for TrieDB { } } -#[test] -fn playpen() { - use overlaydb::*; +#[cfg(test)] +mod tests { + use memorydb::*; use triehash::*; + use super::*; use env_logger; - env_logger::init().unwrap(); + #[test] + fn playpen() { + env_logger::init().unwrap(); - (&[1, 2, 3]).starts_with(&[1, 2]); + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } - let mut t = TrieDB::new(OverlayDB::new_temp()); - t.init(); - assert_eq!(*t.root(), SHA3_NULL_RLP); - assert!(t.is_empty()); + #[test] + fn init() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + assert_eq!(*t.root(), SHA3_NULL_RLP); + assert!(t.is_empty()); + } - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); -} + #[test] + fn insert_on_empty() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); + } + + #[test] + fn insert_replace_root() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); + } +} \ No newline at end of file