From 5ba76d767e7b16a8c2502a7b564828e0bba44c40 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 15:54:17 +0100 Subject: [PATCH 1/3] Account diffing. --- src/account.rs | 131 ++++++++++++++++++++++++++++++++++++++++++++- src/tests/state.rs | 8 +-- 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/src/account.rs b/src/account.rs index 8a51d9b97..4712c30a5 100644 --- a/src/account.rs +++ b/src/account.rs @@ -2,7 +2,24 @@ use util::*; pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); -#[derive(Debug)] +#[derive(Debug,Clone,PartialEq,Eq)] +pub struct Diff where T: Eq { + pub pre: T, + pub post_opt: Option, +} + +impl Diff where T: Eq { + pub fn new_opt(pre: T, post: T) -> Option { if pre == post { None } else { Some(Self::new(pre, post)) } } + pub fn one_opt(t: T) -> Option { Some(Self::one(t)) } + + pub fn new(pre: T, post: T) -> Self { Diff { pre: pre, post_opt: Some(post) }} + pub fn one(t: T) -> Self { Diff { pre: t, post_opt: None }} + + pub fn pre(&self) -> &T { &self.pre } + pub fn post(&self) -> &T { match self.post_opt { Some(ref x) => x, None => &self.pre } } +} + +#[derive(Debug,Clone,PartialEq,Eq)] /// Genesis account data. Does not have a DB overlay cache. pub struct PodAccount { // Balance of the account. @@ -13,6 +30,116 @@ pub struct PodAccount { pub storage: BTreeMap, } +#[derive(Debug,Clone,PartialEq,Eq)] +pub struct PodAccountDiff { + pub exists: Diff, + pub balance: Option>, + pub nonce: Option>, + pub code: Option>, + pub storage: BTreeMap>, +} + +type StateDiff = BTreeMap; + +pub fn diff(pre: &Option, post: &Option) -> Option { + match (pre, post) { + (&Some(ref x), &None) | (&None, &Some(ref x)) => Some(PodAccountDiff { + exists: Diff::new(pre.is_some(), post.is_some()), + balance: Diff::one_opt(x.balance.clone()), + nonce: Diff::one_opt(x.nonce.clone()), + code: Diff::one_opt(x.code.clone()), + storage: x.storage.iter().fold(BTreeMap::new(), |mut m, (k, v)| {m.insert(k.clone(), Diff::one(v.clone())); m}) + }), + (&Some(ref pre), &Some(ref post)) => { + let pre_keys: BTreeSet<_> = pre.storage.keys().collect(); + let post_keys: BTreeSet<_> = post.storage.keys().collect(); + let storage: Vec<_> = pre_keys.union(&post_keys) + .filter(|k| pre.storage.get(k).unwrap_or(&H256::new()) != post.storage.get(k).unwrap_or(&H256::new())) + .collect(); + if pre.balance != post.balance || pre.nonce != post.nonce || pre.code != post.code || storage.len() > 0 { + Some(PodAccountDiff { + exists: Diff::one(true), + balance: Diff::new_opt(pre.balance.clone(), post.balance.clone()), + nonce: Diff::new_opt(pre.nonce.clone(), post.nonce.clone()), + code: Diff::new_opt(pre.code.clone(), post.code.clone()), + storage: storage.into_iter().fold(BTreeMap::new(), |mut m, k| { + let v = Diff::new(pre.storage.get(&k).cloned().unwrap_or(H256::new()), post.storage.get(&k).cloned().unwrap_or(H256::new())); + m.insert((*k).clone(), v); + m + }), + }) + } else { + None + } + }, + _ => None, + } +} + +#[test] +fn account_diff_existence() { + let a = Some(PodAccount{balance: U256::from(69u64), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); + assert_eq!(diff(&a, &a), None); + assert_eq!(diff(&None, &a), Some(PodAccountDiff{ + exists: Diff::new(false, true), + balance: Diff::one_opt(U256::from(69u64)), + nonce: Diff::one_opt(U256::zero()), + code: Diff::one_opt(vec![]), + storage: BTreeMap::new(), + })); +} + +#[test] +fn account_diff_basic() { + let a = Some(PodAccount{balance: U256::from(69u64), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); + let b = Some(PodAccount{balance: U256::from(42u64), nonce: U256::from(1u64), code: vec![], storage: BTreeMap::new()}); + assert_eq!(diff(&a, &b), Some(PodAccountDiff { + exists: Diff::one(true), + balance: Diff::new_opt(U256::from(69u64), U256::from(42u64)), + nonce: Diff::new_opt(U256::zero(), U256::from(1u64)), + code: None, + storage: BTreeMap::new(), + })); +} + +#[test] +fn account_diff_code() { + let a = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); + let b = Some(PodAccount{balance: U256::zero(), nonce: U256::from(1u64), code: vec![0x00u8], storage: BTreeMap::new()}); + assert_eq!(diff(&a, &b), Some(PodAccountDiff { + exists: Diff::one(true), + balance: None, + nonce: Diff::new_opt(U256::zero(), U256::from(1u64)), + code: Diff::new_opt(vec![], vec![0x00u8]), + storage: BTreeMap::new(), + })); +} + +pub fn h256_from_u8(v: u8) -> H256 { + let mut r = H256::new(); + r[31] = v; + r +} + +#[test] +fn account_diff_storage() { + let a = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 2), (3, 3), (4, 4), (5, 0), (6, 0), (7, 0)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}); + let b = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 3), (3, 0), (5, 0), (7, 7), (8, 0), (9, 9)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}); + assert_eq!(diff(&a, &b), Some(PodAccountDiff { + exists: Diff::one(true), + balance: None, + nonce: None, + code: None, + storage: vec![ + (2u8, Diff::new(h256_from_u8(2), h256_from_u8(3))), + (3, Diff::new(h256_from_u8(3), H256::new())), + (4, Diff::new(h256_from_u8(4), H256::new())), + (7, Diff::new(H256::new(), h256_from_u8(7))), + (9, Diff::new(H256::new(), h256_from_u8(9))), + ].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), v); m}) + })); +} + /// Single account in the system. #[derive(Clone)] pub struct Account { @@ -229,7 +356,7 @@ impl Account { /// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this. pub fn commit_code(&mut self, db: &mut HashDB) { - println!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty()); + trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty()); match (self.code_hash.is_none(), self.code_cache.is_empty()) { (true, true) => self.code_hash = Some(SHA3_EMPTY), (true, false) => { diff --git a/src/tests/state.rs b/src/tests/state.rs index 02f5b9cfd..9502f08e4 100644 --- a/src/tests/state.rs +++ b/src/tests/state.rs @@ -55,18 +55,20 @@ fn do_json_test(json_data: &[u8]) -> Vec { println!("Transaction: {:?}", t); println!("Env: {:?}", env); - println!("Out: {:?}", out); - println!("Pre: {:?}", pre); - println!("Post: {:?}", post); + + println!("Pre:\n{:?}", pre); let mut s = State::new_temp(); s.populate_from(pre); + println!("Our-pre:\n{:?}", s.to_pod_map()); + s.apply(&env, engine.deref(), &t).unwrap(); let our_post = s.to_pod_map(); if fail_unless(s.root() == &post_state_root) { println!("EXPECTED:\n{:?}", post); println!("GOT:\n{:?}", our_post); +// println!("DIFF:\n{:?}", pod_map_diff(post, our_post)); } // TODO: Compare logs. From 311aa5c442f9f024ad85139f111918d44ae96b70 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 18:37:09 +0100 Subject: [PATCH 2/3] State diffing. --- Cargo.toml | 1 + src/account.rs | 193 +++++++++++++++++++++++++++++++++++++++++++------ src/lib.rs | 8 +- 3 files changed, 173 insertions(+), 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bffab3ac3..a15f19ec8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ rust-crypto = "0.2.34" time = "0.1" #interpolate_idents = { git = "https://github.com/SkylerLipthay/interpolate_idents" } evmjit = { path = "rust-evmjit", optional = true } +itertools = "0.4" [features] default = ["jit"] diff --git a/src/account.rs b/src/account.rs index 4712c30a5..cd5a310e8 100644 --- a/src/account.rs +++ b/src/account.rs @@ -1,4 +1,5 @@ use util::*; +use itertools::Itertools; pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); @@ -19,6 +20,12 @@ impl Diff where T: Eq { pub fn post(&self) -> &T { match self.post_opt { Some(ref x) => x, None => &self.pre } } } +impl From for Diff where T: Eq { + fn from(t: T) -> Diff { + Diff::one(t) + } +} + #[derive(Debug,Clone,PartialEq,Eq)] /// Genesis account data. Does not have a DB overlay cache. pub struct PodAccount { @@ -31,7 +38,7 @@ pub struct PodAccount { } #[derive(Debug,Clone,PartialEq,Eq)] -pub struct PodAccountDiff { +pub struct AccountDiff { pub exists: Diff, pub balance: Option>, pub nonce: Option>, @@ -39,25 +46,23 @@ pub struct PodAccountDiff { pub storage: BTreeMap>, } -type StateDiff = BTreeMap; +pub type StateDiff = BTreeMap; -pub fn diff(pre: &Option, post: &Option) -> Option { +pub fn pod_diff(pre: Option<&PodAccount>, post: Option<&PodAccount>) -> Option { match (pre, post) { - (&Some(ref x), &None) | (&None, &Some(ref x)) => Some(PodAccountDiff { + (Some(x), None) | (None, Some(x)) => Some(AccountDiff { exists: Diff::new(pre.is_some(), post.is_some()), balance: Diff::one_opt(x.balance.clone()), nonce: Diff::one_opt(x.nonce.clone()), code: Diff::one_opt(x.code.clone()), storage: x.storage.iter().fold(BTreeMap::new(), |mut m, (k, v)| {m.insert(k.clone(), Diff::one(v.clone())); m}) }), - (&Some(ref pre), &Some(ref post)) => { - let pre_keys: BTreeSet<_> = pre.storage.keys().collect(); - let post_keys: BTreeSet<_> = post.storage.keys().collect(); - let storage: Vec<_> = pre_keys.union(&post_keys) + (Some(pre), Some(post)) => { + let storage: Vec<_> = pre.storage.keys().merge(post.storage.keys()) .filter(|k| pre.storage.get(k).unwrap_or(&H256::new()) != post.storage.get(k).unwrap_or(&H256::new())) .collect(); if pre.balance != post.balance || pre.nonce != post.nonce || pre.code != post.code || storage.len() > 0 { - Some(PodAccountDiff { + Some(AccountDiff { exists: Diff::one(true), balance: Diff::new_opt(pre.balance.clone(), post.balance.clone()), nonce: Diff::new_opt(pre.nonce.clone(), post.nonce.clone()), @@ -76,24 +81,164 @@ pub fn diff(pre: &Option, post: &Option) -> Option, post: &BTreeMap) -> StateDiff { + pre.keys() + .merge(post.keys()) + .filter_map(|acc| pod_diff(pre.get(acc), post.get(acc)).map(|d|(acc.clone(), d))) + .collect::>() +} + +macro_rules! map { + ( $( $x:expr => $y:expr ),* ) => { + vec![ $( ($x, $y) ),* ].into_iter().collect::>() + } +} + +macro_rules! x { + ( $x:expr ) => { + From::from($x) + } +} + +macro_rules! xx { + ( $x:expr ) => { + From::from(From::from($x)) + } +} + +#[test] +fn state_diff_create_delete() { + let a = map![ + x!(1) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + } + ]; + assert_eq!(pod_map_diff(&a, &map![]), map![ + x!(1) => AccountDiff{ + exists: Diff::new(true, false), + balance: Diff::one_opt(x!(69)), + nonce: Diff::one_opt(x!(0)), + code: Diff::one_opt(vec![]), + storage: map![], + } + ]); + assert_eq!(pod_map_diff(&map![], &a), map![ + x!(1) => AccountDiff{ + exists: Diff::new(false, true), + balance: Diff::one_opt(x!(69)), + nonce: Diff::one_opt(x!(0)), + code: Diff::one_opt(vec![]), + storage: map![], + } + ]); +} + +#[test] +fn state_diff_cretae_delete_with_unchanged() { + let a = map![ + x!(1) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + } + ]; + let b = map![ + x!(1) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + }, + x!(2) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + } + ]; + assert_eq!(pod_map_diff(&a, &b), map![ + x!(2) => AccountDiff{ + exists: Diff::new(false, true), + balance: Diff::one_opt(x!(69)), + nonce: Diff::one_opt(x!(0)), + code: Diff::one_opt(vec![]), + storage: map![], + } + ]); + assert_eq!(pod_map_diff(&b, &a), map![ + x!(2) => AccountDiff{ + exists: Diff::new(true, false), + balance: Diff::one_opt(x!(69)), + nonce: Diff::one_opt(x!(0)), + code: Diff::one_opt(vec![]), + storage: map![], + } + ]); +} + +#[test] +fn state_diff_change_with_unchanged() { + let a = map![ + x!(1) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + }, + x!(2) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + } + ]; + let b = map![ + x!(1) => PodAccount{ + balance: x!(69), + nonce: x!(1), + code: vec![], + storage: map![] + }, + x!(2) => PodAccount{ + balance: x!(69), + nonce: x!(0), + code: vec![], + storage: map![] + } + ]; + assert_eq!(pod_map_diff(&a, &b), map![ + x!(1) => AccountDiff{ + exists: Diff::one(true), + balance: None, + nonce: Diff::new_opt(x!(0), x!(1)), + code: None, + storage: map![], + } + ]); +} + #[test] fn account_diff_existence() { - let a = Some(PodAccount{balance: U256::from(69u64), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); - assert_eq!(diff(&a, &a), None); - assert_eq!(diff(&None, &a), Some(PodAccountDiff{ + let a = PodAccount{balance: x!(69), nonce: x!(0), code: vec![], storage: map![]}; + assert_eq!(pod_diff(Some(&a), Some(&a)), None); + assert_eq!(pod_diff(None, Some(&a)), Some(AccountDiff{ exists: Diff::new(false, true), - balance: Diff::one_opt(U256::from(69u64)), - nonce: Diff::one_opt(U256::zero()), + balance: Diff::one_opt(x!(69)), + nonce: Diff::one_opt(x!(0)), code: Diff::one_opt(vec![]), - storage: BTreeMap::new(), + storage: map![], })); } #[test] fn account_diff_basic() { - let a = Some(PodAccount{balance: U256::from(69u64), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); - let b = Some(PodAccount{balance: U256::from(42u64), nonce: U256::from(1u64), code: vec![], storage: BTreeMap::new()}); - assert_eq!(diff(&a, &b), Some(PodAccountDiff { + let a = PodAccount{balance: U256::from(69u64), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}; + let b = PodAccount{balance: U256::from(42u64), nonce: U256::from(1u64), code: vec![], storage: BTreeMap::new()}; + assert_eq!(pod_diff(Some(&a), Some(&b)), Some(AccountDiff { exists: Diff::one(true), balance: Diff::new_opt(U256::from(69u64), U256::from(42u64)), nonce: Diff::new_opt(U256::zero(), U256::from(1u64)), @@ -104,9 +249,9 @@ fn account_diff_basic() { #[test] fn account_diff_code() { - let a = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}); - let b = Some(PodAccount{balance: U256::zero(), nonce: U256::from(1u64), code: vec![0x00u8], storage: BTreeMap::new()}); - assert_eq!(diff(&a, &b), Some(PodAccountDiff { + let a = PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: BTreeMap::new()}; + let b = PodAccount{balance: U256::zero(), nonce: U256::from(1u64), code: vec![0x00u8], storage: BTreeMap::new()}; + assert_eq!(pod_diff(Some(&a), Some(&b)), Some(AccountDiff { exists: Diff::one(true), balance: None, nonce: Diff::new_opt(U256::zero(), U256::from(1u64)), @@ -123,9 +268,9 @@ pub fn h256_from_u8(v: u8) -> H256 { #[test] fn account_diff_storage() { - let a = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 2), (3, 3), (4, 4), (5, 0), (6, 0), (7, 0)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}); - let b = Some(PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 3), (3, 0), (5, 0), (7, 7), (8, 0), (9, 9)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}); - assert_eq!(diff(&a, &b), Some(PodAccountDiff { + let a = PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 2), (3, 3), (4, 4), (5, 0), (6, 0), (7, 0)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}; + let b = PodAccount{balance: U256::zero(), nonce: U256::zero(), code: vec![], storage: vec![(1u8, 1u8), (2, 3), (3, 0), (5, 0), (7, 7), (8, 0), (9, 9)].into_iter().fold(BTreeMap::new(), |mut m, (k, v)|{m.insert(h256_from_u8(k), h256_from_u8(v)); m})}; + assert_eq!(pod_diff(Some(&a), Some(&b)), Some(AccountDiff { exists: Diff::one(true), balance: None, nonce: None, diff --git a/src/lib.rs b/src/lib.rs index 76ef7f09b..5cecba2b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,18 +73,16 @@ //! sudo ldconfig //! ``` -#[macro_use] -extern crate log; +#[macro_use] extern crate log; extern crate rustc_serialize; +#[macro_use] extern crate itertools; extern crate flate2; extern crate rocksdb; extern crate heapsize; extern crate crypto; extern crate time; - extern crate env_logger; -#[cfg(feature = "jit" )] -extern crate evmjit; +#[cfg(feature = "jit" )] extern crate evmjit; extern crate ethcore_util as util; From 602d9d99b14ad10d6823ce90b910f9733fa8a0d2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 18:40:18 +0100 Subject: [PATCH 3/3] State conensus tests now print mismatching diff. --- src/tests/state.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/tests/state.rs b/src/tests/state.rs index 9502f08e4..208ba4dfd 100644 --- a/src/tests/state.rs +++ b/src/tests/state.rs @@ -47,7 +47,7 @@ fn do_json_test(json_data: &[u8]) -> Vec { let t = Transaction::from_json(&test["transaction"]); let env = EnvInfo::from_json(&test["env"]); - let out = bytes_from_json(&test["out"]); + let _out = bytes_from_json(&test["out"]); let post_state_root = h256_from_json(&test["postStateRoot"]); let pre = pod_map_from_json(&test["pre"]); let post = pod_map_from_json(&test["post"]); @@ -56,19 +56,14 @@ fn do_json_test(json_data: &[u8]) -> Vec { println!("Transaction: {:?}", t); println!("Env: {:?}", env); - println!("Pre:\n{:?}", pre); - let mut s = State::new_temp(); s.populate_from(pre); - println!("Our-pre:\n{:?}", s.to_pod_map()); s.apply(&env, engine.deref(), &t).unwrap(); let our_post = s.to_pod_map(); if fail_unless(s.root() == &post_state_root) { - println!("EXPECTED:\n{:?}", post); - println!("GOT:\n{:?}", our_post); -// println!("DIFF:\n{:?}", pod_map_diff(post, our_post)); + println!("DIFF:\n{:?}", pod_map_diff(&post, &our_post)); } // TODO: Compare logs.