From 1e40997ff74b25c03a07fcb38e0c89799410bce1 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 11 Mar 2016 22:15:56 +0400 Subject: [PATCH 01/26] state query for archive jdb --- util/src/journaldb/archivedb.rs | 23 +++++++++++++++++++++++ util/src/journaldb/traits.rs | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index e7da1b737..c8c29e765 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -159,6 +159,10 @@ impl JournalDB for ArchiveDB { try!(self.backing.write(batch)); Ok((inserts + deletes) as u32) } + + fn state(&self, id: &H256) -> Option { + self.backing.get_by_prefix(&id.bytes()[0..12]).and_then(|b| Some(b.to_vec())) + } } #[cfg(test)] @@ -385,4 +389,23 @@ mod tests { assert!(jdb.exists(&foo)); } } + + #[test] + fn returns_state() { + let temp = ::devtools::RandomTempPath::new(); + + let key = { + let mut jdb = ArchiveDB::new(temp.as_str()); + let key = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + key + }; + + { + let jdb = ArchiveDB::new(temp.as_str()); + let state = jdb.state(&key); + assert!(state.is_some()); + } + + } } diff --git a/util/src/journaldb/traits.rs b/util/src/journaldb/traits.rs index 25e132339..017c24330 100644 --- a/util/src/journaldb/traits.rs +++ b/util/src/journaldb/traits.rs @@ -34,4 +34,9 @@ pub trait JournalDB : HashDB + Send + Sync { /// Commit all recent insert operations and canonical historical commits' removals from the /// old era to the backing database, reverting any non-canonical historical commit's inserts. fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; + + /// State data query + fn state(&self, _id: &H256) -> Option { + None + } } From 12e1abdfb70e3cd2a662cbcbb4b44bbbe3c56f41 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 12 Mar 2016 09:51:17 +0100 Subject: [PATCH 02/26] Port fixes to new infrastrtcutre. --- util/src/journaldb/optiononedb.rs | 572 +++++++++++++++++++++++++----- 1 file changed, 492 insertions(+), 80 deletions(-) diff --git a/util/src/journaldb/optiononedb.rs b/util/src/journaldb/optiononedb.rs index dfa7c8ec1..b51d0819d 100644 --- a/util/src/journaldb/optiononedb.rs +++ b/util/src/journaldb/optiononedb.rs @@ -25,6 +25,34 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; +#[derive(Clone, PartialEq, Eq)] +struct RefInfo { + queue_refs: usize, + in_archive: bool, +} + +impl HeapSizeOf for RefInfo { + fn heap_size_of_children(&self) -> usize { 0 } +} + +impl fmt::Display for RefInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}+{}", self.queue_refs, if self.in_archive {1} else {0}) + } +} + +impl fmt::Debug for RefInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}+{}", self.queue_refs, if self.in_archive {1} else {0}) + } +} + +#[derive(Clone, PartialEq, Eq)] +enum RemoveFrom { + Queue, + Archive, +} + /// Implementation of the HashDB trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -35,7 +63,7 @@ use std::env; pub struct OptionOneDB { overlay: MemoryDB, backing: Arc, - counters: Option>>>, + refs: Option>>>, } // all keys must be at least 12 bytes @@ -62,11 +90,11 @@ impl OptionOneDB { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); } - let counters = Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))); + let refs = Some(Arc::new(RwLock::new(OptionOneDB::read_refs(&backing)))); OptionOneDB { overlay: MemoryDB::new(), backing: Arc::new(backing), - counters: counters, + refs: refs, } } @@ -91,11 +119,14 @@ impl OptionOneDB { backing.get(&Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some() } - fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, counters: &mut HashMap, batch: &DBTransaction) { + fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, refs: &mut HashMap, batch: &DBTransaction, trace: bool) { for &(ref h, ref d) in inserts { - if let Some(c) = counters.get_mut(h) { + if let Some(c) = refs.get_mut(h) { // already counting. increment. - *c += 1; + c.queue_refs += 1; + if trace { + trace!(target: "jdb.fine", " insert({}): In queue: Incrementing refs to {}", h, c.queue_refs); + } continue; } @@ -103,7 +134,10 @@ impl OptionOneDB { if backing.get(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?").is_some() { // already in the backing DB. start counting, and remember it was already in. Self::set_already_in(batch, &h); - counters.insert(h.clone(), 1); + refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true}); + if trace { + trace!(target: "jdb.fine", " insert({}): New to queue, in DB: Recording and inserting into queue", h); + } continue; } @@ -111,60 +145,104 @@ impl OptionOneDB { //Self::reset_already_in(&h); assert!(!Self::is_already_in(backing, &h)); batch.put(&h.bytes(), d).expect("Low-level database error. Some issue with your hard disk?"); + refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false}); + if trace { + trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h); + } } } - fn replay_keys(inserts: &[H256], backing: &Database, counters: &mut HashMap) { - trace!("replay_keys: inserts={:?}, counters={:?}", inserts, counters); + fn replay_keys(inserts: &[H256], backing: &Database, refs: &mut HashMap) { + trace!(target: "jdb.fine", "replay_keys: inserts={:?}, refs={:?}", inserts, refs); for h in inserts { - if let Some(c) = counters.get_mut(h) { + if let Some(c) = refs.get_mut(h) { // already counting. increment. - *c += 1; + c.queue_refs += 1; continue; } // this is the first entry for this node in the journal. // it is initialised to 1 if it was already in. - if Self::is_already_in(backing, h) { - trace!("replace_keys: Key {} was already in!", h); - counters.insert(h.clone(), 1); - } + refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, h)}); } - trace!("replay_keys: (end) counters={:?}", counters); + trace!(target: "jdb.fine", "replay_keys: (end) refs={:?}", refs); } - fn kill_keys(deletes: Vec, counters: &mut HashMap, batch: &DBTransaction) { - for h in deletes.into_iter() { - let mut n: Option = None; - if let Some(c) = counters.get_mut(&h) { - if *c > 1 { - *c -= 1; + fn kill_keys(deletes: &Vec, refs: &mut HashMap, batch: &DBTransaction, from: RemoveFrom, trace: bool) { + // with a kill on {queue_refs: 1, in_archive: true}, we have two options: + // - convert to {queue_refs: 1, in_archive: false} (i.e. remove it from the conceptual archive) + // - convert to {queue_refs: 0, in_archive: true} (i.e. remove it from the conceptual queue) + // (the latter option would then mean removing the RefInfo, since it would no longer be counted in the queue.) + // both are valid, but we switch between them depending on context. + // All inserts in queue (i.e. those which may yet be reverted) have an entry in refs. + for h in deletes.iter() { + let mut n: Option = None; + if let Some(c) = refs.get_mut(h) { + if c.in_archive && from == RemoveFrom::Archive { + c.in_archive = false; + Self::reset_already_in(batch, h); + if trace { + trace!(target: "jdb.fine", " kill({}): In archive, 1 in queue: Reducing to queue only and recording", h); + } + continue; + } else if c.queue_refs > 1 { + c.queue_refs -= 1; + if trace { + trace!(target: "jdb.fine", " kill({}): In queue > 1 refs: Decrementing ref count to {}", h, c.queue_refs); + } continue; } else { - n = Some(*c); + n = Some(c.clone()); } } match n { - Some(i) if i == 1 => { - counters.remove(&h); - Self::reset_already_in(batch, &h); + Some(RefInfo{queue_refs: 1, in_archive: true}) => { + refs.remove(h); + Self::reset_already_in(batch, h); + if trace { + trace!(target: "jdb.fine", " kill({}): In archive, 1 in queue: Removing from queue and leaving in archive", h); + } + } + Some(RefInfo{queue_refs: 1, in_archive: false}) => { + refs.remove(h); + batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?"); + if trace { + trace!(target: "jdb.fine", " kill({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h); + } } None => { // Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs. //assert!(!Self::is_already_in(db, &h)); batch.delete(&h.bytes()).expect("Low-level database error. Some issue with your hard disk?"); + if trace { + trace!(target: "jdb.fine", " kill({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h); + } } - _ => panic!("Invalid value in counters: {:?}", n), + _ => panic!("Invalid value in refs: {:?}", n), } } } + #[cfg(test)] + fn can_reconstruct_refs(&self) -> bool { + let reconstructed = Self::read_refs(&self.backing); + let refs = self.refs.as_ref().unwrap().write().unwrap(); + if *refs != reconstructed { + let clean_refs = refs.iter().filter_map(|(k, v)| if reconstructed.get(k) == Some(v) {None} else {Some((k.clone(), v.clone()))}).collect::>(); + let clean_recon = reconstructed.into_iter().filter_map(|(k, v)| if refs.get(&k) == Some(&v) {None} else {Some((k.clone(), v.clone()))}).collect::>(); + warn!(target: "jdb", "mem: {:?} != log: {:?}", clean_refs, clean_recon); + false + } else { + true + } + } + fn payload(&self, key: &H256) -> Option { self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_counters(db: &Database) -> HashMap { - let mut counters = HashMap::new(); + fn read_refs(db: &Database) -> HashMap { + let mut refs = HashMap::new(); if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { let mut era = decode::(&val); loop { @@ -176,10 +254,9 @@ impl OptionOneDB { r.append(&&PADDING[..]); &r.drain() }).expect("Low-level database error.") { - trace!("read_counters: era={}, index={}", era, index); let rlp = Rlp::new(&rlp_data); let inserts: Vec = rlp.val_at(1); - Self::replay_keys(&inserts, db, &mut counters); + Self::replay_keys(&inserts, db, &mut refs); index += 1; }; if index == 0 || era == 0 { @@ -188,10 +265,9 @@ impl OptionOneDB { era -= 1; } } - trace!("Recovered {} counters", counters.len()); - counters + refs } -} + } impl HashDB for OptionOneDB { fn keys(&self) -> HashMap { @@ -243,23 +319,23 @@ impl JournalDB for OptionOneDB { Box::new(OptionOneDB { overlay: MemoryDB::new(), backing: self.backing.clone(), - counters: self.counters.clone(), + refs: self.refs.clone(), }) } - fn mem_used(&self) -> usize { - self.overlay.mem_used() + match self.counters { - Some(ref c) => c.read().unwrap().heap_size_of_children(), - None => 0 - } - } - fn is_empty(&self) -> bool { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } + fn mem_used(&self) -> usize { + self.overlay.mem_used() + match self.refs { + Some(ref c) => c.read().unwrap().heap_size_of_children(), + None => 0 + } + } + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { - // journal format: + // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, n] => [ ... ] @@ -304,9 +380,9 @@ impl JournalDB for OptionOneDB { // // record new commit's details. - trace!("commit: #{} ({}), end era: {:?}", now, id, end); - let mut counters = self.counters.as_ref().unwrap().write().unwrap(); + let mut refs = self.refs.as_ref().unwrap().write().unwrap(); let batch = DBTransaction::new(); + let trace = false; { let mut index = 0usize; let mut last; @@ -323,6 +399,11 @@ impl JournalDB for OptionOneDB { } let drained = self.overlay.drain(); + + if trace { + trace!(target: "jdb", "commit: #{} ({}), end era: {:?}", now, id, end); + } + let removes: Vec = drained .iter() .filter_map(|(k, &(_, c))| if c < 0 {Some(k.clone())} else {None}) @@ -332,6 +413,9 @@ impl JournalDB for OptionOneDB { .filter_map(|(k, (v, r))| if r > 0 { assert!(r == 1); Some((k, v)) } else { assert!(r >= -1); None }) .collect(); + + // TODO: check all removes are in the db. + let mut r = RlpStream::new_list(3); r.append(id); @@ -344,7 +428,12 @@ impl JournalDB for OptionOneDB { r.begin_list(inserts.len()); inserts.iter().foreach(|&(k, _)| {r.append(&k);}); r.append(&removes); - Self::insert_keys(&inserts, &self.backing, &mut counters, &batch); + Self::insert_keys(&inserts, &self.backing, &mut refs, &batch, trace); + if trace { + let ins = inserts.iter().map(|&(k, _)| k).collect::>(); + trace!(target: "jdb.ops", " Inserts: {:?}", ins); + trace!(target: "jdb.ops", " Deletes: {:?}", removes); + } try!(batch.put(&last, r.as_raw())); try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); } @@ -363,17 +452,64 @@ impl JournalDB for OptionOneDB { })) { let rlp = Rlp::new(&rlp_data); let inserts: Vec = rlp.val_at(1); - let deletes: Vec = rlp.val_at(2); - // Collect keys to be removed. These are removed keys for canonical block, inserted for non-canonical - Self::kill_keys(if canon_id == rlp.val_at(0) {deletes} else {inserts}, &mut counters, &batch); + + if canon_id == rlp.val_at(0) { + // Collect keys to be removed. Canon block - remove the (enacted) deletes. + let deletes: Vec = rlp.val_at(2); + if trace { + trace!(target: "jdb.ops", " Expunging: {:?}", deletes); + } + Self::kill_keys(&deletes, &mut refs, &batch, RemoveFrom::Archive, trace); + + if trace { + trace!(target: "jdb.ops", " Finalising: {:?}", inserts); + } + for k in inserts.iter() { + match refs.get(k).cloned() { + None => { + // [in archive] -> SHIFT remove -> SHIFT insert None->Some{queue_refs: 1, in_archive: true} -> TAKE remove Some{queue_refs: 1, in_archive: true}->None -> TAKE insert + // already expunged from the queue (which is allowed since the key is in the archive). + // leave well alone. + } + Some( RefInfo{queue_refs: 1, in_archive: false} ) => { + // just delete the refs entry. + refs.remove(k); + } + Some( RefInfo{queue_refs: x, in_archive: false} ) => { + // must set already in; , + Self::set_already_in(&batch, k); + refs.insert(k.clone(), RefInfo{ queue_refs: x - 1, in_archive: true }); + } + Some( RefInfo{queue_refs: _, in_archive: true} ) => { + // Invalid! Reinserted the same key twice. + warn!("Key {} inserted twice into same fork.", k); + } + } + } + } else { + // Collect keys to be removed. Non-canon block - remove the (reverted) inserts. + if trace { + trace!(target: "jdb.ops", " Reverting: {:?}", inserts); + } + Self::kill_keys(&inserts, &mut refs, &batch, RemoveFrom::Queue, trace); + } + try!(batch.delete(&last)); index += 1; } - trace!("OptionOneDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); + if trace { + trace!(target: "jdb", "delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); + } } try!(self.backing.write(batch)); -// trace!("OptionOneDB::commit() deleted {} nodes", deletes); + + // Comment out for now. TODO: automatically enable in tests. + + if trace { + trace!(target: "jdb", "OK: {:?}", refs.clone()); + } + Ok(0) } } @@ -383,26 +519,34 @@ mod tests { use common::*; use super::*; use hashdb::*; - use journaldb::traits::JournalDB; + use log::init_log; #[test] fn insert_same_in_fork() { // history is 1 - let mut jdb = OptionOneDB::new_temp(); + let mut jdb = JournalDB::new_temp(); let x = jdb.insert(b"X"); jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.commit(3, &b"1002a".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.commit(4, &b"1003a".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.remove(&x); jdb.commit(3, &b"1002b".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); let x = jdb.insert(b"X"); jdb.commit(4, &b"1003b".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.commit(5, &b"1004a".sha3(), Some((3, b"1002a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.commit(6, &b"1005a".sha3(), Some((4, b"1003a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&x)); } @@ -410,29 +554,35 @@ mod tests { #[test] fn long_history() { // history is 3 - let mut jdb = OptionOneDB::new_temp(); + let mut jdb = JournalDB::new_temp(); let h = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&h)); jdb.remove(&h); jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&h)); jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&h)); jdb.commit(3, &b"3".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&h)); jdb.commit(4, &b"4".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(!jdb.exists(&h)); } #[test] fn complex() { // history is 1 - let mut jdb = OptionOneDB::new_temp(); + let mut jdb = JournalDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); @@ -440,6 +590,7 @@ mod tests { jdb.remove(&bar); let baz = jdb.insert(b"baz"); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); assert!(jdb.exists(&baz)); @@ -447,17 +598,20 @@ mod tests { let foo = jdb.insert(b"foo"); jdb.remove(&baz); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&bar)); assert!(jdb.exists(&baz)); jdb.remove(&foo); jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&bar)); assert!(!jdb.exists(&baz)); jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(!jdb.exists(&foo)); assert!(!jdb.exists(&bar)); assert!(!jdb.exists(&baz)); @@ -466,26 +620,30 @@ mod tests { #[test] fn fork() { // history is 1 - let mut jdb = OptionOneDB::new_temp(); + let mut jdb = JournalDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); jdb.remove(&foo); let baz = jdb.insert(b"baz"); jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); assert!(jdb.exists(&baz)); jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&baz)); assert!(!jdb.exists(&bar)); @@ -494,39 +652,117 @@ mod tests { #[test] fn overwrite() { // history is 1 - let mut jdb = OptionOneDB::new_temp(); + let mut jdb = JournalDB::new_temp(); let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); assert!(jdb.exists(&foo)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); jdb.commit(3, &b"2".sha3(), Some((0, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); } #[test] - fn fork_same_key() { - // history is 1 - let mut jdb = OptionOneDB::new_temp(); + fn fork_same_key_one() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); let foo = jdb.insert(b"foo"); jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.insert(b"foo"); jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); jdb.commit(2, &b"2a".sha3(), Some((1, b"1a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); } + #[test] + fn fork_same_key_other() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + let foo = jdb.insert(b"foo"); + jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(1, &b"1c".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + assert!(jdb.exists(&foo)); + + jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); + } + + #[test] + fn fork_ins_del_ins() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + let foo = jdb.insert(b"foo"); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(2, &b"2a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(2, &b"2b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(3, &b"3a".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(3, &b"3b".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.commit(4, &b"4a".sha3(), Some((2, b"2a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.commit(5, &b"5a".sha3(), Some((3, b"3a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + } #[test] fn reopen() { @@ -535,81 +771,257 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); foo }; { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); } { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(!jdb.exists(&foo)); } } #[test] - fn reopen_remove() { + fn insert_delete_insert_delete_insert_expunge() { + init_log(); let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let foo = { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + + // history is 4 + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.remove(&foo); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.insert(b"foo"); + jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.remove(&foo); + jdb.commit(3, &b"3".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.insert(b"foo"); + jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + // expunge foo + jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + } + + #[test] + fn forked_insert_delete_insert_delete_insert_expunge() { + init_log(); + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + + // history is 4 + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(1, &b"1a".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(1, &b"1b".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(2, &b"2a".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(2, &b"2b".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(3, &b"3a".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&foo); + jdb.commit(3, &b"3b".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(4, &b"4a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(4, &b"4b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + // expunge foo + jdb.commit(5, &b"5".sha3(), Some((1, b"1a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + } + + #[test] + fn broken_assert() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + // history is 1 + let foo = jdb.insert(b"foo"); + jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + // foo is ancient history. + + jdb.remove(&foo); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.insert(b"foo"); + jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); // BROKEN + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); + + jdb.remove(&foo); + jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(!jdb.exists(&foo)); + } + + #[test] + fn reopen_test() { + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let mut jdb = JournalDB::new(dir.to_str().unwrap()); + // history is 4 + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(2, &b"2".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(3, &b"3".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(4, &b"4".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + // foo is ancient history. + + jdb.insert(b"foo"); + let bar = jdb.insert(b"bar"); + jdb.commit(5, &b"5".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.remove(&foo); + jdb.remove(&bar); + jdb.commit(6, &b"6".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.insert(b"foo"); + jdb.insert(b"bar"); + jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + } + + #[test] + fn reopen_remove_three() { + init_log(); + + let mut dir = ::std::env::temp_dir(); + dir.push(H32::random().hex()); + + let foo = b"foo".sha3(); + + { + let mut jdb = JournalDB::new(dir.to_str().unwrap()); // history is 1 - let foo = jdb.insert(b"foo"); + jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); - jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(1, &b"1".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); // foo is ancient history. - jdb.insert(b"foo"); - jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); - foo - }; - - { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); - jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); + jdb.commit(2, &b"2".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); + + jdb.insert(b"foo"); + jdb.commit(3, &b"3".sha3(), Some((1, b"1".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); + + // incantation to reopen the db + }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + jdb.remove(&foo); - jdb.commit(4, &b"4".sha3(), Some((3, b"3".sha3()))).unwrap(); - jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); + jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); + + // incantation to reopen the db + }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + + jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + assert!(jdb.exists(&foo)); + + // incantation to reopen the db + }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + + jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(!jdb.exists(&foo)); } } + #[test] fn reopen_fork() { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); jdb.commit(0, &b"0".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.remove(&foo); let baz = jdb.insert(b"baz"); jdb.commit(1, &b"1a".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); jdb.remove(&bar); jdb.commit(1, &b"1b".sha3(), Some((0, b"0".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); (foo, bar, baz) }; { - let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&baz)); assert!(!jdb.exists(&bar)); From e6a273f3a796f3353985fdf34a32d26838e3c5a9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 12 Mar 2016 09:53:54 +0100 Subject: [PATCH 03/26] Fix tests. --- util/src/journaldb/optiononedb.rs | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/util/src/journaldb/optiononedb.rs b/util/src/journaldb/optiononedb.rs index b51d0819d..567f620b1 100644 --- a/util/src/journaldb/optiononedb.rs +++ b/util/src/journaldb/optiononedb.rs @@ -524,7 +524,7 @@ mod tests { #[test] fn insert_same_in_fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let x = jdb.insert(b"X"); jdb.commit(1, &b"1".sha3(), None).unwrap(); @@ -554,7 +554,7 @@ mod tests { #[test] fn long_history() { // history is 3 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let h = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -577,7 +577,7 @@ mod tests { #[test] fn complex() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -620,7 +620,7 @@ mod tests { #[test] fn fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -652,7 +652,7 @@ mod tests { #[test] fn overwrite() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -677,7 +677,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -705,7 +705,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -733,7 +733,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -771,7 +771,7 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); @@ -781,14 +781,14 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); @@ -803,7 +803,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 4 let foo = jdb.insert(b"foo"); @@ -832,7 +832,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 4 let foo = jdb.insert(b"foo"); @@ -881,7 +881,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); @@ -912,7 +912,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 4 let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -952,7 +952,7 @@ mod tests { let foo = b"foo".sha3(); { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -973,7 +973,7 @@ mod tests { assert!(jdb.exists(&foo)); // incantation to reopen the db - }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + }; { let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap(); @@ -981,14 +981,14 @@ mod tests { assert!(jdb.exists(&foo)); // incantation to reopen the db - }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + }; { let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); // incantation to reopen the db - }; { let mut jdb = JournalDB::new(dir.to_str().unwrap()); + }; { let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -1001,7 +1001,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -1019,7 +1019,7 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); assert!(jdb.exists(&foo)); From 874393ba06dbce3c91ab59afb3b110c37c1b1896 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 12 Mar 2016 09:57:57 +0100 Subject: [PATCH 04/26] Fix tests, --- util/src/journaldb/archivedb.rs | 2 +- util/src/journaldb/optiononedb.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index e7da1b737..a8b9c1f74 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -364,7 +364,7 @@ mod tests { fn reopen_fork() { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let (foo, bar, baz) = { + let (foo, _, _) = { let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); diff --git a/util/src/journaldb/optiononedb.rs b/util/src/journaldb/optiononedb.rs index 567f620b1..e44c337db 100644 --- a/util/src/journaldb/optiononedb.rs +++ b/util/src/journaldb/optiononedb.rs @@ -518,6 +518,7 @@ impl JournalDB for OptionOneDB { mod tests { use common::*; use super::*; + use super::super::traits::JournalDB; use hashdb::*; use log::init_log; From e10457d2352bc8a18c266f79d2ab224d62821836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 12 Mar 2016 09:59:57 +0100 Subject: [PATCH 05/26] Bumping clippy --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- ethcore/Cargo.toml | 2 +- rpc/Cargo.toml | 2 +- sync/Cargo.toml | 2 +- util/Cargo.toml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f26ac923..ce508c937 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,7 +2,7 @@ name = "parity" version = "0.9.99" dependencies = [ - "clippy 0.0.49 (registry+https://github.com/rust-lang/crates.io-index)", + "clippy 0.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "ctrlc 1.1.1 (git+https://github.com/tomusdrw/rust-ctrlc.git)", "daemonize 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 0.6.78 (registry+https://github.com/rust-lang/crates.io-index)", @@ -94,7 +94,7 @@ dependencies = [ [[package]] name = "clippy" -version = "0.0.49" +version = "0.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "regex-syntax 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -207,7 +207,7 @@ dependencies = [ name = "ethcore" version = "0.9.99" dependencies = [ - "clippy 0.0.49 (registry+https://github.com/rust-lang/crates.io-index)", + "clippy 0.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethash 0.9.99", @@ -233,7 +233,7 @@ dependencies = [ name = "ethcore-rpc" version = "0.9.99" dependencies = [ - "clippy 0.0.49 (registry+https://github.com/rust-lang/crates.io-index)", + "clippy 0.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "ethash 0.9.99", "ethcore 0.9.99", "ethcore-util 0.9.99", @@ -256,7 +256,7 @@ dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "bigint 0.1.0", "chrono 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)", - "clippy 0.0.49 (registry+https://github.com/rust-lang/crates.io-index)", + "clippy 0.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -288,7 +288,7 @@ dependencies = [ name = "ethsync" version = "0.9.99" dependencies = [ - "clippy 0.0.49 (registry+https://github.com/rust-lang/crates.io-index)", + "clippy 0.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", "ethcore-util 0.9.99", diff --git a/Cargo.toml b/Cargo.toml index 77d1e57ae..efe794d5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ ctrlc = { git = "https://github.com/tomusdrw/rust-ctrlc.git" } fdlimit = { path = "util/fdlimit" } daemonize = "0.2" number_prefix = "0.2" -clippy = { version = "0.0.49", optional = true } +clippy = { version = "0.0.50", optional = true } ethcore = { path = "ethcore" } ethcore-util = { path = "util" } ethsync = { path = "sync" } diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 5ef83842f..be4212f5d 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -17,7 +17,7 @@ ethcore-util = { path = "../util" } evmjit = { path = "../evmjit", optional = true } ethash = { path = "../ethash" } num_cpus = "0.2" -clippy = { version = "0.0.49", optional = true } +clippy = { version = "0.0.50", optional = true } crossbeam = "0.1.5" lazy_static = "0.1" ethcore-devtools = { path = "../devtools" } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index a1f154ca8..900b10548 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -18,7 +18,7 @@ ethcore-util = { path = "../util" } ethcore = { path = "../ethcore" } ethash = { path = "../ethash" } ethsync = { path = "../sync" } -clippy = { version = "0.0.49", optional = true } +clippy = { version = "0.0.50", optional = true } rustc-serialize = "0.3" transient-hashmap = "0.1" serde_macros = { version = "0.7.0", optional = true } diff --git a/sync/Cargo.toml b/sync/Cargo.toml index cf0027368..6022beb9c 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -10,7 +10,7 @@ authors = ["Ethcore Date: Sat, 12 Mar 2016 10:07:55 +0100 Subject: [PATCH 06/26] Fixing warnings --- ethcore/src/client/test_client.rs | 8 +++++++- util/src/keys/store.rs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 207f1090f..9b311081c 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -55,9 +55,15 @@ pub enum EachBlockWith { UncleAndTransaction } +impl Default for TestBlockChainClient { + fn default() -> Self { + TestBlockChainClient::new() + } +} + impl TestBlockChainClient { /// Creates new test client. - pub fn new() -> TestBlockChainClient { + pub fn new() -> Self { let mut client = TestBlockChainClient { blocks: RwLock::new(HashMap::new()), diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 6a5efc87d..37786be8b 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -120,9 +120,15 @@ impl AccountProvider for AccountService { } } +impl Default for AccountService { + fn default() -> Self { + AccountService::new() + } +} + impl AccountService { /// New account service with the default location - pub fn new() -> AccountService { + pub fn new() -> Self { let secret_store = RwLock::new(SecretStore::new()); secret_store.write().unwrap().try_import_existing(); AccountService { From 4b6e1dd4d2e4e62efa92db028cb77bfbdbfb51c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 12 Mar 2016 10:07:55 +0100 Subject: [PATCH 07/26] Fixing warnings --- ethcore/src/block_queue.rs | 2 +- ethcore/src/chainfilter/tests.rs | 8 +++++- ethcore/src/client/test_client.rs | 8 +++++- ethcore/src/externalities.rs | 8 +++++- ethcore/src/verification/verification.rs | 8 +++++- parity/main.rs | 17 +++++++----- util/src/journaldb/archivedb.rs | 2 +- util/src/keys/store.rs | 10 +++++-- util/src/network/connection.rs | 34 ++++++++++++++++++------ 9 files changed, 74 insertions(+), 23 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 50db23dfe..042df1dc1 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -523,7 +523,7 @@ mod tests { let engine = spec.to_engine().unwrap(); let mut config = BlockQueueConfig::default(); config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 - let mut queue = BlockQueue::new(config, Arc::new(engine), IoChannel::disconnected()); + let queue = BlockQueue::new(config, Arc::new(engine), IoChannel::disconnected()); assert!(!queue.queue_info().is_full()); let mut blocks = get_good_dummy_block_seq(50); for b in blocks.drain(..) { diff --git a/ethcore/src/chainfilter/tests.rs b/ethcore/src/chainfilter/tests.rs index 08af44720..7dac29f11 100644 --- a/ethcore/src/chainfilter/tests.rs +++ b/ethcore/src/chainfilter/tests.rs @@ -28,9 +28,15 @@ pub struct MemoryCache { blooms: HashMap, } +impl Default for MemoryCache { + fn default() -> Self { + MemoryCache::new() + } +} + impl MemoryCache { /// Default constructor for MemoryCache - pub fn new() -> MemoryCache { + pub fn new() -> Self { MemoryCache { blooms: HashMap::new() } } diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 207f1090f..9b311081c 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -55,9 +55,15 @@ pub enum EachBlockWith { UncleAndTransaction } +impl Default for TestBlockChainClient { + fn default() -> Self { + TestBlockChainClient::new() + } +} + impl TestBlockChainClient { /// Creates new test client. - pub fn new() -> TestBlockChainClient { + pub fn new() -> Self { let mut client = TestBlockChainClient { blocks: RwLock::new(HashMap::new()), diff --git a/ethcore/src/externalities.rs b/ethcore/src/externalities.rs index 598921580..d37bc20fb 100644 --- a/ethcore/src/externalities.rs +++ b/ethcore/src/externalities.rs @@ -301,8 +301,14 @@ mod tests { env_info: EnvInfo } + impl Default for TestSetup { + fn default() -> Self { + TestSetup::new() + } + } + impl TestSetup { - fn new() -> TestSetup { + fn new() -> Self { TestSetup { state: get_temp_state(), engine: get_test_spec().to_engine().unwrap(), diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index ed3db3791..60cbed56c 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -255,8 +255,14 @@ mod tests { numbers: HashMap, } + impl Default for TestBlockChain { + fn default() -> Self { + TestBlockChain::new() + } + } + impl TestBlockChain { - pub fn new() -> TestBlockChain { + pub fn new() -> Self { TestBlockChain { blocks: HashMap::new(), numbers: HashMap::new(), diff --git a/parity/main.rs b/parity/main.rs index 840921dce..85e29a0ae 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -315,7 +315,7 @@ impl Configuration { fn init_nodes(&self, spec: &Spec) -> Vec { let mut r = if self.args.flag_no_bootstrap { Vec::new() } else { spec.nodes().clone() }; if let Some(ref x) = self.args.flag_bootnodes { - r.extend(x.split(",").map(|s| Self::normalize_enode(s).unwrap_or_else(|| die!("{}: Invalid node address format given for a boot node.", s)))); + r.extend(x.split(',').map(|s| Self::normalize_enode(s).unwrap_or_else(|| die!("{}: Invalid node address format given for a boot node.", s)))); } r } @@ -328,7 +328,7 @@ impl Configuration { let host = IpAddr::from_str(host).unwrap_or_else(|_| die!("Invalid host given with `--nat extip:{}`", host)); Some(SocketAddr::new(host, self.args.flag_port)) } else { - listen_address.clone() + listen_address }; (listen_address, public_address) } @@ -389,7 +389,7 @@ impl Configuration { } if self.args.cmd_list { println!("Known addresses:"); - for &(addr, _) in secret_store.accounts().unwrap().iter() { + for &(addr, _) in &secret_store.accounts().unwrap() { println!("{:?}", addr); } } @@ -407,7 +407,11 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); let mut sync_config = SyncConfig::default(); - sync_config.network_id = self.args.flag_networkid.as_ref().map(|id| U256::from_str(id).unwrap_or_else(|_| die!("{}: Invalid index given with --networkid", id))).unwrap_or(spec.network_id()); + sync_config.network_id = self.args.flag_networkid.as_ref().map_or(spec.network_id(), |id| { + U256::from_str(id).unwrap_or_else(|_| { + die!("{}: Invalid index given with --networkid", id) + }) + }); // Build client let mut client_config = ClientConfig::default(); @@ -422,8 +426,7 @@ impl Configuration { } } client_config.pruning = match self.args.flag_pruning.as_str() { - "" => journaldb::Algorithm::Archive, - "archive" => journaldb::Algorithm::Archive, + "" | "archive" => journaldb::Algorithm::Archive, "pruned" => journaldb::Algorithm::EarlyMerge, // "fast" => journaldb::Algorithm::OverlayRecent, // TODO: @arkpar uncomment this once option 2 is merged. // "slow" => journaldb::Algorithm::RefCounted, // TODO: @gavofyork uncomment this once ref-count algo is merged. @@ -453,7 +456,7 @@ impl Configuration { let cors = self.args.flag_rpccorsdomain.as_ref().unwrap_or(&self.args.flag_jsonrpc_cors); // TODO: use this as the API list. let apis = self.args.flag_rpcapi.as_ref().unwrap_or(&self.args.flag_jsonrpc_apis); - let server_handler = setup_rpc_server(service.client(), sync.clone(), account_service.clone(), &url, cors, apis.split(",").collect()); + let server_handler = setup_rpc_server(service.client(), sync.clone(), account_service.clone(), &url, cors, apis.split(',').collect()); if let Some(handler) = server_handler { panic_handler.forward_from(handler.deref()); } diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index e7da1b737..f92058f92 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -364,7 +364,7 @@ mod tests { fn reopen_fork() { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); - let (foo, bar, baz) = { + let (foo, _bar, _baz) = { let mut jdb = ArchiveDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 6a5efc87d..d514863bb 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -120,9 +120,15 @@ impl AccountProvider for AccountService { } } +impl Default for AccountService { + fn default() -> Self { + AccountService::new() + } +} + impl AccountService { /// New account service with the default location - pub fn new() -> AccountService { + pub fn new() -> Self { let secret_store = RwLock::new(SecretStore::new()); secret_store.write().unwrap().try_import_existing(); AccountService { @@ -568,7 +574,7 @@ mod tests { let temp = RandomTempPath::create_dir(); let mut sstore = SecretStore::new_test(&temp); let addr = sstore.new_account("test").unwrap(); - let _ok = sstore.unlock_account(&addr, "test").unwrap(); + sstore.unlock_account(&addr, "test").unwrap(); let secret = sstore.account_secret(&addr).unwrap(); let kp = KeyPair::from_secret(secret).unwrap(); assert_eq!(Address::from(kp.public().sha3()), addr); diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index fe65be6d1..a560c1a91 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -160,12 +160,12 @@ impl Connection { } } - /// Get socket token + /// Get socket token pub fn token(&self) -> StreamToken { self.token } - /// Replace socket token + /// Replace socket token pub fn set_token(&mut self, token: StreamToken) { self.token = token; } @@ -261,13 +261,13 @@ pub struct EncryptedConnection { } impl EncryptedConnection { - - /// Get socket token + + /// Get socket token pub fn token(&self) -> StreamToken { self.connection.token } - /// Replace socket token + /// Replace socket token pub fn set_token(&mut self, token: StreamToken) { self.connection.set_token(token); } @@ -513,8 +513,14 @@ mod tests { buf_size: usize, } + impl Default for TestSocket { + fn default() -> Self { + TestSocket::new() + } + } + impl TestSocket { - fn new() -> TestSocket { + fn new() -> Self { TestSocket { read_buffer: vec![], write_buffer: vec![], @@ -593,8 +599,14 @@ mod tests { type TestConnection = GenericConnection; + impl Default for TestConnection { + fn default() -> Self { + TestConnection::new() + } + } + impl TestConnection { - pub fn new() -> TestConnection { + pub fn new() -> Self { TestConnection { token: 999998888usize, socket: TestSocket::new(), @@ -609,8 +621,14 @@ mod tests { type TestBrokenConnection = GenericConnection; + impl Default for TestBrokenConnection { + fn default() -> Self { + TestBrokenConnection::new() + } + } + impl TestBrokenConnection { - pub fn new() -> TestBrokenConnection { + pub fn new() -> Self { TestBrokenConnection { token: 999998888usize, socket: TestBrokenSocket { error: "test broken socket".to_owned() }, From c6dc6c0c41c45ee4279952120fb4112c55cd9932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 12 Mar 2016 10:41:35 +0100 Subject: [PATCH 08/26] One more warning --- util/src/journaldb/overlay.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/util/src/journaldb/overlay.rs b/util/src/journaldb/overlay.rs index e91709041..76eff9aa1 100644 --- a/util/src/journaldb/overlay.rs +++ b/util/src/journaldb/overlay.rs @@ -33,14 +33,14 @@ use super::JournalDB; /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before /// the removals actually take effect. /// -/// There are two memory overlays: -/// - Transaction overlay contains current transaction data. It is merged with with history +/// There are two memory overlays: +/// - Transaction overlay contains current transaction data. It is merged with with history /// overlay on each `commit()` -/// - History overlay contains all data inserted during the history period. When the node +/// - History overlay contains all data inserted during the history period. When the node /// in the overlay becomes ancient it is written to disk on `commit()` /// -/// There is also a journal maintained in memory and on the disk as well which lists insertions -/// and removals for each commit during the history period. This is used to track +/// There is also a journal maintained in memory and on the disk as well which lists insertions +/// and removals for each commit during the history period. This is used to track /// data nodes that go out of history scope and must be written to disk. /// /// Commit workflow: @@ -50,12 +50,12 @@ use super::JournalDB; /// 3. Clear the transaction overlay. /// 4. For a canonical journal record that becomes ancient inserts its insertions into the disk DB /// 5. For each journal record that goes out of the history scope (becomes ancient) remove its -/// insertions from the history overlay, decreasing the reference counter and removing entry if +/// insertions from the history overlay, decreasing the reference counter and removing entry if /// if reaches zero. -/// 6. For a canonical journal record that becomes ancient delete its removals from the disk only if +/// 6. For a canonical journal record that becomes ancient delete its removals from the disk only if /// the removed key is not present in the history overlay. /// 7. Delete ancient record from memory and disk. -/// +/// pub struct JournalOverlayDB { transaction_overlay: MemoryDB, backing: Arc, @@ -223,7 +223,7 @@ impl JournalDB for JournalOverlayDB { let mut tx = self.transaction_overlay.drain(); let inserted_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c > 0 { Some(k.clone()) } else { None }).collect(); let removed_keys: Vec<_> = tx.iter().filter_map(|(k, &(_, c))| if c < 0 { Some(k.clone()) } else { None }).collect(); - // Increase counter for each inserted key no matter if the block is canonical or not. + // Increase counter for each inserted key no matter if the block is canonical or not. let insertions = tx.drain().filter_map(|(k, (v, c))| if c > 0 { Some((k, v)) } else { None }); r.append(id); r.begin_list(inserted_keys.len()); @@ -236,7 +236,7 @@ impl JournalDB for JournalOverlayDB { r.append(&removed_keys); let mut k = RlpStream::new_list(3); - let index = journal_overlay.journal.get(&now).map(|j| j.len()).unwrap_or(0); + let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len()); k.append(&now); k.append(&index); k.append(&&PADDING[..]); @@ -345,14 +345,14 @@ impl HashDB for JournalOverlayDB { self.lookup(key).is_some() } - fn insert(&mut self, value: &[u8]) -> H256 { + fn insert(&mut self, value: &[u8]) -> H256 { self.transaction_overlay.insert(value) } fn emplace(&mut self, key: H256, value: Bytes) { - self.transaction_overlay.emplace(key, value); + self.transaction_overlay.emplace(key, value); } - fn kill(&mut self, key: &H256) { - self.transaction_overlay.kill(key); + fn kill(&mut self, key: &H256) { + self.transaction_overlay.kill(key); } } @@ -749,7 +749,7 @@ mod tests { assert!(jdb.can_reconstruct_refs()); assert!(!jdb.exists(&foo)); } - + #[test] fn reopen_test() { let mut dir = ::std::env::temp_dir(); @@ -784,7 +784,7 @@ mod tests { jdb.commit(7, &b"7".sha3(), Some((3, b"3".sha3()))).unwrap(); assert!(jdb.can_reconstruct_refs()); } - + #[test] fn reopen_remove_three() { init_log(); @@ -838,7 +838,7 @@ mod tests { assert!(!jdb.exists(&foo)); } } - + #[test] fn reopen_fork() { let mut dir = ::std::env::temp_dir(); From d7039b72e237fb739dc98e344aa06ebf8cd49028 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 12 Mar 2016 10:48:28 +0100 Subject: [PATCH 09/26] Update archivedb.rs --- util/src/journaldb/archivedb.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index c8c29e765..8db239cff 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -305,7 +305,6 @@ mod tests { assert!(jdb.exists(&foo)); } - #[test] fn reopen() { let mut dir = ::std::env::temp_dir(); @@ -364,6 +363,7 @@ mod tests { jdb.commit(5, &b"5".sha3(), Some((4, b"4".sha3()))).unwrap(); } } + #[test] fn reopen_fork() { let mut dir = ::std::env::temp_dir(); @@ -406,6 +406,5 @@ mod tests { let state = jdb.state(&key); assert!(state.is_some()); } - } } From 19b6c74675cbf96cb0118cb13e5279860d182876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 12 Mar 2016 13:39:17 +0100 Subject: [PATCH 10/26] Two more warnings --- ethcore/src/verification/noop_verifier.rs | 1 + parity/main.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index ae2a153fe..20c15c3f1 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -20,6 +20,7 @@ use error::Error; use header::Header; use super::Verifier; +#[allow(dead_code)] pub struct NoopVerifier; impl Verifier for NoopVerifier { diff --git a/parity/main.rs b/parity/main.rs index 3770fe04d..b16801ad5 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -394,6 +394,7 @@ impl Configuration { } } + #[cfg_attr(feature="dev", allow(useless_format))] fn execute_client(&self) { // Setup panic handler let panic_handler = PanicHandler::new_in_arc(); From 451a5d78e35da7da15c44945375083322fe6b013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 12 Mar 2016 13:40:39 +0100 Subject: [PATCH 11/26] Removing unused (?) serde::Error --- rpc/src/v1/types/transaction.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index 0518a58ea..232cf0bf3 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -17,7 +17,6 @@ use util::numbers::*; use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; -use serde::Error; #[derive(Debug, Default, Serialize)] pub struct Transaction { From 7ad7996144cc3f3cbf982f0d78af25be2f1c2846 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 11 Mar 2016 18:50:29 +0100 Subject: [PATCH 12/26] Fixed common block detection --- sync/src/chain.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 4a2d941a7..5ba2e8773 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -377,10 +377,14 @@ impl ChainSync { let hash = info.hash(); match io.chain().block_status(BlockId::Hash(hash.clone())) { BlockStatus::InChain => { - self.have_common_block = true; - self.last_imported_block = Some(number); - self.last_imported_hash = Some(hash.clone()); - trace!(target: "sync", "Found common header {} ({})", number, hash); + if !self.have_common_block { + self.have_common_block = true; + self.last_imported_block = Some(number); + self.last_imported_hash = Some(hash.clone()); + trace!(target: "sync", "Found common header {} ({})", number, hash); + } else { + trace!(target: "sync", "Header already in chain {} ({})", number, hash); + } }, _ => { if self.have_common_block { From 1fe575bf7bbdf276c1f3f8242d278fc1a573a099 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 11 Mar 2016 20:17:36 +0100 Subject: [PATCH 13/26] Download bodies for validated hash chain only --- sync/src/chain.rs | 19 +++---------------- sync/src/range_collection.rs | 11 ++++++++--- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 5ba2e8773..657ef6a78 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -660,10 +660,7 @@ impl ChainSync { let mut needed_numbers: Vec = Vec::new(); if self.have_common_block && !self.headers.is_empty() && self.headers.range_iter().next().unwrap().0 == self.current_base_block() + 1 { - for (start, ref items) in self.headers.range_iter() { - if needed_bodies.len() >= MAX_BODIES_TO_REQUEST { - break; - } + if let Some((start, ref items)) = self.headers.range_iter().next() { let mut index: BlockNumber = 0; while index != items.len() as BlockNumber && needed_bodies.len() < MAX_BODIES_TO_REQUEST { let block = start + index; @@ -848,18 +845,8 @@ impl ChainSync { /// Remove downloaded bocks/headers starting from specified number. /// Used to recover from an error and re-download parts of the chain detected as bad. fn remove_downloaded_blocks(&mut self, start: BlockNumber) { - for n in self.headers.get_tail(&start) { - if let Some(ref header_data) = self.headers.find_item(&n) { - let header_to_delete = HeaderView::new(&header_data.data); - let header_id = HeaderId { - transactions_root: header_to_delete.transactions_root(), - uncles: header_to_delete.uncles_hash() - }; - self.header_ids.remove(&header_id); - } - self.downloading_bodies.remove(&n); - self.downloading_headers.remove(&n); - } + self.downloading_bodies.clear(); + self.downloading_headers.clear(); self.headers.remove_from(&start); self.bodies.remove_from(&start); } diff --git a/sync/src/range_collection.rs b/sync/src/range_collection.rs index 664d7c7a3..0628df401 100644 --- a/sync/src/range_collection.rs +++ b/sync/src/range_collection.rs @@ -300,12 +300,17 @@ fn test_range() { let mut r = ranges.clone(); r.remove_from(&20); assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); - r.remove_from(&17); - assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p'][..])]), Ordering::Equal); - r.remove_from(&15); + r.remove_from(&18); + assert!(!r.have_item(&18)); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q'][..])]), Ordering::Equal); + r.remove_from(&16); + assert!(!r.have_item(&16)); assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..])]), Ordering::Equal); r.remove_from(&3); assert_eq!(r.range_iter().cmp(vec![(2, &['b'][..])]), Ordering::Equal); + r.remove_from(&1); + assert_eq!(r.range_iter().next(), None); + let mut r = ranges.clone(); r.remove_from(&2); assert_eq!(r.range_iter().next(), None); } From 16618094f5211ad26d83e2ef2c7cb57420c48388 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 11 Mar 2016 21:23:36 +0100 Subject: [PATCH 14/26] Cleanup header_ids --- sync/src/chain.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 657ef6a78..a30db0423 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -377,10 +377,12 @@ impl ChainSync { let hash = info.hash(); match io.chain().block_status(BlockId::Hash(hash.clone())) { BlockStatus::InChain => { - if !self.have_common_block { - self.have_common_block = true; + if self.current_base_block() < number { self.last_imported_block = Some(number); self.last_imported_hash = Some(hash.clone()); + } + if !self.have_common_block { + self.have_common_block = true; trace!(target: "sync", "Found common header {} ({})", number, hash); } else { trace!(target: "sync", "Header already in chain {} ({})", number, hash); @@ -845,8 +847,12 @@ impl ChainSync { /// Remove downloaded bocks/headers starting from specified number. /// Used to recover from an error and re-download parts of the chain detected as bad. fn remove_downloaded_blocks(&mut self, start: BlockNumber) { - self.downloading_bodies.clear(); - self.downloading_headers.clear(); + let ids = self.header_ids.drain().filter(|&(_, v)| v < start).collect(); + self.header_ids = ids; + let hdrs = self.downloading_headers.drain().filter(|v| *v < start).collect(); + self.downloading_headers = hdrs; + let bodies = self.downloading_bodies.drain().filter(|v| *v < start).collect(); + self.downloading_bodies = bodies; self.headers.remove_from(&start); self.bodies.remove_from(&start); } From 1e23a4c888c9feab5c8343e1f2c49a953f560cb8 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Mar 2016 14:54:44 +0100 Subject: [PATCH 15/26] Don't redownload queued blocks on restart --- sync/src/chain.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index a30db0423..84ca5500a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -298,8 +298,6 @@ impl ChainSync { /// Restart sync pub fn restart(&mut self, io: &mut SyncIo) { self.reset(); - self.last_imported_block = None; - self.last_imported_hash = None; self.starting_block = 0; self.highest_block = None; self.have_common_block = false; @@ -366,7 +364,7 @@ impl ChainSync { for i in 0..item_count { let info: BlockHeader = try!(r.val_at(i)); let number = BlockNumber::from(info.number); - if number <= self.current_base_block() || self.headers.have_item(&number) { + if (number <= self.current_base_block() && self.have_common_block) || self.headers.have_item(&number) { trace!(target: "sync", "Skipping existing block header"); continue; } @@ -376,8 +374,8 @@ impl ChainSync { } let hash = info.hash(); match io.chain().block_status(BlockId::Hash(hash.clone())) { - BlockStatus::InChain => { - if self.current_base_block() < number { + BlockStatus::InChain | BlockStatus::Queued => { + if !self.have_common_block || self.current_base_block() < number { self.last_imported_block = Some(number); self.last_imported_hash = Some(hash.clone()); } @@ -706,7 +704,10 @@ impl ChainSync { if !self.have_common_block { // download backwards until common block is found 1 header at a time let chain_info = io.chain().chain_info(); - start = chain_info.best_block_number; + start = match self.last_imported_block { + Some(n) => n, + None => chain_info.best_block_number, + }; if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 - 1); } From 1ca7c35c19adffe3775d137698d8da67362e093f Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Mar 2016 17:30:46 +0100 Subject: [PATCH 16/26] Fix latest era marker --- util/src/journaldb/earlymergedb.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 83be31fb4..f411691d9 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -64,6 +64,7 @@ pub struct EarlyMergeDB { overlay: MemoryDB, backing: Arc, refs: Option>>>, + latest_era: u64, } // all keys must be at least 12 bytes @@ -90,11 +91,13 @@ impl EarlyMergeDB { backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); } - let refs = Some(Arc::new(RwLock::new(EarlyMergeDB::read_refs(&backing)))); + let (latest_era, refs) = EarlyMergeDB::read_refs(&backing); + let refs = Some(Arc::new(RwLock::new(refs))); EarlyMergeDB { overlay: MemoryDB::new(), backing: Arc::new(backing), refs: refs, + latest_era: latest_era, } } @@ -225,9 +228,9 @@ impl EarlyMergeDB { #[cfg(test)] fn can_reconstruct_refs(&self) -> bool { - let reconstructed = Self::read_refs(&self.backing); + let (latest_era, reconstructed) = Self::read_refs(&self.backing); let refs = self.refs.as_ref().unwrap().write().unwrap(); - if *refs != reconstructed { + if *refs != reconstructed || latest_era != self.latest_era { let clean_refs = refs.iter().filter_map(|(k, v)| if reconstructed.get(k) == Some(v) {None} else {Some((k.clone(), v.clone()))}).collect::>(); let clean_recon = reconstructed.into_iter().filter_map(|(k, v)| if refs.get(&k) == Some(&v) {None} else {Some((k.clone(), v.clone()))}).collect::>(); warn!(target: "jdb", "mem: {:?} != log: {:?}", clean_refs, clean_recon); @@ -241,10 +244,12 @@ impl EarlyMergeDB { self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_refs(db: &Database) -> HashMap { + fn read_refs(db: &Database) -> (u64, HashMap) { let mut refs = HashMap::new(); + let mut latest_era = 0u64; if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { - let mut era = decode::(&val); + latest_era = decode::(&val); + let mut era = latest_era; loop { let mut index = 0usize; while let Some(rlp_data) = db.get({ @@ -265,7 +270,7 @@ impl EarlyMergeDB { era -= 1; } } - refs + (latest_era, refs) } } @@ -320,6 +325,7 @@ impl JournalDB for EarlyMergeDB { overlay: MemoryDB::new(), backing: self.backing.clone(), refs: self.refs.clone(), + latest_era: self.latest_era, }) } @@ -435,7 +441,10 @@ impl JournalDB for EarlyMergeDB { trace!(target: "jdb.ops", " Deletes: {:?}", removes); } try!(batch.put(&last, r.as_raw())); - try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + if now >= self.latest_era { + try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); + self.latest_era = now; + } } // apply old commits' details From 0b4355d549d30a03287543d95b2a482ccbe1687b Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Mar 2016 19:03:33 +0100 Subject: [PATCH 17/26] rocksdb version bump --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e3a4b168..1c58cd431 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.4.2 (git+https://github.com/arkpar/rust-rocksdb.git)", + "rocksdb 0.4.3 (git+https://github.com/arkpar/rust-rocksdb.git)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -466,8 +466,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "librocksdb-sys" -version = "0.2.2" -source = "git+https://github.com/arkpar/rust-rocksdb.git#a4f89fea20ee3ae92b692df65d56426a5c0b6fd5" +version = "0.2.3" +source = "git+https://github.com/arkpar/rust-rocksdb.git#ebb602fc74b4067f9f51310bdc0401b8e59b7156" dependencies = [ "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -688,11 +688,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" -version = "0.4.2" -source = "git+https://github.com/arkpar/rust-rocksdb.git#a4f89fea20ee3ae92b692df65d56426a5c0b6fd5" +version = "0.4.3" +source = "git+https://github.com/arkpar/rust-rocksdb.git#ebb602fc74b4067f9f51310bdc0401b8e59b7156" dependencies = [ "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "librocksdb-sys 0.2.2 (git+https://github.com/arkpar/rust-rocksdb.git)", + "librocksdb-sys 0.2.3 (git+https://github.com/arkpar/rust-rocksdb.git)", ] [[package]] From 707f67c6b4b588e43af86adad2f64d9299d8e3e8 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Mar 2016 19:19:45 +0100 Subject: [PATCH 18/26] Optional last era --- util/src/journaldb/earlymergedb.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index f411691d9..0931d42d1 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -64,7 +64,7 @@ pub struct EarlyMergeDB { overlay: MemoryDB, backing: Arc, refs: Option>>>, - latest_era: u64, + latest_era: Option, } // all keys must be at least 12 bytes @@ -244,12 +244,12 @@ impl EarlyMergeDB { self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_refs(db: &Database) -> (u64, HashMap) { + fn read_refs(db: &Database) -> (Option, HashMap) { let mut refs = HashMap::new(); - let mut latest_era = 0u64; + let mut latest_era = None; if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { - latest_era = decode::(&val); - let mut era = latest_era; + let mut era = decode::(&val); + latest_era = Some(era); loop { let mut index = 0usize; while let Some(rlp_data) = db.get({ @@ -325,7 +325,7 @@ impl JournalDB for EarlyMergeDB { overlay: MemoryDB::new(), backing: self.backing.clone(), refs: self.refs.clone(), - latest_era: self.latest_era, + latest_era: self.latest_era.clone(), }) } @@ -441,9 +441,9 @@ impl JournalDB for EarlyMergeDB { trace!(target: "jdb.ops", " Deletes: {:?}", removes); } try!(batch.put(&last, r.as_raw())); - if now >= self.latest_era { + if self.latest_era.map_or(true, |e| now > e) { try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); - self.latest_era = now; + self.latest_era = Some(now); } } From a9a1c80fac13732e9cd642822f768e2884663aac Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Mar 2016 19:21:08 +0100 Subject: [PATCH 19/26] implemented eth_accounts, fixed personal_accounts, added test account provider, tests for eth_accounts --- rpc/src/v1/impls/eth.rs | 8 ++ rpc/src/v1/impls/personal.rs | 7 +- rpc/src/v1/tests/eth.rs | 59 ++++++++++++++ rpc/src/v1/tests/helpers/account_provider.rs | 84 ++++++++++++++++++++ rpc/src/v1/tests/helpers/mod.rs | 2 + rpc/src/v1/tests/mod.rs | 1 + 6 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 rpc/src/v1/tests/eth.rs create mode 100644 rpc/src/v1/tests/helpers/account_provider.rs diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 38e363624..47b5471b3 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -155,6 +155,14 @@ impl Eth for EthClient where C: BlockChainClient + 'static, S: } } + fn accounts(&self, _: Params) -> Result { + let store = take_weak!(self.accounts); + match store.accounts() { + Ok(account_list) => to_value(&account_list), + Err(_) => Err(Error::internal_error()) + } + } + fn block_number(&self, params: Params) -> Result { match params { Params::None => to_value(&U256::from(take_weak!(self.client).chain_info().best_block_number)), diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index ce200244c..0cd3f0040 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -39,12 +39,7 @@ impl Personal for PersonalClient where A: AccountProvider + 'static { fn accounts(&self, _: Params) -> Result { let store = take_weak!(self.accounts); match store.accounts() { - Ok(account_list) => { - Ok(Value::Array(account_list.iter() - .map(|&account| Value::String(format!("{:?}", account))) - .collect::>()) - ) - } + Ok(account_list) => to_value(&account_list), Err(_) => Err(Error::internal_error()) } } diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs new file mode 100644 index 000000000..c36a9d172 --- /dev/null +++ b/rpc/src/v1/tests/eth.rs @@ -0,0 +1,59 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::collections::HashMap; +use std::sync::Arc; +use jsonrpc_core::IoHandler; +use util::hash::{Address}; +use ethcore::client::{TestBlockChainClient, EachBlockWith}; +use v1::{Eth, EthClient}; +use v1::tests::helpers::{TestAccount, TestAccountProvider, TestSyncProvider, Config}; + +fn blockchain_client() -> Arc { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Nothing); + Arc::new(client) +} + +fn accounts_provider() -> Arc { + let mut accounts = HashMap::new(); + accounts.insert(Address::from(1), TestAccount::new("test")); + let ap = TestAccountProvider::new(accounts); + Arc::new(ap) +} + +fn sync_provider() -> Arc { + Arc::new(TestSyncProvider::new(Config { + protocol_version: 65, + num_peers: 120, + })) +} + +#[test] +fn rpc_eth_accounts() { + let client = blockchain_client(); + let sync = sync_provider(); + let ap = accounts_provider(); + + let eth = EthClient::new(&client, &sync, &ap).to_delegate(); + let io = IoHandler::new(); + io.add_delegate(eth); + + let request = r#"{"jsonrpc": "2.0", "method": "eth_accounts", "params": [], "id": 1}"#; + let response = r#"{"jsonrpc":"2.0","result":["0x0000000000000000000000000000000000000001"],"id":1}"#; + + assert_eq!(io.handle_request(request), Some(response.to_owned())); +} diff --git a/rpc/src/v1/tests/helpers/account_provider.rs b/rpc/src/v1/tests/helpers/account_provider.rs new file mode 100644 index 000000000..66f085f74 --- /dev/null +++ b/rpc/src/v1/tests/helpers/account_provider.rs @@ -0,0 +1,84 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::sync::RwLock; +use std::collections::HashMap; +use std::io; +use util::hash::{Address, H256}; +use util::crypto::{Secret, Signature}; +use util::keys::store::{AccountProvider, SigningError, EncryptedHashMapError}; + +/// Account mock. +#[derive(Clone)] +pub struct TestAccount { + /// True if account is unlocked. + pub unlocked: bool, + /// Account's password. + pub password: String, +} + +impl TestAccount { + pub fn new(password: &str) -> Self { + TestAccount { + unlocked: false, + password: password.to_owned(), + } + } +} + +/// Test account provider. +pub struct TestAccountProvider { + accounts: RwLock>, +} + +impl TestAccountProvider { + /// Basic constructor. + pub fn new(accounts: HashMap) -> Self { + TestAccountProvider { + accounts: RwLock::new(accounts), + } + } +} + +impl AccountProvider for TestAccountProvider { + fn accounts(&self) -> Result, io::Error> { + Ok(self.accounts.read().unwrap().keys().cloned().collect()) + } + + fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { + match self.accounts.write().unwrap().get_mut(account) { + Some(ref mut acc) if acc.password == pass => { + acc.unlocked = true; + Ok(()) + }, + Some(_) => Err(EncryptedHashMapError::InvalidPassword), + None => Err(EncryptedHashMapError::UnknownIdentifier), + } + } + + fn new_account(&self, _pass: &str) -> Result { + unimplemented!() + } + fn account_secret(&self, _account: &Address) -> Result { + unimplemented!() + } + + fn sign(&self, _account: &Address, _message: &H256) -> Result { + unimplemented!() + } + +} + diff --git a/rpc/src/v1/tests/helpers/mod.rs b/rpc/src/v1/tests/helpers/mod.rs index 501bfb2d3..3bd74bab7 100644 --- a/rpc/src/v1/tests/helpers/mod.rs +++ b/rpc/src/v1/tests/helpers/mod.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +mod account_provider; mod sync_provider; +pub use self::account_provider::{TestAccount, TestAccountProvider}; pub use self::sync_provider::{Config, TestSyncProvider}; diff --git a/rpc/src/v1/tests/mod.rs b/rpc/src/v1/tests/mod.rs index 3a38ced15..3374bad36 100644 --- a/rpc/src/v1/tests/mod.rs +++ b/rpc/src/v1/tests/mod.rs @@ -16,6 +16,7 @@ //!TODO: load custom blockchain state and test +mod eth; mod net; mod web3; mod helpers; From e09de6ea3d8877cb8926dff11b9cef644ae07506 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Mar 2016 19:51:24 +0100 Subject: [PATCH 20/26] added missing eth_getBalance rpc method and tests for it --- ethcore/src/client/client.rs | 4 ++++ ethcore/src/client/mod.rs | 3 +++ ethcore/src/client/test_client.rs | 11 +++++++++++ rpc/src/v1/impls/eth.rs | 5 +++++ rpc/src/v1/tests/eth.rs | 25 ++++++++++++++++++++++++- rpc/src/v1/traits/eth.rs | 2 +- 6 files changed, 48 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 4e8c34b33..a860dd752 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -449,6 +449,10 @@ impl BlockChainClient for Client where V: Verifier { self.state().code(address) } + fn balance(&self, address: &Address) -> U256 { + self.state().balance(address) + } + fn transaction(&self, id: TransactionId) -> Option { match id { TransactionId::Hash(ref hash) => self.chain.transaction_address(hash), diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index afdfb200a..af2c6ac14 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -66,6 +66,9 @@ pub trait BlockChainClient : Sync + Send { /// Get address code. fn code(&self, address: &Address) -> Option; + /// Get address balance. + fn balance(&self, address: &Address) -> U256; + /// Get transaction with given hash. fn transaction(&self, id: TransactionId) -> Option; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 207f1090f..cf1352bc8 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -40,6 +40,8 @@ pub struct TestBlockChainClient { pub last_hash: RwLock, /// Difficulty. pub difficulty: RwLock, + /// Balances. + pub balances: RwLock>, } #[derive(Clone)] @@ -65,12 +67,17 @@ impl TestBlockChainClient { genesis_hash: H256::new(), last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), + balances: RwLock::new(HashMap::new()), }; client.add_blocks(1, EachBlockWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } + pub fn set_balance(&mut self, address: Address, balance: U256) { + self.balances.write().unwrap().insert(address, balance); + } + /// Add blocks to test client. pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { let len = self.numbers.read().unwrap().len(); @@ -165,6 +172,10 @@ impl BlockChainClient for TestBlockChainClient { unimplemented!(); } + fn balance(&self, address: &Address) -> U256 { + self.balances.read().unwrap().get(address).cloned().unwrap_or_else(U256::zero) + } + fn transaction(&self, _id: TransactionId) -> Option { unimplemented!(); } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 47b5471b3..f5159f55f 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -170,6 +170,11 @@ impl Eth for EthClient where C: BlockChainClient + 'static, S: } } + fn balance(&self, params: Params) -> Result { + from_params::<(Address, BlockNumber)>(params) + .and_then(|(address, _block_number)| to_value(&take_weak!(self.client).balance(&address))) + } + fn block_transaction_count_by_hash(&self, params: Params) -> Result { from_params::<(H256,)>(params) .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index c36a9d172..320d583d1 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -17,7 +17,8 @@ use std::collections::HashMap; use std::sync::Arc; use jsonrpc_core::IoHandler; -use util::hash::{Address}; +use util::hash::Address; +use util::numbers::U256; use ethcore::client::{TestBlockChainClient, EachBlockWith}; use v1::{Eth, EthClient}; use v1::tests::helpers::{TestAccount, TestAccountProvider, TestSyncProvider, Config}; @@ -25,6 +26,7 @@ use v1::tests::helpers::{TestAccount, TestAccountProvider, TestSyncProvider, Con fn blockchain_client() -> Arc { let mut client = TestBlockChainClient::new(); client.add_blocks(10, EachBlockWith::Nothing); + client.set_balance(Address::from(1), U256::from(5)); Arc::new(client) } @@ -57,3 +59,24 @@ fn rpc_eth_accounts() { assert_eq!(io.handle_request(request), Some(response.to_owned())); } + +#[test] +fn rpc_eth_balance() { + let client = blockchain_client(); + let sync = sync_provider(); + let ap = accounts_provider(); + + let eth = EthClient::new(&client, &sync, &ap).to_delegate(); + let io = IoHandler::new(); + io.add_delegate(eth); + + let request = r#"{ + "jsonrpc": "2.0", + "method": "eth_getBalance", + "params": ["0x0000000000000000000000000000000000000001", "latest"], + "id": 1 + }"#; + let response = r#"{"jsonrpc":"2.0","result":"0x05","id":1}"#; + + assert_eq!(io.handle_request(request), Some(response.to_owned())); +} diff --git a/rpc/src/v1/traits/eth.rs b/rpc/src/v1/traits/eth.rs index 8c24dd38c..7d33cb63f 100644 --- a/rpc/src/v1/traits/eth.rs +++ b/rpc/src/v1/traits/eth.rs @@ -130,7 +130,7 @@ pub trait Eth: Sized + Send + Sync + 'static { delegate.add_method("eth_gasPrice", Eth::gas_price); delegate.add_method("eth_accounts", Eth::accounts); delegate.add_method("eth_blockNumber", Eth::block_number); - delegate.add_method("eth_balance", Eth::balance); + delegate.add_method("eth_getBalance", Eth::balance); delegate.add_method("eth_getStorageAt", Eth::storage_at); delegate.add_method("eth_getTransactionCount", Eth::transaction_count); delegate.add_method("eth_getBlockTransactionCountByHash", Eth::block_transaction_count_by_hash); From 361280a9bebb9b799effaf0689a26d7b2020a217 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 12 Mar 2016 19:52:37 +0100 Subject: [PATCH 21/26] Limit incoming connections --- util/src/network/host.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index f63c75e9f..57aae51d7 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -541,7 +541,7 @@ impl Host where Message: Send + Sync + Clone { match TcpStream::connect(&address) { Ok(socket) => socket, Err(e) => { - warn!("Can't connect to address {:?}: {:?}", address, e); + debug!("Can't connect to address {:?}: {:?}", address, e); return; } } @@ -695,6 +695,14 @@ impl Host where Message: Send + Sync + Clone { return; } }; + if !originated { + let session_count = sessions.count(); + let ideal_peers = { self.info.read().unwrap().deref().config.ideal_peers }; + if session_count >= ideal_peers as usize { + session.disconnect(DisconnectReason::TooManyPeers); + return; + } + } let result = sessions.insert_with(move |session_token| { session.set_token(session_token); io.deregister_stream(token).expect("Error deleting handshake registration"); From 49dd6661998689a7e373b98f706eb9bb14307252 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Mar 2016 20:06:55 +0100 Subject: [PATCH 22/26] EthTester --- rpc/src/v1/tests/eth.rs | 44 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 320d583d1..8c61c2ed9 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -44,32 +44,40 @@ fn sync_provider() -> Arc { })) } +struct EthTester { + client: Arc, + sync: Arc, + accounts_provider: Arc, + pub io: IoHandler, +} + +impl Default for EthTester { + fn default() -> Self { + let client = blockchain_client(); + let sync = sync_provider(); + let ap = accounts_provider(); + let eth = EthClient::new(&client, &sync, &ap).to_delegate(); + let io = IoHandler::new(); + io.add_delegate(eth); + EthTester { + client: client, + sync: sync, + accounts_provider: ap, + io: io + } + } +} + #[test] fn rpc_eth_accounts() { - let client = blockchain_client(); - let sync = sync_provider(); - let ap = accounts_provider(); - - let eth = EthClient::new(&client, &sync, &ap).to_delegate(); - let io = IoHandler::new(); - io.add_delegate(eth); - let request = r#"{"jsonrpc": "2.0", "method": "eth_accounts", "params": [], "id": 1}"#; let response = r#"{"jsonrpc":"2.0","result":["0x0000000000000000000000000000000000000001"],"id":1}"#; - assert_eq!(io.handle_request(request), Some(response.to_owned())); + assert_eq!(EthTester::default().io.handle_request(request), Some(response.to_owned())); } #[test] fn rpc_eth_balance() { - let client = blockchain_client(); - let sync = sync_provider(); - let ap = accounts_provider(); - - let eth = EthClient::new(&client, &sync, &ap).to_delegate(); - let io = IoHandler::new(); - io.add_delegate(eth); - let request = r#"{ "jsonrpc": "2.0", "method": "eth_getBalance", @@ -78,5 +86,5 @@ fn rpc_eth_balance() { }"#; let response = r#"{"jsonrpc":"2.0","result":"0x05","id":1}"#; - assert_eq!(io.handle_request(request), Some(response.to_owned())); + assert_eq!(EthTester::default().io.handle_request(request), Some(response.to_owned())); } From 0a6fea1b77c58420d0014118786961770dcce713 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 13 Mar 2016 10:33:55 +0100 Subject: [PATCH 23/26] Silenced some non-important warnings --- sync/src/chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 84ca5500a..332b78d83 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -592,7 +592,7 @@ impl ChainSync { pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) { trace!(target: "sync", "== Connected {}", peer); if let Err(e) = self.send_status(io) { - warn!(target:"sync", "Error sending status request: {:?}", e); + debug!(target:"sync", "Error sending status request: {:?}", e); io.disable_peer(peer); } } @@ -1093,7 +1093,7 @@ impl ChainSync { let rlp = UntrustedRlp::new(data); if packet_id != STATUS_PACKET && !self.peers.contains_key(&peer) { - warn!(target:"sync", "Unexpected packet from unregistered peer: {}:{}", peer, io.peer_info(peer)); + debug!(target:"sync", "Unexpected packet from unregistered peer: {}:{}", peer, io.peer_info(peer)); return; } let result = match packet_id { From 0f21779ec4d5ea156437c0142cc21d5995f5e876 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sun, 13 Mar 2016 11:06:57 +0100 Subject: [PATCH 24/26] make heavy --- util/src/keys/geth_import.rs | 1 + util/src/keys/store.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs index dbd9f0fe0..6c684c37d 100644 --- a/util/src/keys/geth_import.rs +++ b/util/src/keys/geth_import.rs @@ -161,6 +161,7 @@ mod tests { } #[test] + #[cfg(feature="heavy-tests")] fn can_decrypt_with_imported() { use keys::store::EncryptedHashMap; diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 6a5efc87d..f80d91123 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -363,6 +363,7 @@ mod vector_tests { #[test] + #[cfg(feature="heavy-tests")] fn mac_vector() { let password = "testpassword"; let salt = H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(); @@ -464,6 +465,7 @@ mod tests { } #[test] + #[cfg(feature="heavy-tests")] fn can_get() { let temp = RandomTempPath::create_dir(); let key_id = { From ff51d0fa676d1ebeca0648afc44e9bcbdd413dc5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 13 Mar 2016 11:50:09 +0100 Subject: [PATCH 25/26] Additional tests --- util/src/journaldb/earlymergedb.rs | 20 +++++++++++++++++ util/src/journaldb/overlayrecentdb.rs | 32 ++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 0931d42d1..64c7fa69d 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -561,6 +561,26 @@ mod tests { assert!(jdb.exists(&x)); } + #[test] + fn insert_older_era() { + let mut jdb = EarlyMergeDB::new_temp(); + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0a".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + let bar = jdb.insert(b"bar"); + jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&bar); + jdb.commit(0, &b"0b".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(2, &b"1".sha3(), Some((1, b"1".sha3()))).unwrap(); + + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + } + #[test] fn long_history() { // history is 3 diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index 8dd4d1752..36e8b68ad 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -66,7 +66,7 @@ pub struct OverlayRecentDB { struct JournalOverlay { backing_overlay: MemoryDB, journal: HashMap>, - latest_era: u64, + latest_era: Option, } #[derive(PartialEq)] @@ -152,10 +152,10 @@ impl OverlayRecentDB { let mut journal = HashMap::new(); let mut overlay = MemoryDB::new(); let mut count = 0; - let mut latest_era = 0; + let mut latest_era = None; if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { - latest_era = decode::(&val); - let mut era = latest_era; + let mut era = decode::(&val); + latest_era = Some(era); loop { let mut index = 0usize; while let Some(rlp_data) = db.get({ @@ -241,9 +241,9 @@ impl JournalDB for OverlayRecentDB { k.append(&index); k.append(&&PADDING[..]); try!(batch.put(&k.drain(), r.as_raw())); - if now >= journal_overlay.latest_era { + if journal_overlay.latest_era.map_or(true, |e| now > e) { try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); - journal_overlay.latest_era = now; + journal_overlay.latest_era = Some(now); } journal_overlay.journal.entry(now).or_insert_with(Vec::new).push(JournalEntry { id: id.clone(), insertions: inserted_keys, deletions: removed_keys }); } @@ -870,4 +870,24 @@ mod tests { assert!(!jdb.exists(&bar)); } } + + #[test] + fn insert_older_era() { + let mut jdb = OverlayRecentDB::new_temp(); + let foo = jdb.insert(b"foo"); + jdb.commit(0, &b"0a".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + let bar = jdb.insert(b"bar"); + jdb.commit(1, &b"1".sha3(), Some((0, b"0a".sha3()))).unwrap(); + assert!(jdb.can_reconstruct_refs()); + + jdb.remove(&bar); + jdb.commit(0, &b"0b".sha3(), None).unwrap(); + assert!(jdb.can_reconstruct_refs()); + jdb.commit(2, &b"1".sha3(), Some((1, b"1".sha3()))).unwrap(); + + assert!(jdb.exists(&foo)); + assert!(jdb.exists(&bar)); + } } From 113161863060661f1b2afdf183f6bcfbb0706527 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 13 Mar 2016 11:55:48 +0100 Subject: [PATCH 26/26] Fixed test --- util/src/journaldb/earlymergedb.rs | 2 +- util/src/journaldb/overlayrecentdb.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 64c7fa69d..7f0f50da2 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -575,7 +575,7 @@ mod tests { jdb.remove(&bar); jdb.commit(0, &b"0b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"1".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index 36e8b68ad..7eb5266b0 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -885,7 +885,7 @@ mod tests { jdb.remove(&bar); jdb.commit(0, &b"0b".sha3(), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.commit(2, &b"1".sha3(), Some((1, b"1".sha3()))).unwrap(); + jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar));