From 852233e4ac1b3109e2b73f876edaba68052f0c2f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 03:08:57 +0100 Subject: [PATCH 1/8] HashDB and initial OverlayDB --- src/hashdb.rs | 6 +++--- src/lib.rs | 1 + src/memorydb.rs | 24 ++++++++++++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/hashdb.rs b/src/hashdb.rs index d023fe70d..48f5182bc 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -14,10 +14,10 @@ pub trait HashDB { /// let mut m = MemoryDB::new(); /// let hello_bytes = "Hello world!".as_bytes(); /// let hash = m.insert(hello_bytes); - /// assert_eq!(m.lookup(&hash).unwrap(), &hello_bytes); + /// assert_eq!(m.lookup(&hash).unwrap(), hello_bytes); /// } /// ``` - fn lookup(&self, key: &H256) -> Option<&Bytes>; + fn lookup(&self, key: &H256) -> Option; /// Check for the existance of a hash-key. /// @@ -75,7 +75,7 @@ pub trait HashDB { /// m.insert(d); // OK - now it's "empty" again. /// assert!(!m.exists(key)); /// m.insert(d); // OK - now we've - /// assert_eq!(m.lookup(key).unwrap(), &d); + /// assert_eq!(m.lookup(key).unwrap(), d); /// } /// ``` fn kill(&mut self, key: &H256); diff --git a/src/lib.rs b/src/lib.rs index f7f446ddc..cb03d00b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod db; pub mod sha3; pub mod hashdb; pub mod memorydb; +pub mod overlaydb; pub mod bloom; //pub mod network; diff --git a/src/memorydb.rs b/src/memorydb.rs index 55fdc12da..2f54b4524 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -25,7 +25,7 @@ use std::collections::HashMap; /// /// let k = m.insert(d); /// assert!(m.exists(&k)); -/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// assert_eq!(m.lookup(&k).unwrap(), d); /// /// m.insert(d); /// assert!(m.exists(&k)); @@ -38,7 +38,7 @@ use std::collections::HashMap; /// /// m.insert(d); /// assert!(m.exists(&k)); -/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// assert_eq!(m.lookup(&k).unwrap(), d); /// /// m.kill(&k); /// assert!(!m.exists(&k)); @@ -84,12 +84,27 @@ impl MemoryDB { .collect(); for empty in empties { self.data.remove(&empty); } } + + /// Grab the number of references a particular `key` has. Returns None if the key + /// doesn't exist. + fn refs(&self, key: &H256) -> Option { + self.data.get(key).map(|&(_, rc)| rc) + } + + /// Grab the value associated with a particular `key`. Returns None if the key + /// doesn't exist. + /// + /// Even when Some is returned, this is only guaranteed to return something useful + /// when `refs(key) > 0`. + fn value(&self, key: &H256) -> Option<&Bytes> { + self.data.get(key).map(|&(ref d, _)| d) + } } impl HashDB for MemoryDB { - fn lookup(&self, key: &H256) -> Option<&Bytes> { + fn lookup(&self, key: &H256) -> Option { match self.data.get(key) { - Some(&(ref d, rc)) if rc > 0 => Some(d), + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), _ => None } } @@ -116,6 +131,7 @@ impl HashDB for MemoryDB { } key } + fn kill(&mut self, key: &H256) { if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } From 71dc9c0ad46f79afef069026b4ba1bb34ac0e4a8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 03:09:36 +0100 Subject: [PATCH 2/8] OVerlayDB draft. --- src/overlaydb.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/overlaydb.rs diff --git a/src/overlaydb.rs b/src/overlaydb.rs new file mode 100644 index 000000000..99d66cd5f --- /dev/null +++ b/src/overlaydb.rs @@ -0,0 +1,70 @@ +//! Disk-backed HashDB implementation. + +use hash::*; +use bytes::*; +use sha3::*; +use hashdb::*; +use memorydb::*; +use std::ops::*; +use rocksdb::{DB, Writable}; + +#[derive(Clone)] +pub struct OverlayDB { + overlay: MemoryDB, + backing: DB, +} + +impl OverlayDB { + /// Create a new instance of OverlayDB given a `backing` database. + fn new(backing: DB) { + self.backing = backing; + overlay = MemoryDB::new(); + } + + /// Commit all memory operations to the backing database. + fn commit(&mut self) { + unimplemented!(); + } + + /// Get the refs and value of the given key. + fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { + unimplemented!(); + } +} + +impl HashDB for OverlayDB { + fn lookup(&self, key: &H256) -> Option { + // TODO: return ok if positive; if negative, check backing - might be enough references there to make + // it positive again. + let k = self.overlay.data.get(key); + match k { + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + _ => { + let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); + match self.payload(key) { + Some((d, rc)) if rc + memrc > 0 => Some(d), + _ => None, + } + } + } + } + fn exists(&self, key: &H256) -> bool { + // TODO: copy and adapt code above. + m_overlay.exists(key) + } + fn insert(&mut self, value: &[u8]) -> H256 { m_overlay.insert(value) } + fn kill(&mut self, key: &H256) { m_overlay.kill(key); } +} + +#[test] +fn playpen() { + let mut db: DB = DB::open_default("/tmp/test").unwrap(); + db.put(b"test", b"test2"); + match db.get(b"test") { + Ok(Some(value)) => println!("Got value {:?}", value.deref()), + Ok(None) => println!("No value for that key"), + Err(e) => println!("Gah"), + } + db.delete(b"test"); + assert!(false); +} \ No newline at end of file From 4640d64965aa7220c8a097a2374a7f35b109be6c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 18:29:50 +0100 Subject: [PATCH 3/8] Reference-counted rocksdb backed DB. Just need DB I/O now. --- src/error.rs | 29 ++++++++++++++++- src/memorydb.rs | 23 ++++++------- src/overlaydb.rs | 84 ++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 110 insertions(+), 26 deletions(-) diff --git a/src/error.rs b/src/error.rs index 9dc471f67..bb09e2e1b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,9 +1,17 @@ +#![feature(concat_idents)] + use rustc_serialize::hex::*; +#[derive(Debug)] +pub enum BaseDataError { + NegativelyReferencedHash, +} + #[derive(Debug)] pub enum EthcoreError { FromHex(FromHexError), - BadSize + BaseData(BaseDataError), + BadSize, } impl From for EthcoreError { @@ -11,3 +19,22 @@ impl From for EthcoreError { EthcoreError::FromHex(err) } } + +impl From for EthcoreError { + fn from(err: BaseDataError) -> EthcoreError { + EthcoreError::BaseData(err) + } +} + +// TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted. +/*macro_rules! assimilate { + ($name:ident) => ( + impl From for EthcoreError { + fn from(err: concat_idents!($name, Error)) -> EthcoreError { + EthcoreError:: $name (err) + } + } + ) +} +assimilate!(FromHex); +assimilate!(BaseData);*/ \ No newline at end of file diff --git a/src/memorydb.rs b/src/memorydb.rs index 2f54b4524..cbb9e426b 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -4,6 +4,7 @@ use hash::*; use bytes::*; use sha3::*; use hashdb::*; +use std::mem; use std::collections::HashMap; #[derive(Debug,Clone)] @@ -85,19 +86,19 @@ impl MemoryDB { for empty in empties { self.data.remove(&empty); } } - /// Grab the number of references a particular `key` has. Returns None if the key - /// doesn't exist. - fn refs(&self, key: &H256) -> Option { - self.data.get(key).map(|&(_, rc)| rc) - } - - /// Grab the value associated with a particular `key`. Returns None if the key + /// Grab the raw information associated with a key. Returns None if the key /// doesn't exist. /// - /// Even when Some is returned, this is only guaranteed to return something useful - /// when `refs(key) > 0`. - fn value(&self, key: &H256) -> Option<&Bytes> { - self.data.get(key).map(|&(ref d, _)| d) + /// Even when Some is returned, the data is only guaranteed to be useful + /// when the refs > 0. + pub fn raw(&self, key: &H256) -> Option<&(Bytes, i32)> { + self.data.get(key) + } + + pub fn drain(&mut self) -> HashMap { + let mut data = HashMap::new(); + mem::swap(&mut self.data, &mut data); + data } } diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 99d66cd5f..cb6d396c4 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -1,59 +1,115 @@ //! Disk-backed HashDB implementation. +use error::*; use hash::*; use bytes::*; use sha3::*; use hashdb::*; use memorydb::*; use std::ops::*; +use std::sync::*; use rocksdb::{DB, Writable}; #[derive(Clone)] pub struct OverlayDB { overlay: MemoryDB, - backing: DB, + backing: Arc, } impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - fn new(backing: DB) { - self.backing = backing; - overlay = MemoryDB::new(); + fn new(backing: DB) -> OverlayDB { + OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } - /// Commit all memory operations to the backing database. - fn commit(&mut self) { - unimplemented!(); + /// Commit all memory operations to the backing database. + fn commit(&mut self) -> Result { + let mut ret = 0u32; + for i in self.overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc != 0 { + let new_entry = match self.payload(&key) { + Some(x) => { + let (back_value, back_rc) = x; + if back_rc + rc < 0 { + return Err(From::from(BaseDataError::NegativelyReferencedHash)); + } + self.put_payload(&key, (&back_value, rc + back_rc)); + } + None => { + self.put_payload(&key, (&value, rc)); + } + }; + ret += 1; + } + } + Ok(ret) } /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { unimplemented!(); } + + /// Get the refs and value of the given key. + fn put_payload(&self, key: &H256, payload: (&Bytes, i32)) { + unimplemented!(); + } } impl HashDB for OverlayDB { fn lookup(&self, key: &H256) -> Option { - // TODO: return ok if positive; if negative, check backing - might be enough references there to make + // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. - let k = self.overlay.data.get(key); + let k = self.overlay.raw(key); match k { Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { - Some((d, rc)) if rc + memrc > 0 => Some(d), + Some(x) => { + let (d, rc) = x; + if rc + memrc > 0 { + Some(d) + } + else { + None + } + } + // Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done. + //Some((d, rc)) if rc + memrc > 0 => Some(d), _ => None, } } } } fn exists(&self, key: &H256) -> bool { - // TODO: copy and adapt code above. - m_overlay.exists(key) + // return ok if positive; if negative, check backing - might be enough references there to make + // it positive again. + let k = self.overlay.raw(key); + match k { + Some(&(ref d, rc)) if rc > 0 => true, + _ => { + let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); + match self.payload(key) { + Some(x) => { + let (d, rc) = x; + if rc + memrc > 0 { + true + } + else { + false + } + } + // Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done. + //Some((d, rc)) if rc + memrc > 0 => true, + _ => false, + } + } + } } - fn insert(&mut self, value: &[u8]) -> H256 { m_overlay.insert(value) } - fn kill(&mut self, key: &H256) { m_overlay.kill(key); } + fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } + fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } #[test] From 7733faaab5961e947da66b37be2cdee17317b0df Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 19:03:50 +0100 Subject: [PATCH 4/8] overlaydb put_payload. --- src/overlaydb.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index cb6d396c4..c59be085a 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -4,6 +4,7 @@ use error::*; use hash::*; use bytes::*; use sha3::*; +use rlp::*; use hashdb::*; use memorydb::*; use std::ops::*; @@ -34,10 +35,10 @@ impl OverlayDB { if back_rc + rc < 0 { return Err(From::from(BaseDataError::NegativelyReferencedHash)); } - self.put_payload(&key, (&back_value, rc + back_rc)); + self.put_payload(&key, (back_value, rc + back_rc)); } None => { - self.put_payload(&key, (&value, rc)); + self.put_payload(&key, (value, rc)); } }; ret += 1; @@ -48,12 +49,20 @@ impl OverlayDB { /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { - unimplemented!(); + db.get(&key.bytes()) + .expect("Low-level database error. Some issue with your hard disk?") + .map(|d| { + Rlp r(d.deref()); + r(Bytes, i32) + }) } /// Get the refs and value of the given key. - fn put_payload(&self, key: &H256, payload: (&Bytes, i32)) { - unimplemented!(); + fn put_payload(&self, key: &H256, payload: (Bytes, i32)) { + let mut s = RlpStream::new_list(2); + s.append(payload.1); + s.append(payload.0); + backing.put(&key.bytes(), &s.out().unwrap()); } } From 358962d2ea5f17a1e8f89adaad07501b760d6b1a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 19:28:59 +0100 Subject: [PATCH 5/8] Implemented, in principle. --- src/error.rs | 5 ++--- src/lib.rs | 1 - src/overlaydb.rs | 41 ++++++++++++++++++++++------------------- src/rlp.rs | 8 ++++---- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/error.rs b/src/error.rs index bb09e2e1b..f6c64a54f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,3 @@ -#![feature(concat_idents)] - use rustc_serialize::hex::*; #[derive(Debug)] @@ -27,7 +25,8 @@ impl From for EthcoreError { } // TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted. -/*macro_rules! assimilate { +/*#![feature(concat_idents)] +macro_rules! assimilate { ($name:ident) => ( impl From for EthcoreError { fn from(err: concat_idents!($name, Error)) -> EthcoreError { diff --git a/src/lib.rs b/src/lib.rs index 1c8ad9ccc..e918c7ca9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,6 @@ pub mod sha3; pub mod hashdb; pub mod memorydb; pub mod overlaydb; -pub mod bloom; pub mod math; //pub mod network; diff --git a/src/overlaydb.rs b/src/overlaydb.rs index c59be085a..d2f838bb7 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -3,7 +3,6 @@ use error::*; use hash::*; use bytes::*; -use sha3::*; use rlp::*; use hashdb::*; use memorydb::*; @@ -19,26 +18,30 @@ pub struct OverlayDB { impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - fn new(backing: DB) -> OverlayDB { + pub fn new(backing: DB) -> OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } /// Commit all memory operations to the backing database. - fn commit(&mut self) -> Result { + pub fn commit(&mut self) -> Result { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { let (key, (value, rc)) = i; if rc != 0 { - let new_entry = match self.payload(&key) { + match self.payload(&key) { Some(x) => { let (back_value, back_rc) = x; - if back_rc + rc < 0 { + let total_rc: i32 = back_rc as i32 + rc; + if total_rc < 0 { return Err(From::from(BaseDataError::NegativelyReferencedHash)); } - self.put_payload(&key, (back_value, rc + back_rc)); + self.put_payload(&key, (back_value, total_rc as u32)); } None => { - self.put_payload(&key, (value, rc)); + if rc < 0 { + return Err(From::from(BaseDataError::NegativelyReferencedHash)); + } + self.put_payload(&key, (value, rc as u32)); } }; ret += 1; @@ -48,21 +51,21 @@ impl OverlayDB { } /// Get the refs and value of the given key. - fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { - db.get(&key.bytes()) + fn payload(&self, key: &H256) -> Option<(Bytes, u32)> { + self.backing.get(&key.bytes()) .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { - Rlp r(d.deref()); - r(Bytes, i32) + let r = Rlp::new(d.deref()); + (Bytes::decode(&r.at(1).unwrap()).unwrap(), u32::decode(&r.at(0).unwrap()).unwrap()) }) } /// Get the refs and value of the given key. - fn put_payload(&self, key: &H256, payload: (Bytes, i32)) { + fn put_payload(&self, key: &H256, payload: (Bytes, u32)) { let mut s = RlpStream::new_list(2); - s.append(payload.1); - s.append(payload.0); - backing.put(&key.bytes(), &s.out().unwrap()); + s.append(&payload.1); + s.append(&payload.0); + self.backing.put(&key.bytes(), &s.out().unwrap()).expect("Low-level database error. Some issue with your hard disk?"); } } @@ -78,7 +81,7 @@ impl HashDB for OverlayDB { match self.payload(key) { Some(x) => { let (d, rc) = x; - if rc + memrc > 0 { + if rc as i32 + memrc > 0 { Some(d) } else { @@ -97,13 +100,13 @@ impl HashDB for OverlayDB { // it positive again. let k = self.overlay.raw(key); match k { - Some(&(ref d, rc)) if rc > 0 => true, + Some(&(_, rc)) if rc > 0 => true, _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { Some(x) => { - let (d, rc) = x; - if rc + memrc > 0 { + let (_, rc) = x; + if rc as i32 + memrc > 0 { true } else { diff --git a/src/rlp.rs b/src/rlp.rs index 4b6df2c6e..53860579b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -50,10 +50,10 @@ //! } //! //! fn decode_list() { -//! // ["cat", "dog"] -//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = Rlp::new(&data); -//! let _ : Vec = Decodable::decode(&rlp).unwrap(); +//! // ["cat", "dog"] +//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +//! let rlp = Rlp::new(&data); +//! let _ : Vec = Decodable::decode(&rlp).unwrap(); //! } //! //! fn decode_list2() { From cefb1fa9cefa37f7d197ef7340614a35791a773f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 22:54:12 +0100 Subject: [PATCH 6/8] OverlayDB complete and tested. --- src/hash.rs | 7 +++ src/overlaydb.rs | 117 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index a0493f2bb..c03650be3 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -232,9 +232,15 @@ macro_rules! impl_hash { } } + impl $from { + pub fn hex(&self) -> String { + format!("{}", self) + } + } } } +impl_hash!(H32, 4); impl_hash!(H64, 8); impl_hash!(H128, 16); impl_hash!(Address, 20); @@ -255,6 +261,7 @@ mod tests { assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); assert_eq!(format!("{}", h), "0123456789abcdef"); assert_eq!(format!("{:?}", h), "0123456789abcdef"); + assert_eq!(h.hex(), "0123456789abcdef"); assert!(h == h); assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); assert!(h != H64([0; 8])); diff --git a/src/overlaydb.rs b/src/overlaydb.rs index d2f838bb7..2f712da24 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -8,6 +8,7 @@ use hashdb::*; use memorydb::*; use std::ops::*; use std::sync::*; +use std::env; use rocksdb::{DB, Writable}; #[derive(Clone)] @@ -22,6 +23,13 @@ impl OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } + /// Create a new instance of OverlayDB with an anonymous temporary database. + pub fn new_temp() -> OverlayDB { + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + } + /// Commit all memory operations to the backing database. pub fn commit(&mut self) -> Result { let mut ret = 0u32; @@ -50,6 +58,8 @@ impl OverlayDB { Ok(ret) } + pub fn revert(&mut self) { self.overlay.clear(); } + /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, u32)> { self.backing.get(&key.bytes()) @@ -124,15 +134,106 @@ impl HashDB for OverlayDB { fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } +#[test] +fn overlaydb_overlay_insert_and_kill() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); +} + +#[test] +fn overlaydb_backing_insert_revert() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.revert(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); +} + +#[test] +fn overlaydb_backing_kill() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&h), None); + trie.revert(); + assert_eq!(trie.lookup(&h), None); +} + +#[test] +fn overlaydb_backing_kill_revert() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); + trie.revert(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); +} + +#[test] +fn overlaydb_negative() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + trie.kill(&h); //bad - sends us into negative refs. + assert_eq!(trie.lookup(&h), None); + assert!(trie.commit().is_err()); +} + +#[test] +fn overlaydb_complex() { + let mut trie = OverlayDB::new_temp(); + let hfoo = trie.insert(b"foo"); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + let hbar = trie.insert(b"bar"); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.insert(b"foo"); // two refs + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.kill(&hbar); // zero refs - delete + assert_eq!(trie.lookup(&hbar), None); + trie.kill(&hfoo); // one ref - keep + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.kill(&hfoo); // zero ref - would delete, but... + assert_eq!(trie.lookup(&hfoo), None); + trie.insert(b"foo"); // one ref - keep after all. + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.kill(&hfoo); // zero ref - delete + assert_eq!(trie.lookup(&hfoo), None); + trie.commit().unwrap(); // + assert_eq!(trie.lookup(&hfoo), None); +} + #[test] fn playpen() { - let mut db: DB = DB::open_default("/tmp/test").unwrap(); - db.put(b"test", b"test2"); - match db.get(b"test") { - Ok(Some(value)) => println!("Got value {:?}", value.deref()), - Ok(None) => println!("No value for that key"), - Err(e) => println!("Gah"), + use std::fs; + { + let db: DB = DB::open_default("/tmp/test").unwrap(); + db.put(b"test", b"test2").unwrap(); + match db.get(b"test") { + Ok(Some(value)) => println!("Got value {:?}", value.deref()), + Ok(None) => println!("No value for that key"), + Err(..) => println!("Gah"), + } + db.delete(b"test").unwrap(); } - db.delete(b"test"); - assert!(false); + fs::remove_dir_all("/tmp/test").unwrap(); } \ No newline at end of file From 13a692cc9dd13a1d3b1aa71a100492ece51c3137 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 23:49:07 +0100 Subject: [PATCH 7/8] Docuemntation udpate and tests. --- src/overlaydb.rs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 2f712da24..335edbc9e 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -12,6 +12,14 @@ use std::env; use rocksdb::{DB, Writable}; #[derive(Clone)] +/// Implementation of the HashDB trait for a disk-backed database with a memory overlay. +/// +/// The operations `insert()` and `kill()` take place on the memory overlay; batches of +/// such operations may be flushed to the disk-backed DB with `commit()` or discarded with +/// `revert()`. +/// +/// `lookup()` and `exists()` maintain normal behaviour - all `insert()` and `kill()` +/// queries have an immediate effect in terms of these functions. pub struct OverlayDB { overlay: MemoryDB, backing: Arc, @@ -30,7 +38,33 @@ impl OverlayDB { Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) } - /// Commit all memory operations to the backing database. + /// Commit all memory operations to the backing database. + /// + /// Returns either an error or the number of items changed in the backing database. + /// + /// Will return an error if the number of `kill()`s ever exceeds the number of + /// `insert()`s for any key. This will leave the database in an undeterminate + /// state. Don't ever let it happen. + /// + /// # Example + /// ``` + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::overlaydb::*; + /// fn main() { + /// let mut m = OverlayDB::new_temp(); + /// let key = m.insert(b"foo"); // insert item. + /// assert!(m.exists(&key)); // key exists (in memory). + /// assert_eq!(m.commit().unwrap(), 1); // 1 item changed. + /// assert!(m.exists(&key)); // key still exists (in backing). + /// m.kill(&key); // delete item. + /// assert!(!m.exists(&key)); // key "doesn't exist" (though still does in backing). + /// m.kill(&key); // oh dear... more kills than inserts for the key... + /// //m.commit().unwrap(); // this commit/unwrap would cause a panic. + /// m.revert(); // revert both kills. + /// assert!(m.exists(&key)); // key now still exists. + /// } + /// ``` pub fn commit(&mut self) -> Result { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { @@ -58,6 +92,8 @@ impl OverlayDB { Ok(ret) } + /// Revert all changes though `insert()` and `kill()` to this object since the + /// last `commit()`. pub fn revert(&mut self) { self.overlay.clear(); } /// Get the refs and value of the given key. From be96b2638de20d4cd3ceefb5da9b019250bbdf88 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 23:54:58 +0100 Subject: [PATCH 8/8] Doc tweaks. --- src/overlaydb.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 335edbc9e..7db67467b 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -92,8 +92,27 @@ impl OverlayDB { Ok(ret) } - /// Revert all changes though `insert()` and `kill()` to this object since the + /// Revert all operations on this object (i.e. `insert()`s and `kill()`s) since the /// last `commit()`. + /// + /// # Example + /// ``` + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::overlaydb::*; + /// fn main() { + /// let mut m = OverlayDB::new_temp(); + /// let foo = m.insert(b"foo"); // insert foo. + /// m.commit().unwrap(); // commit - new operations begin here... + /// let bar = m.insert(b"bar"); // insert bar. + /// m.kill(&foo); // kill foo. + /// assert!(!m.exists(&foo)); // foo is gone. + /// assert!(m.exists(&bar)); // bar is here. + /// m.revert(); // revert the last two operations. + /// assert!(m.exists(&foo)); // foo is here. + /// assert!(!m.exists(&bar)); // bar is gone. + /// } + /// ``` pub fn revert(&mut self) { self.overlay.clear(); } /// Get the refs and value of the given key.