openethereum/util/src/overlaydb.rs

380 lines
12 KiB
Rust
Raw Normal View History

2016-02-05 13:40:41 +01:00
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
2016-04-06 10:07:24 +02:00
//! Disk-backed `HashDB` implementation.
2015-11-28 03:09:36 +01:00
use error::*;
2015-11-28 03:09:36 +01:00
use hash::*;
use bytes::*;
2015-11-28 19:03:50 +01:00
use rlp::*;
2015-11-28 03:09:36 +01:00
use hashdb::*;
use memorydb::*;
use std::ops::*;
use std::sync::*;
2015-11-28 22:54:12 +01:00
use std::env;
use std::collections::HashMap;
2016-03-13 18:07:36 +01:00
use kvdb::{Database, DBTransaction};
2015-11-28 03:09:36 +01:00
2016-04-06 10:07:24 +02:00
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay.
2015-11-28 23:49:07 +01:00
///
2016-01-18 12:41:31 +01:00
/// The operations `insert()` and `remove()` take place on the memory overlay; batches of
2015-11-28 23:49:07 +01:00
/// such operations may be flushed to the disk-backed DB with `commit()` or discarded with
/// `revert()`.
///
/// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()`
2015-11-28 23:49:07 +01:00
/// queries have an immediate effect in terms of these functions.
2016-03-13 18:07:36 +01:00
#[derive(Clone)]
2015-11-28 03:09:36 +01:00
pub struct OverlayDB {
overlay: MemoryDB,
2016-02-18 03:46:24 +01:00
backing: Arc<Database>,
2015-11-28 03:09:36 +01:00
}
impl OverlayDB {
/// Create a new instance of OverlayDB given a `backing` database.
2016-02-18 03:46:24 +01:00
pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) }
2016-01-18 12:41:31 +01:00
/// Create a new instance of OverlayDB given a `backing` database.
2016-02-18 03:46:24 +01:00
pub fn new_with_arc(backing: Arc<Database>) -> OverlayDB {
2016-01-18 12:41:31 +01:00
OverlayDB{ overlay: MemoryDB::new(), backing: backing }
2015-11-28 03:09:36 +01:00
}
2015-11-28 22:54:12 +01:00
/// Create a new instance of OverlayDB with an anonymous temporary database.
pub fn new_temp() -> OverlayDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
2016-02-18 03:46:24 +01:00
Self::new(Database::open_default(dir.to_str().unwrap()).unwrap())
2015-11-28 22:54:12 +01:00
}
2016-03-13 18:07:36 +01:00
/// Commit all operations to given batch.
pub fn commit_to_batch(&mut self, batch: &DBTransaction) -> Result<u32, UtilError> {
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc != 0 {
match self.payload(&key) {
Some(x) => {
let (back_value, back_rc) = x;
let total_rc: i32 = back_rc as i32 + rc;
if total_rc < 0 {
2016-03-13 21:54:06 +01:00
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
2016-03-13 18:07:36 +01:00
}
2016-03-13 19:22:42 +01:00
deletes += if self.put_payload_in_batch(batch, &key, (back_value, total_rc as u32)) {1} else {0};
2016-03-13 18:07:36 +01:00
}
None => {
if rc < 0 {
2016-03-13 21:54:06 +01:00
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
2016-03-13 18:07:36 +01:00
}
2016-03-13 19:22:42 +01:00
self.put_payload_in_batch(batch, &key, (value, rc as u32));
2016-03-13 18:07:36 +01:00
}
};
ret += 1;
}
}
trace!("OverlayDB::commit() deleted {} nodes", deletes);
Ok(ret)
}
2015-11-28 23:49:07 +01:00
/// Commit all memory operations to the backing database.
///
/// Returns either an error or the number of items changed in the backing database.
///
/// Will return an error if the number of `remove()`s ever exceeds the number of
2015-11-28 23:49:07 +01:00
/// `insert()`s for any key. This will leave the database in an undeterminate
/// state. Don't ever let it happen.
///
/// # Example
/// ```
/// extern crate ethcore_util;
/// use ethcore_util::hashdb::*;
/// use ethcore_util::overlaydb::*;
/// fn main() {
/// let mut m = OverlayDB::new_temp();
/// let key = m.insert(b"foo"); // insert item.
/// assert!(m.contains(&key)); // key exists (in memory).
2015-11-28 23:49:07 +01:00
/// assert_eq!(m.commit().unwrap(), 1); // 1 item changed.
/// assert!(m.contains(&key)); // key still exists (in backing).
/// m.remove(&key); // delete item.
/// assert!(!m.contains(&key)); // key "doesn't exist" (though still does in backing).
/// m.remove(&key); // oh dear... more removes than inserts for the key...
2015-11-28 23:49:07 +01:00
/// //m.commit().unwrap(); // this commit/unwrap would cause a panic.
/// m.revert(); // revert both removes.
/// assert!(m.contains(&key)); // key now still exists.
2015-11-28 23:49:07 +01:00
/// }
/// ```
pub fn commit(&mut self) -> Result<u32, UtilError> {
2016-03-13 19:22:42 +01:00
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i;
if rc != 0 {
match self.payload(&key) {
Some(x) => {
let (back_value, back_rc) = x;
let total_rc: i32 = back_rc as i32 + rc;
if total_rc < 0 {
2016-03-13 21:54:06 +01:00
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
2016-03-13 19:22:42 +01:00
}
deletes += if self.put_payload(&key, (back_value, total_rc as u32)) {1} else {0};
}
None => {
if rc < 0 {
2016-03-13 21:54:06 +01:00
return Err(From::from(BaseDataError::NegativelyReferencedHash(key)));
2016-03-13 19:22:42 +01:00
}
self.put_payload(&key, (value, rc as u32));
}
};
ret += 1;
}
}
trace!("OverlayDB::commit() deleted {} nodes", deletes);
Ok(ret)
2015-11-28 03:09:36 +01:00
}
/// Revert all operations on this object (i.e. `insert()`s and `remove()`s) since the
2015-11-28 23:49:07 +01:00
/// last `commit()`.
2015-11-28 23:54:58 +01:00
///
/// # Example
/// ```
/// extern crate ethcore_util;
/// use ethcore_util::hashdb::*;
/// use ethcore_util::overlaydb::*;
/// fn main() {
/// let mut m = OverlayDB::new_temp();
/// let foo = m.insert(b"foo"); // insert foo.
/// m.commit().unwrap(); // commit - new operations begin here...
/// let bar = m.insert(b"bar"); // insert bar.
/// m.remove(&foo); // remove foo.
/// assert!(!m.contains(&foo)); // foo is gone.
/// assert!(m.contains(&bar)); // bar is here.
2015-11-28 23:54:58 +01:00
/// m.revert(); // revert the last two operations.
/// assert!(m.contains(&foo)); // foo is here.
/// assert!(!m.contains(&bar)); // bar is gone.
2015-11-28 23:54:58 +01:00
/// }
/// ```
2015-11-28 22:54:12 +01:00
pub fn revert(&mut self) { self.overlay.clear(); }
2016-03-13 21:21:30 +01:00
/// Get the number of references that would be committed.
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(&key).map_or(0, |&(_, refs)| refs) }
2015-11-28 03:09:36 +01:00
/// Get the refs and value of the given key.
2015-11-28 19:28:59 +01:00
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
self.backing.get(&key.bytes())
2015-11-28 19:03:50 +01:00
.expect("Low-level database error. Some issue with your hard disk?")
.map(|d| {
2015-11-28 19:28:59 +01:00
let r = Rlp::new(d.deref());
2015-12-08 00:45:04 +01:00
(r.at(1).as_val(), r.at(0).as_val())
2015-11-28 19:03:50 +01:00
})
2015-11-28 03:09:36 +01:00
}
/// Put the refs and value of the given key, possibly deleting it from the db.
2016-03-13 19:22:42 +01:00
fn put_payload_in_batch(&self, batch: &DBTransaction, key: &H256, payload: (Bytes, u32)) -> bool {
2016-01-18 13:54:46 +01:00
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
2016-03-13 18:07:36 +01:00
batch.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
2016-01-18 14:44:06 +01:00
false
2016-01-18 13:54:46 +01:00
} else {
2016-03-13 18:07:36 +01:00
batch.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
2016-01-18 14:44:06 +01:00
true
2016-01-18 13:54:46 +01:00
}
}
2016-03-13 19:22:42 +01:00
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool {
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
self.backing.put(&key.bytes(), s.as_raw()).expect("Low-level database error. Some issue with your hard disk?");
false
} else {
self.backing.delete(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?");
true
}
}
2015-11-28 03:09:36 +01:00
}
impl HashDB for OverlayDB {
2015-12-04 18:05:59 +01:00
fn keys(&self) -> HashMap<H256, i32> {
let mut ret: HashMap<H256, i32> = HashMap::new();
2016-02-18 03:46:24 +01:00
for (key, _) in self.backing.iter() {
let h = H256::from_slice(key.deref());
let r = self.payload(&h).unwrap().1;
2015-12-04 18:05:59 +01:00
ret.insert(h, r as i32);
}
2015-12-04 18:05:59 +01:00
for (key, refs) in self.overlay.keys().into_iter() {
let refs = *ret.get(&key).unwrap_or(&0) + refs;
ret.insert(key, refs);
}
ret
}
fn get(&self, key: &H256) -> Option<&[u8]> {
// return ok if positive; if negative, check backing - might be enough references there to make
2015-11-28 03:09:36 +01:00
// it positive again.
let k = self.overlay.raw(key);
2015-11-28 03:09:36 +01:00
match k {
2015-11-29 15:50:33 +01:00
Some(&(ref d, rc)) if rc > 0 => Some(d),
2015-11-28 03:09:36 +01:00
_ => {
2016-01-19 12:14:29 +01:00
let memrc = k.map_or(0, |&(_, rc)| rc);
2015-11-28 03:09:36 +01:00
match self.payload(key) {
Some(x) => {
let (d, rc) = x;
2015-11-28 19:28:59 +01:00
if rc as i32 + memrc > 0 {
2015-11-29 15:50:33 +01:00
Some(&self.overlay.denote(key, d).0)
}
else {
None
}
}
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => Some(d),
2015-11-28 03:09:36 +01:00
_ => None,
}
}
}
}
fn contains(&self, key: &H256) -> bool {
// return ok if positive; if negative, check backing - might be enough references there to make
// it positive again.
let k = self.overlay.raw(key);
match k {
2015-11-28 19:28:59 +01:00
Some(&(_, rc)) if rc > 0 => true,
_ => {
2016-01-19 12:14:29 +01:00
let memrc = k.map_or(0, |&(_, rc)| rc);
match self.payload(key) {
Some(x) => {
2015-11-28 19:28:59 +01:00
let (_, rc) = x;
2016-01-19 12:14:29 +01:00
rc as i32 + memrc > 0
}
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => true,
_ => false,
}
}
}
2015-11-28 03:09:36 +01:00
}
fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) }
2015-11-30 02:57:02 +01:00
fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); }
fn remove(&mut self, key: &H256) { self.overlay.remove(key); }
2015-11-28 03:09:36 +01:00
}
2015-11-28 22:54:12 +01:00
#[test]
fn overlaydb_overlay_insert_and_remove() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), b"hello world");
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_insert_revert() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), b"hello world");
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
2015-11-28 22:54:12 +01:00
trie.revert();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_remove() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.revert();
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_remove_revert() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.revert();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_negative() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
trie.remove(&h); //bad - sends us into negative refs.
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
assert!(trie.commit().is_err());
}
#[test]
fn overlaydb_complex() {
let mut trie = OverlayDB::new_temp();
let hfoo = trie.insert(b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
2015-11-28 22:54:12 +01:00
let hbar = trie.insert(b"bar");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
2015-11-28 22:54:12 +01:00
trie.insert(b"foo"); // two refs
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
trie.remove(&hbar); // zero refs - delete
assert_eq!(trie.get(&hbar), None);
trie.remove(&hfoo); // one ref - keep
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
trie.remove(&hfoo); // zero ref - would delete, but...
assert_eq!(trie.get(&hfoo), None);
2015-11-28 22:54:12 +01:00
trie.insert(b"foo"); // one ref - keep after all.
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
trie.remove(&hfoo); // zero ref - delete
assert_eq!(trie.get(&hfoo), None);
trie.commit().unwrap(); //
assert_eq!(trie.get(&hfoo), None);
2015-11-28 22:54:12 +01:00
}
2015-11-28 03:09:36 +01:00
#[test]
fn playpen() {
2015-11-28 22:54:12 +01:00
use std::fs;
{
2016-02-18 03:46:24 +01:00
let db: Database = Database::open_default("/tmp/test").unwrap();
2015-11-28 22:54:12 +01:00
db.put(b"test", b"test2").unwrap();
match db.get(b"test") {
Ok(Some(value)) => println!("Got value {:?}", value.deref()),
Ok(None) => println!("No value for that key"),
Err(..) => println!("Gah"),
}
db.delete(b"test").unwrap();
2015-11-28 03:09:36 +01:00
}
2015-11-28 22:54:12 +01:00
fs::remove_dir_all("/tmp/test").unwrap();
2015-11-29 09:16:15 +01:00
}