openethereum/util/journaldb/src/overlaydb.rs

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

361 lines
12 KiB
Rust
Raw Normal View History

// Copyright 2015-2019 Parity Technologies (UK) Ltd.
// This file is part of Parity Ethereum.
2016-02-05 13:40:41 +01:00
// Parity Ethereum is free software: you can redistribute it and/or modify
2016-02-05 13:40:41 +01:00
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity Ethereum is distributed in the hope that it will be useful,
2016-02-05 13:40:41 +01:00
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
2016-02-05 13:40:41 +01:00
2016-04-06 10:07:24 +02:00
//! Disk-backed `HashDB` implementation.
2015-11-28 03:09:36 +01:00
2017-08-26 19:09:32 +02:00
use std::{
collections::{hash_map::Entry, HashMap},
io,
sync::Arc,
};
use super::error_negatively_reference_hash;
use ethereum_types::H256;
use hash_db::HashDB;
use keccak_hasher::KeccakHasher;
use kvdb::{DBTransaction, DBValue, KeyValueDB};
use memory_db::*;
use rlp::{decode, encode, Decodable, DecoderError, Encodable, Rlp, RlpStream};
2015-11-28 03:09:36 +01:00
2016-04-06 10:07:24 +02:00
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay.
2015-11-28 23:49:07 +01:00
///
2016-01-18 12:41:31 +01:00
/// The operations `insert()` and `remove()` take place on the memory overlay; batches of
2015-11-28 23:49:07 +01:00
/// such operations may be flushed to the disk-backed DB with `commit()` or discarded with
/// `revert()`.
///
/// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()`
2015-11-28 23:49:07 +01:00
/// queries have an immediate effect in terms of these functions.
2016-03-13 18:07:36 +01:00
#[derive(Clone)]
2015-11-28 03:09:36 +01:00
pub struct OverlayDB {
overlay: MemoryDB<KeccakHasher, DBValue>,
2020-07-29 10:36:15 +02:00
backing: Arc<dyn KeyValueDB>,
column: Option<u32>,
2015-11-28 03:09:36 +01:00
}
struct Payload {
count: u32,
value: DBValue,
}
impl Payload {
fn new(count: u32, value: DBValue) -> Self {
Payload { count, value }
}
}
impl Encodable for Payload {
fn rlp_append(&self, s: &mut RlpStream) {
s.begin_list(2);
s.append(&self.count);
s.append(&&*self.value);
}
}
impl Decodable for Payload {
fn decode(rlp: &Rlp) -> Result<Self, DecoderError> {
let payload = Payload {
count: rlp.val_at(0)?,
value: DBValue::from_slice(rlp.at(1)?.data()?),
};
2020-08-05 06:08:03 +02:00
Ok(payload)
}
}
2015-11-28 03:09:36 +01:00
impl OverlayDB {
/// Create a new instance of OverlayDB given a `backing` database.
2020-07-29 10:36:15 +02:00
pub fn new(backing: Arc<dyn KeyValueDB>, col: Option<u32>) -> OverlayDB {
OverlayDB {
overlay: ::new_memory_db(),
backing: backing,
column: col,
2015-11-28 03:09:36 +01:00
}
2020-08-05 06:08:03 +02:00
}
2015-11-28 22:54:12 +01:00
/// Create a new instance of OverlayDB with an anonymous temporary database.
#[cfg(test)]
2015-11-28 22:54:12 +01:00
pub fn new_temp() -> OverlayDB {
let backing = Arc::new(::kvdb_memorydb::create(0));
Self::new(backing, None)
}
2020-08-05 06:08:03 +02:00
/// Commit all operations in a single batch.
#[cfg(test)]
pub fn commit(&mut self) -> io::Result<u32> {
let mut batch = self.backing.transaction();
let res = self.commit_to_batch(&mut batch)?;
self.backing.write(batch).map(|_| res).map_err(|e| e.into())
2015-11-28 22:54:12 +01:00
}
2020-08-05 06:08:03 +02:00
2016-03-13 18:07:36 +01:00
/// Commit all operations to given batch.
pub fn commit_to_batch(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
2016-03-13 18:07:36 +01:00
let mut ret = 0u32;
let mut deletes = 0usize;
for i in self.overlay.drain() {
2016-03-13 18:07:36 +01:00
let (key, (value, rc)) = i;
if rc != 0 {
match self.payload(&key) {
Some(x) => {
let total_rc: i32 = x.count as i32 + rc;
2016-03-13 18:07:36 +01:00
if total_rc < 0 {
return Err(error_negatively_reference_hash(&key));
2016-03-13 18:07:36 +01:00
}
let payload = Payload::new(total_rc as u32, x.value);
deletes += if self.put_payload_in_batch(batch, &key, &payload) {
2020-08-05 06:08:03 +02:00
1
} else {
2020-08-05 06:08:03 +02:00
0
};
2016-03-13 18:07:36 +01:00
}
None => {
if rc < 0 {
return Err(error_negatively_reference_hash(&key));
2016-03-13 18:07:36 +01:00
}
let payload = Payload::new(rc as u32, value);
self.put_payload_in_batch(batch, &key, &payload);
2016-03-13 18:07:36 +01:00
}
};
ret += 1;
2020-08-05 06:08:03 +02:00
}
2016-03-13 18:07:36 +01:00
}
trace!("OverlayDB::commit() deleted {} nodes", deletes);
Ok(ret)
}
2020-08-05 06:08:03 +02:00
/// Revert all operations on this object (i.e. `insert()`s and `remove()`s) since the
2015-11-28 23:49:07 +01:00
/// last `commit()`.
2015-11-28 22:54:12 +01:00
pub fn revert(&mut self) {
self.overlay.clear();
}
2020-08-05 06:08:03 +02:00
2016-03-13 21:21:30 +01:00
/// Get the number of references that would be committed.
pub fn commit_refs(&self, key: &H256) -> i32 {
self.overlay.raw(key).map_or(0, |(_, refs)| refs)
}
2020-08-05 06:08:03 +02:00
2015-11-28 03:09:36 +01:00
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<Payload> {
self.backing
.get(self.column, key)
2015-11-28 19:03:50 +01:00
.expect("Low-level database error. Some issue with your hard disk?")
.map(|ref d| decode(d).expect("decoding db value failed"))
2015-11-28 03:09:36 +01:00
}
2020-08-05 06:08:03 +02:00
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload_in_batch(
&self,
batch: &mut DBTransaction,
key: &H256,
payload: &Payload,
) -> bool {
if payload.count > 0 {
batch.put(self.column, key, &encode(payload));
2016-01-18 14:44:06 +01:00
false
2016-01-18 13:54:46 +01:00
} else {
batch.delete(self.column, key);
2016-03-13 19:22:42 +01:00
true
}
2020-08-05 06:08:03 +02:00
}
2015-11-28 03:09:36 +01:00
}
impl crate::KeyedHashDB for OverlayDB {
2015-12-04 18:05:59 +01:00
fn keys(&self) -> HashMap<H256, i32> {
2017-08-26 19:09:32 +02:00
let mut ret: HashMap<H256, i32> = self
.backing
.iter(self.column)
.map(|(key, _)| {
let h = H256::from_slice(&*key);
let r = self.payload(&h).unwrap().count;
2017-08-26 19:09:32 +02:00
(h, r as i32)
})
.collect();
2020-08-05 06:08:03 +02:00
for (key, refs) in self.overlay.keys() {
2017-08-26 19:09:32 +02:00
match ret.entry(key) {
Entry::Occupied(mut entry) => {
*entry.get_mut() += refs;
}
Entry::Vacant(entry) => {
2017-08-26 19:16:08 +02:00
entry.insert(refs);
2017-08-26 19:09:32 +02:00
}
}
2020-08-05 06:08:03 +02:00
}
ret
}
}
impl HashDB<KeccakHasher, DBValue> for OverlayDB {
2016-10-26 13:53:47 +02:00
fn get(&self, key: &H256) -> Option<DBValue> {
// return ok if positive; if negative, check backing - might be enough references there to make
2015-11-28 03:09:36 +01:00
// it positive again.
let k = self.overlay.raw(key);
2016-10-26 13:53:47 +02:00
let memrc = {
if let Some((d, rc)) = k {
if rc > 0 {
return Some(d.clone());
}
2016-10-26 13:53:47 +02:00
rc
} else {
0
}
};
match self.payload(key) {
Some(x) => {
if x.count as i32 + memrc > 0 {
Some(x.value)
2016-10-26 13:53:47 +02:00
} else {
None
2020-08-05 06:08:03 +02:00
}
2015-11-28 03:09:36 +01:00
}
2016-10-26 13:53:47 +02:00
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => Some(d),
_ => None,
2020-08-05 06:08:03 +02:00
}
2015-11-28 03:09:36 +01:00
}
2020-08-05 06:08:03 +02:00
fn contains(&self, key: &H256) -> bool {
// return ok if positive; if negative, check backing - might be enough references there to make
// it positive again.
let k = self.overlay.raw(key);
match k {
Some((_, rc)) if rc > 0 => true,
_ => {
let memrc = k.map_or(0, |(_, rc)| rc);
match self.payload(key) {
Some(x) => x.count as i32 + memrc > 0,
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => true,
_ => false,
2020-08-05 06:08:03 +02:00
}
}
}
}
2020-08-05 06:08:03 +02:00
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
}
2016-10-26 13:53:47 +02:00
fn emplace(&mut self, key: H256, value: DBValue) {
self.overlay.emplace(key, value);
}
fn remove(&mut self, key: &H256) {
self.overlay.remove(key);
}
2015-11-28 03:09:36 +01:00
}
#[test]
fn overlaydb_revert() {
let mut m = OverlayDB::new_temp();
let foo = m.insert(b"foo"); // insert foo.
let mut batch = m.backing.transaction();
m.commit_to_batch(&mut batch).unwrap(); // commit - new operations begin here...
m.backing.write(batch).unwrap();
let bar = m.insert(b"bar"); // insert bar.
m.remove(&foo); // remove foo.
assert!(!m.contains(&foo)); // foo is gone.
assert!(m.contains(&bar)); // bar is here.
m.revert(); // revert the last two operations.
assert!(m.contains(&foo)); // foo is here.
assert!(!m.contains(&bar)); // bar is gone.
}
2015-11-28 22:54:12 +01:00
#[test]
fn overlaydb_overlay_insert_and_remove() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_insert_revert() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
2015-11-28 22:54:12 +01:00
trie.revert();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_remove() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.revert();
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_backing_remove_revert() {
2015-11-28 22:54:12 +01:00
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
trie.revert();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
2015-11-28 22:54:12 +01:00
}
#[test]
fn overlaydb_negative() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
trie.commit().unwrap();
trie.remove(&h);
trie.remove(&h); //bad - sends us into negative refs.
assert_eq!(trie.get(&h), None);
2015-11-28 22:54:12 +01:00
assert!(trie.commit().is_err());
}
#[test]
fn overlaydb_complex() {
let mut trie = OverlayDB::new_temp();
let hfoo = trie.insert(b"foo");
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
2015-11-28 22:54:12 +01:00
let hbar = trie.insert(b"bar");
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
2015-11-28 22:54:12 +01:00
trie.insert(b"foo"); // two refs
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.remove(&hbar); // zero refs - delete
assert_eq!(trie.get(&hbar), None);
trie.remove(&hfoo); // one ref - keep
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - would delete, but...
assert_eq!(trie.get(&hfoo), None);
2015-11-28 22:54:12 +01:00
trie.insert(b"foo"); // one ref - keep after all.
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
2015-11-28 22:54:12 +01:00
trie.commit().unwrap();
2016-10-26 13:53:47 +02:00
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - delete
assert_eq!(trie.get(&hfoo), None);
trie.commit().unwrap(); //
assert_eq!(trie.get(&hfoo), None);
2015-11-28 22:54:12 +01:00
}