fatdb and fatdb iterator module

This commit is contained in:
debris 2016-06-27 09:16:34 +02:00
parent 54fc5a533b
commit 1b4f67151f
9 changed files with 191 additions and 13 deletions

View File

@ -104,6 +104,16 @@ pub trait HashDB: AsHashDB {
/// } /// }
/// ``` /// ```
fn remove(&mut self, key: &H256); fn remove(&mut self, key: &H256);
/// Insert auxiliary data into hashdb.
fn insert_aux(&mut self, _hash: Vec<u8>, _value: Vec<u8>) {
unimplemented!();
}
/// Get auxiliary data from hashdb.
fn get_aux(&self, _hash: &[u8]) -> Option<Vec<u8>> {
unimplemented!();
}
} }
/// Upcast trait. /// Upcast trait.

View File

@ -26,6 +26,8 @@ use kvdb::{Database, DBTransaction, DatabaseConfig};
#[cfg(test)] #[cfg(test)]
use std::env; use std::env;
const AUX_FLAG: u8 = 255;
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
/// and latent-removal semantics. /// and latent-removal semantics.
/// ///
@ -126,6 +128,23 @@ impl HashDB for ArchiveDB {
fn remove(&mut self, key: &H256) { fn remove(&mut self, key: &H256) {
self.overlay.remove(key); self.overlay.remove(key);
} }
fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
self.overlay.insert_aux(hash, value);
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
if let Some(res) = self.overlay.get_aux(hash) {
return Some(res)
}
let mut db_hash = hash.to_vec();
db_hash.push(AUX_FLAG);
self.backing.get(&db_hash)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|v| v.to_vec())
}
} }
impl JournalDB for ArchiveDB { impl JournalDB for ArchiveDB {
@ -149,6 +168,7 @@ impl JournalDB for ArchiveDB {
let batch = DBTransaction::new(); let batch = DBTransaction::new();
let mut inserts = 0usize; let mut inserts = 0usize;
let mut deletes = 0usize; let mut deletes = 0usize;
for i in self.overlay.drain().into_iter() { for i in self.overlay.drain().into_iter() {
let (key, (value, rc)) = i; let (key, (value, rc)) = i;
if rc > 0 { if rc > 0 {
@ -161,6 +181,12 @@ impl JournalDB for ArchiveDB {
deletes += 1; deletes += 1;
} }
} }
for (mut key, value) in self.overlay.drain_aux().into_iter() {
key.push(AUX_FLAG);
batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
}
if self.latest_era.map_or(true, |e| now > e) { if self.latest_era.map_or(true, |e| now > e) {
try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
self.latest_era = Some(now); self.latest_era = Some(now);

View File

@ -74,6 +74,7 @@ use std::default::Default;
pub struct MemoryDB { pub struct MemoryDB {
data: HashMap<H256, (Bytes, i32)>, data: HashMap<H256, (Bytes, i32)>,
static_null_rlp: (Bytes, i32), static_null_rlp: (Bytes, i32),
aux: HashMap<Bytes, Bytes>,
} }
impl Default for MemoryDB { impl Default for MemoryDB {
@ -88,6 +89,7 @@ impl MemoryDB {
MemoryDB { MemoryDB {
data: HashMap::new(), data: HashMap::new(),
static_null_rlp: (vec![0x80u8; 1], 1), static_null_rlp: (vec![0x80u8; 1], 1),
aux: HashMap::new(),
} }
} }
@ -139,6 +141,13 @@ impl MemoryDB {
data data
} }
/// Return the internal map of auxiliary data, clearing the current state.
pub fn drain_aux(&mut self) -> HashMap<Bytes, Bytes> {
let mut aux = HashMap::new();
mem::swap(&mut self.aux, &mut aux);
aux
}
/// Denote than an existing value has the given key. Used when a key gets removed without /// Denote than an existing value has the given key. Used when a key gets removed without
/// a prior insert and thus has a negative reference with no value. /// a prior insert and thus has a negative reference with no value.
/// ///
@ -233,6 +242,14 @@ impl HashDB for MemoryDB {
self.data.insert(key.clone(), (Bytes::new(), -1)); self.data.insert(key.clone(), (Bytes::new(), -1));
} }
} }
fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
self.aux.insert(hash, value);
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
self.aux.get(hash).cloned()
}
} }
#[test] #[test]

122
util/src/trie/fatdb.rs Normal file
View File

@ -0,0 +1,122 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use super::{TrieDBMut, Trie, TrieDB, TrieMut, TrieDBIterator, TrieError};
/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
///
/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object.
pub struct FatDB<'db> {
raw: TrieDBMut<'db>,
}
impl<'db> FatDB<'db> {
/// Create a new trie with the backing database `db` and empty `root`
/// Initialise to the state entailed by the genesis block.
/// This guarantees the trie is built correctly.
pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
FatDB { raw: TrieDBMut::new(db, root) }
}
/// Create a new trie with the backing database `db` and `root`.
///
/// Returns an error if root does not exist.
pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result<Self, TrieError> {
Ok(FatDB { raw: try!(TrieDBMut::from_existing(db, root)) })
}
/// Get the backing database.
pub fn db(&self) -> &HashDB {
self.raw.db()
}
/// Get the backing database.
pub fn db_mut(&mut self) -> &mut HashDB {
self.raw.db_mut()
}
}
impl<'db> Trie for FatDB<'db> {
fn root(&self) -> &H256 {
self.raw.root()
}
fn contains(&self, key: &[u8]) -> bool {
self.raw.contains(&key.sha3())
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
self.raw.get(&key.sha3())
}
}
impl<'db> TrieMut for FatDB<'db> {
fn insert(&mut self, key: &[u8], value: &[u8]) {
let hash = key.sha3();
self.raw.insert(&hash, value);
let db = self.raw.db_mut();
db.insert_aux(hash.to_vec(), key.to_vec());
}
fn remove(&mut self, key: &[u8]) {
self.raw.remove(&key.sha3());
}
}
/// Itarator over inserted pairs of key values.
pub struct FatDBIterator<'db> {
trie_iterator: TrieDBIterator<'db>,
trie: &'db TrieDB<'db>,
}
impl<'db> FatDBIterator<'db> {
pub fn new(trie: &'db TrieDB) -> Self {
FatDBIterator {
trie_iterator: TrieDBIterator::new(trie),
trie: trie,
}
}
}
impl<'db> Iterator for FatDBIterator<'db> {
type Item = (Vec<u8>, &'db [u8]);
fn next(&mut self) -> Option<Self::Item> {
self.trie_iterator.next()
.map(|(hash, value)| {
(self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value)
})
}
}
#[test]
fn fatdb_to_trie() {
use memorydb::MemoryDB;
use super::TrieDB;
let mut memdb = MemoryDB::new();
let mut root = H256::default();
{
let mut t = FatDB::new(&mut memdb, &mut root);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
}
let t = TrieDB::new(&memdb, &root).unwrap();
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]);
assert_eq!(FatDBIterator::new(&t).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]);
}

View File

@ -35,12 +35,15 @@ pub mod sectriedb;
/// Export the sectriedbmut module. /// Export the sectriedbmut module.
pub mod sectriedbmut; pub mod sectriedbmut;
mod fatdb;
pub use self::trietraits::{Trie, TrieMut}; pub use self::trietraits::{Trie, TrieMut};
pub use self::standardmap::{Alphabet, StandardMap, ValueMode}; pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
pub use self::triedbmut::TrieDBMut; pub use self::triedbmut::TrieDBMut;
pub use self::triedb::TrieDB; pub use self::triedb::{TrieDB, TrieDBIterator};
pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedbmut::SecTrieDBMut;
pub use self::sectriedb::SecTrieDB; pub use self::sectriedb::SecTrieDB;
pub use self::fatdb::{FatDB, FatDBIterator};
/// Trie Errors /// Trie Errors
#[derive(Debug)] #[derive(Debug)]

View File

@ -14,8 +14,8 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>. // along with Parity. If not, see <http://www.gnu.org/licenses/>.
use hash::*; use hash::H256;
use sha3::*; use sha3::Hashable;
use hashdb::HashDB; use hashdb::HashDB;
use super::triedb::TrieDB; use super::triedb::TrieDB;
use super::trietraits::Trie; use super::trietraits::Trie;
@ -68,7 +68,7 @@ fn trie_to_sectrie() {
use super::trietraits::TrieMut; use super::trietraits::TrieMut;
let mut memdb = MemoryDB::new(); let mut memdb = MemoryDB::new();
let mut root = H256::new(); let mut root = H256::default();
{ {
let mut t = TrieDBMut::new(&mut memdb, &mut root); let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]); t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]);

View File

@ -14,8 +14,8 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>. // along with Parity. If not, see <http://www.gnu.org/licenses/>.
use hash::*; use hash::H256;
use sha3::*; use sha3::Hashable;
use hashdb::HashDB; use hashdb::HashDB;
use super::triedbmut::TrieDBMut; use super::triedbmut::TrieDBMut;
use super::trietraits::{Trie, TrieMut}; use super::trietraits::{Trie, TrieMut};
@ -44,10 +44,10 @@ impl<'db> SecTrieDBMut<'db> {
} }
/// Get the backing database. /// Get the backing database.
pub fn db(&'db self) -> &'db HashDB { self.raw.db() } pub fn db(&self) -> &HashDB { self.raw.db() }
/// Get the backing database. /// Get the backing database.
pub fn db_mut(&'db mut self) -> &'db mut HashDB { self.raw.db_mut() } pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() }
} }
impl<'db> Trie for SecTrieDBMut<'db> { impl<'db> Trie for SecTrieDBMut<'db> {
@ -78,7 +78,7 @@ fn sectrie_to_trie() {
use super::triedb::*; use super::triedb::*;
let mut memdb = MemoryDB::new(); let mut memdb = MemoryDB::new();
let mut root = H256::new(); let mut root = H256::default();
{ {
let mut t = SecTrieDBMut::new(&mut memdb, &mut root); let mut t = SecTrieDBMut::new(&mut memdb, &mut root);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);

View File

@ -257,7 +257,7 @@ pub struct TrieDBIterator<'a> {
impl<'a> TrieDBIterator<'a> { impl<'a> TrieDBIterator<'a> {
/// Create a new iterator. /// Create a new iterator.
fn new(db: &'a TrieDB) -> TrieDBIterator<'a> { pub fn new(db: &'a TrieDB) -> TrieDBIterator<'a> {
let mut r = TrieDBIterator { let mut r = TrieDBIterator {
db: db, db: db,
trail: vec![], trail: vec![],

View File

@ -99,12 +99,12 @@ impl<'db> TrieDBMut<'db> {
} }
/// Get the backing database. /// Get the backing database.
pub fn db(&'db self) -> &'db HashDB { pub fn db(&self) -> &HashDB {
self.db self.db
} }
/// Get the backing database. /// Get the backing database.
pub fn db_mut(&'db mut self) -> &'db mut HashDB { pub fn db_mut(&mut self) -> &mut HashDB {
self.db self.db
} }