grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove calls to heapsize (#10432)

Browse Source 
* update memorydb trait
* use malloc_size_of instead of heapsize_of
* use jemalloc as default allocator for parity client.
This commit is contained in:
cheme
2019-06-19 13:54:05 +02:00
committed by GitHub
parent 859a41308c
commit 6fc5014b4d
84 changed files with 926 additions and 1074 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
util/journaldb/Cargo.toml
View File

@@ -8,12 +8,12 @@ license = "GPL3"
[dependencies]
parity-bytes = "0.1"
ethereum-types = "0.6.0"
hash-db = "0.11.0"
heapsize = "0.4"
hash-db = "0.12.4"
parity-util-mem = "0.1"
keccak-hasher = { path = "../keccak-hasher" }
kvdb = "0.1"
log = "0.4"
memory-db = "0.11.0"
memory-db = "0.12.4"
parking_lot = "0.7"
fastmap = { path = "../../util/fastmap" }
rlp = "0.4.0"

180
util/journaldb/src/archivedb.rs
View File

@@ -23,12 +23,12 @@ use std::sync::Arc;
use bytes::Bytes;
use ethereum_types::H256;
use hash_db::{HashDB};
use parity_util_mem::MallocSizeOfExt;
use hash_db::{HashDB, Prefix};
use keccak_hasher::KeccakHasher;
use kvdb::{KeyValueDB, DBTransaction, DBValue};
use rlp::{encode, decode};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, error_key_already_exists, error_negatively_reference_hash};
use super::memory_db::*;
use traits::JournalDB;
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
@@ -39,7 +39,7 @@ use traits::JournalDB;
/// immediately. As this is an "archive" database, nothing is ever removed. This means
/// that the states of any block the node has ever processed will be accessible.
pub struct ArchiveDB {
overlay: MemoryDB<KeccakHasher, DBValue>,
overlay: super::MemoryDB,
backing: Arc<KeyValueDB>,
latest_era: Option<u64>,
column: Option<u32>,
@@ -66,8 +66,8 @@ impl ArchiveDB {
}
impl HashDB<KeccakHasher, DBValue> for ArchiveDB {
fn get(&self, key: &H256) -> Option<DBValue> {
if let Some((d, rc)) = self.overlay.raw(key) {
fn get(&self, key: &H256, prefix: Prefix) -> Option<DBValue> {
if let Some((d, rc)) = self.overlay.raw(key, prefix) {
if rc > 0 {
return Some(d.clone());
}
@@ -75,20 +75,20 @@ impl HashDB<KeccakHasher, DBValue> for ArchiveDB {
self.payload(key)
}
fn contains(&self, key: &H256) -> bool {
self.get(key).is_some()
fn contains(&self, key: &H256, prefix: Prefix) -> bool {
self.get(key, prefix).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 {
self.overlay.insert(prefix, value)
}
fn emplace(&mut self, key: H256, value: DBValue) {
self.overlay.emplace(key, value);
fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) {
self.overlay.emplace(key, prefix, value);
}
fn remove(&mut self, key: &H256) {
self.overlay.remove(key);
fn remove(&mut self, key: &H256, prefix: Prefix) {
self.overlay.remove(key, prefix);
}
}
@@ -124,7 +124,7 @@ impl JournalDB for ArchiveDB {
}
fn mem_used(&self) -> usize {
self.overlay.mem_used()
self.overlay.malloc_size_of()
}
fn is_empty(&self) -> bool {
@@ -197,7 +197,7 @@ impl JournalDB for ArchiveDB {
&self.backing
}
fn consolidate(&mut self, with: MemoryDB<KeccakHasher, DBValue>) {
fn consolidate(&mut self, with: super::MemoryDB) {
self.overlay.consolidate(with);
}
}
@@ -206,7 +206,7 @@ impl JournalDB for ArchiveDB {
mod tests {
use keccak::keccak;
use hash_db::HashDB;
use hash_db::{HashDB, EMPTY_PREFIX};
use super::*;
use {kvdb_memorydb, JournalDB};
@@ -215,50 +215,50 @@ mod tests {
// history is 1
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
jdb.commit_batch(3, &keccak(b"1002a"), Some((1, keccak(b"1")))).unwrap();
jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap();
jdb.remove(&x);
jdb.remove(&x, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap();
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap();
jdb.commit_batch(5, &keccak(b"1004a"), Some((3, keccak(b"1002a")))).unwrap();
jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap();
assert!(jdb.contains(&x));
assert!(jdb.contains(&x, EMPTY_PREFIX));
}
#[test]
fn long_history() {
// history is 3
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&h));
jdb.remove(&h);
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.remove(&h, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
}
#[test]
#[should_panic]
fn multiple_owed_removal_not_allowed() {
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&h));
jdb.remove(&h);
jdb.remove(&h);
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.remove(&h, EMPTY_PREFIX);
jdb.remove(&h, EMPTY_PREFIX);
// commit_batch would call journal_under(),
// and we don't allow multiple owned removals.
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
@@ -269,29 +269,29 @@ mod tests {
// history is 1
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.remove(&baz, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
}
@@ -301,25 +301,25 @@ mod tests {
// history is 1
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -327,18 +327,18 @@ mod tests {
// history is 1
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.insert(EMPTY_PREFIX, b"foo");
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -347,15 +347,15 @@ mod tests {
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -366,22 +366,22 @@ mod tests {
let foo = {
let mut jdb = ArchiveDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(shared_db.clone(), None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
}
{
let mut jdb = ArchiveDB::new(shared_db, None);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
}
}
@@ -393,23 +393,23 @@ mod tests {
let foo = {
let mut jdb = ArchiveDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
// foo is ancient history.
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
foo
};
{
let mut jdb = ArchiveDB::new(shared_db, None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.contains(&foo));
jdb.remove(&foo);
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap();
}
@@ -421,14 +421,14 @@ mod tests {
let (foo, _, _) = {
let mut jdb = ArchiveDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
(foo, bar, baz)
};
@@ -436,7 +436,7 @@ mod tests {
{
let mut jdb = ArchiveDB::new(shared_db, None);
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
}
@@ -446,7 +446,7 @@ mod tests {
let key = {
let mut jdb = ArchiveDB::new(shared_db.clone(), None);
let key = jdb.insert(b"foo");
let key = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
key
};
@@ -461,13 +461,13 @@ mod tests {
#[test]
fn inject() {
let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None);
let key = jdb.insert(b"dog");
let key = jdb.insert(EMPTY_PREFIX, b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key, EMPTY_PREFIX);
jdb.inject_batch().unwrap();
assert!(jdb.get(&key).is_none());
assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
}
}

296
util/journaldb/src/earlymergedb.rs
View File

@@ -23,27 +23,22 @@ use std::sync::Arc;
use bytes::Bytes;
use ethereum_types::H256;
use hash_db::{HashDB};
use heapsize::HeapSizeOf;
use hash_db::{HashDB, Prefix};
use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops};
use keccak_hasher::KeccakHasher;
use kvdb::{KeyValueDB, DBTransaction, DBValue};
use memory_db::*;
use parking_lot::RwLock;
use rlp::{encode, decode};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, error_negatively_reference_hash, error_key_already_exists};
use super::traits::JournalDB;
use util::{DatabaseKey, DatabaseValueView, DatabaseValueRef};
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)]
struct RefInfo {
queue_refs: usize,
in_archive: bool,
}
impl HeapSizeOf for RefInfo {
fn heap_size_of_children(&self) -> usize { 0 }
}
#[derive(Clone, PartialEq, Eq)]
enum RemoveFrom {
Queue,
@@ -107,7 +102,7 @@ enum RemoveFrom {
///
/// TODO: `store_reclaim_period`
pub struct EarlyMergeDB {
overlay: MemoryDB<KeccakHasher, DBValue>,
overlay: super::MemoryDB,
backing: Arc<KeyValueDB>,
refs: Option<Arc<RwLock<HashMap<H256, RefInfo>>>>,
latest_era: Option<u64>,
@@ -292,8 +287,8 @@ impl EarlyMergeDB {
}
impl HashDB<KeccakHasher, DBValue> for EarlyMergeDB {
fn get(&self, key: &H256) -> Option<DBValue> {
if let Some((d, rc)) = self.overlay.raw(key) {
fn get(&self, key: &H256, prefix: Prefix) -> Option<DBValue> {
if let Some((d, rc)) = self.overlay.raw(key, prefix) {
if rc > 0 {
return Some(d.clone())
}
@@ -301,18 +296,18 @@ impl HashDB<KeccakHasher, DBValue> for EarlyMergeDB {
self.payload(key)
}
fn contains(&self, key: &H256) -> bool {
self.get(key).is_some()
fn contains(&self, key: &H256, prefix: Prefix) -> bool {
self.get(key, prefix).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 {
self.overlay.insert(prefix, value)
}
fn emplace(&mut self, key: H256, value: DBValue) {
self.overlay.emplace(key, value);
fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) {
self.overlay.emplace(key, prefix, value);
}
fn remove(&mut self, key: &H256) {
self.overlay.remove(key);
fn remove(&mut self, key: &H256, prefix: Prefix) {
self.overlay.remove(key, prefix);
}
}
@@ -358,8 +353,9 @@ impl JournalDB for EarlyMergeDB {
fn latest_era(&self) -> Option<u64> { self.latest_era }
fn mem_used(&self) -> usize {
self.overlay.mem_used() + match self.refs {
Some(ref c) => c.read().heap_size_of_children(),
let mut ops = new_malloc_size_ops();
self.overlay.size_of(&mut ops) + match self.refs {
Some(ref c) => c.read().size_of(&mut ops),
None => 0
}
}
@@ -520,7 +516,7 @@ impl JournalDB for EarlyMergeDB {
Ok(ops)
}
fn consolidate(&mut self, with: MemoryDB<KeccakHasher, DBValue>) {
fn consolidate(&mut self, with: super::MemoryDB) {
self.overlay.consolidate(with);
}
}
@@ -529,7 +525,7 @@ impl JournalDB for EarlyMergeDB {
mod tests {
use keccak::keccak;
use hash_db::HashDB;
use hash_db::{HashDB, EMPTY_PREFIX};
use super::*;
use super::super::traits::JournalDB;
use kvdb_memorydb;
@@ -539,7 +535,7 @@ mod tests {
// history is 1
let mut jdb = new_db();
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
@@ -549,10 +545,10 @@ mod tests {
jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&x);
jdb.remove(&x, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -561,50 +557,50 @@ mod tests {
jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&x));
assert!(jdb.contains(&x, EMPTY_PREFIX));
}
#[test]
fn insert_older_era() {
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let bar = jdb.insert(b"bar");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(0, &keccak(b"0b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
}
#[test]
fn long_history() {
// history is 3
let mut jdb = new_db();
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.remove(&h);
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.remove(&h, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&h));
assert!(!jdb.contains(&h, EMPTY_PREFIX));
}
#[test]
@@ -612,42 +608,42 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.remove(&baz, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
}
#[test]
@@ -655,31 +651,31 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
}
#[test]
@@ -687,22 +683,22 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.insert(EMPTY_PREFIX, b"foo");
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -712,23 +708,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -737,23 +733,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -762,23 +758,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3a"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3b"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -802,8 +798,8 @@ mod tests {
let foo = {
let mut jdb = EarlyMergeDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
@@ -811,18 +807,18 @@ mod tests {
{
let mut jdb = EarlyMergeDB::new(shared_db.clone(), None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = EarlyMergeDB::new(shared_db, None);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
}
@@ -833,19 +829,19 @@ mod tests {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
@@ -859,39 +855,39 @@ mod tests {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -905,28 +901,28 @@ mod tests {
let mut jdb = new_db();
// history is 1
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); // BROKEN
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -934,7 +930,7 @@ mod tests {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
@@ -948,16 +944,16 @@ mod tests {
// foo is ancient history.
jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(5, &keccak(b"5"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&bar);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(6, &keccak(b"6"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(b"bar");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(7, &keccak(b"7"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
@@ -972,7 +968,7 @@ mod tests {
{
let mut jdb = EarlyMergeDB::new(shared_db.clone(), None);
// history is 1
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
@@ -980,24 +976,24 @@ mod tests {
// foo is ancient history.
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
let mut jdb = EarlyMergeDB::new(shared_db.clone(), None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(4, &keccak(b"4"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
@@ -1005,7 +1001,7 @@ mod tests {
jdb.commit_batch(5, &keccak(b"5"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
@@ -1013,7 +1009,7 @@ mod tests {
jdb.commit_batch(6, &keccak(b"6"), Some((4, keccak(b"4")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
}
@@ -1024,16 +1020,16 @@ mod tests {
let (foo, bar, baz) = {
let mut jdb = EarlyMergeDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
(foo, bar, baz)
@@ -1043,22 +1039,22 @@ mod tests {
let mut jdb = EarlyMergeDB::new(shared_db, None);
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
}
}
#[test]
fn inject() {
let mut jdb = new_db();
let key = jdb.insert(b"dog");
let key = jdb.insert(EMPTY_PREFIX, b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key, EMPTY_PREFIX);
jdb.inject_batch().unwrap();
assert!(jdb.get(&key).is_none());
assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
}
}

16
util/journaldb/src/lib.rs
View File

@@ -16,7 +16,9 @@
//! `JournalDB` interface and implementation.
extern crate heapsize;
extern crate parity_util_mem;
extern crate parity_util_mem as mem;
extern crate parity_util_mem as malloc_size_of;
#[macro_use]
extern crate log;
@@ -59,6 +61,14 @@ pub use self::traits::KeyedHashDB;
/// Export as keyed hash trait
pub use self::traits::AsKeyedHashDB;
/// Alias to ethereum MemoryDB
type MemoryDB = memory_db::MemoryDB<
keccak_hasher::KeccakHasher,
memory_db::HashKey<keccak_hasher::KeccakHasher>,
kvdb::DBValue,
>;
/// Journal database operating strategy.
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum Algorithm {
@@ -163,8 +173,8 @@ fn error_negatively_reference_hash(hash: &ethereum_types::H256) -> io::Error {
io::Error::new(io::ErrorKind::Other, format!("Entry {} removed from database more times than it was added.", hash))
}
pub fn new_memory_db() -> memory_db::MemoryDB<keccak_hasher::KeccakHasher, kvdb::DBValue> {
memory_db::MemoryDB::from_null_node(&rlp::NULL_RLP, rlp::NULL_RLP.as_ref().into())
pub fn new_memory_db() -> MemoryDB {
MemoryDB::from_null_node(&rlp::NULL_RLP, rlp::NULL_RLP.as_ref().into())
}
#[cfg(test)]

123
util/journaldb/src/overlaydb.rs
View File

@@ -23,9 +23,8 @@ use std::sync::Arc;
use ethereum_types::H256;
use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable, encode, decode};
use hash_db::{HashDB};
use hash_db::{HashDB, Prefix, EMPTY_PREFIX};
use keccak_hasher::KeccakHasher;
use memory_db::*;
use kvdb::{KeyValueDB, DBTransaction, DBValue};
use super::{error_negatively_reference_hash};
@@ -39,7 +38,7 @@ use super::{error_negatively_reference_hash};
/// queries have an immediate effect in terms of these functions.
#[derive(Clone)]
pub struct OverlayDB {
overlay: MemoryDB<KeccakHasher, DBValue>,
overlay: super::MemoryDB,
backing: Arc<KeyValueDB>,
column: Option<u32>,
}
@@ -134,7 +133,7 @@ impl OverlayDB {
pub fn revert(&mut self) { self.overlay.clear(); }
/// Get the number of references that would be committed.
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key).map_or(0, |(_, refs)| refs) }
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key, EMPTY_PREFIX).map_or(0, |(_, refs)| refs) }
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<Payload> {
@@ -182,10 +181,10 @@ impl crate::KeyedHashDB for OverlayDB {
}
impl HashDB<KeccakHasher, DBValue> for OverlayDB {
fn get(&self, key: &H256) -> Option<DBValue> {
fn get(&self, key: &H256, prefix: Prefix) -> Option<DBValue> {
// return ok if positive; if negative, check backing - might be enough references there to make
// it positive again.
let k = self.overlay.raw(key);
let k = self.overlay.raw(key, prefix);
let memrc = {
if let Some((d, rc)) = k {
if rc > 0 { return Some(d.clone()); }
@@ -209,10 +208,10 @@ impl HashDB<KeccakHasher, DBValue> for OverlayDB {
}
}
fn contains(&self, key: &H256) -> bool {
fn contains(&self, key: &H256, prefix: Prefix) -> bool {
// return ok if positive; if negative, check backing - might be enough references there to make
// it positive again.
let k = self.overlay.raw(key);
let k = self.overlay.raw(key, prefix);
match k {
Some((_, rc)) if rc > 0 => true,
_ => {
@@ -229,111 +228,111 @@ impl HashDB<KeccakHasher, DBValue> for OverlayDB {
}
}
fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) }
fn emplace(&mut self, key: H256, value: DBValue) { self.overlay.emplace(key, value); }
fn remove(&mut self, key: &H256) { self.overlay.remove(key); }
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { self.overlay.insert(prefix, value) }
fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { self.overlay.emplace(key, prefix, value); }
fn remove(&mut self, key: &H256, prefix: Prefix) { self.overlay.remove(key, prefix); }
}
#[test]
fn overlaydb_revert() {
let mut m = OverlayDB::new_temp();
let foo = m.insert(b"foo"); // insert foo.
let foo = m.insert(EMPTY_PREFIX, b"foo"); // insert foo.
let mut batch = m.backing.transaction();
m.commit_to_batch(&mut batch).unwrap(); // commit - new operations begin here...
m.backing.write(batch).unwrap();
let bar = m.insert(b"bar"); // insert bar.
m.remove(&foo); // remove foo.
assert!(!m.contains(&foo)); // foo is gone.
assert!(m.contains(&bar)); // bar is here.
let bar = m.insert(EMPTY_PREFIX, b"bar"); // insert bar.
m.remove(&foo, EMPTY_PREFIX); // remove foo.
assert!(!m.contains(&foo, EMPTY_PREFIX)); // foo is gone.
assert!(m.contains(&bar, EMPTY_PREFIX)); // bar is here.
m.revert(); // revert the last two operations.
assert!(m.contains(&foo)); // foo is here.
assert!(!m.contains(&bar)); // bar is gone.
assert!(m.contains(&foo, EMPTY_PREFIX)); // foo is here.
assert!(!m.contains(&bar, EMPTY_PREFIX)); // bar is gone.
}
#[test]
fn overlaydb_overlay_insert_and_remove() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
trie.remove(&h);
assert_eq!(trie.get(&h), None);
let h = trie.insert(EMPTY_PREFIX, b"hello world");
assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world"));
trie.remove(&h, EMPTY_PREFIX);
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
}
#[test]
fn overlaydb_backing_insert_revert() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
let h = trie.insert(EMPTY_PREFIX, b"hello world");
assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world"));
trie.commit().unwrap();
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world"));
trie.revert();
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world"));
}
#[test]
fn overlaydb_backing_remove() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
let h = trie.insert(EMPTY_PREFIX, b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
trie.remove(&h, EMPTY_PREFIX);
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
trie.commit().unwrap();
assert_eq!(trie.get(&h), None);
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
trie.revert();
assert_eq!(trie.get(&h), None);
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
}
#[test]
fn overlaydb_backing_remove_revert() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
let h = trie.insert(EMPTY_PREFIX, b"hello world");
trie.commit().unwrap();
trie.remove(&h);
assert_eq!(trie.get(&h), None);
trie.remove(&h, EMPTY_PREFIX);
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
trie.revert();
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world"));
}
#[test]
fn overlaydb_negative() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
let h = trie.insert(EMPTY_PREFIX, b"hello world");
trie.commit().unwrap();
trie.remove(&h);
trie.remove(&h); //bad - sends us into negative refs.
assert_eq!(trie.get(&h), None);
trie.remove(&h, EMPTY_PREFIX);
trie.remove(&h, EMPTY_PREFIX); //bad - sends us into negative refs.
assert_eq!(trie.get(&h, EMPTY_PREFIX), None);
assert!(trie.commit().is_err());
}
#[test]
fn overlaydb_complex() {
let mut trie = OverlayDB::new_temp();
let hfoo = trie.insert(b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
let hbar = trie.insert(b"bar");
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
let hfoo = trie.insert(EMPTY_PREFIX, b"foo");
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
let hbar = trie.insert(EMPTY_PREFIX, b"bar");
assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.insert(b"foo"); // two refs
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar"));
trie.insert(EMPTY_PREFIX, b"foo"); // two refs
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.remove(&hbar); // zero refs - delete
assert_eq!(trie.get(&hbar), None);
trie.remove(&hfoo); // one ref - keep
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar"));
trie.remove(&hbar, EMPTY_PREFIX); // zero refs - delete
assert_eq!(trie.get(&hbar, EMPTY_PREFIX), None);
trie.remove(&hfoo, EMPTY_PREFIX); // one ref - keep
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - would delete, but...
assert_eq!(trie.get(&hfoo), None);
trie.insert(b"foo"); // one ref - keep after all.
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo, EMPTY_PREFIX); // zero ref - would delete, but...
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None);
trie.insert(EMPTY_PREFIX, b"foo"); // one ref - keep after all.
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - delete
assert_eq!(trie.get(&hfoo), None);
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo, EMPTY_PREFIX); // zero ref - delete
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None);
trie.commit().unwrap(); //
assert_eq!(trie.get(&hfoo), None);
assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None);
}

324
util/journaldb/src/overlayrecentdb.rs
View File

@@ -23,11 +23,10 @@ use std::sync::Arc;
use bytes::Bytes;
use ethereum_types::H256;
use hash_db::{HashDB};
use heapsize::HeapSizeOf;
use hash_db::{HashDB, Prefix, EMPTY_PREFIX};
use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops};
use keccak_hasher::KeccakHasher;
use kvdb::{KeyValueDB, DBTransaction, DBValue};
use memory_db::*;
use parking_lot::RwLock;
use fastmap::H256FastMap;
use rlp::{Rlp, RlpStream, encode, decode, DecoderError, Decodable, Encodable};
@@ -66,7 +65,7 @@ use util::DatabaseKey;
/// 7. Delete ancient record from memory and disk.
pub struct OverlayRecentDB {
transaction_overlay: MemoryDB<KeccakHasher, DBValue>,
transaction_overlay: super::MemoryDB,
backing: Arc<KeyValueDB>,
journal_overlay: Arc<RwLock<JournalOverlay>>,
column: Option<u32>,
@@ -120,7 +119,7 @@ impl<'a> Encodable for DatabaseValueRef<'a> {
#[derive(PartialEq)]
struct JournalOverlay {
backing_overlay: MemoryDB<KeccakHasher, DBValue>, // Nodes added in the history period
backing_overlay: super::MemoryDB, // Nodes added in the history period
pending_overlay: H256FastMap<DBValue>, // Nodes being transfered from backing_overlay to backing db
journal: HashMap<u64, Vec<JournalEntry>>,
latest_era: Option<u64>,
@@ -128,19 +127,13 @@ struct JournalOverlay {
cumulative_size: usize, // cumulative size of all entries.
}
#[derive(PartialEq)]
#[derive(PartialEq, MallocSizeOf)]
struct JournalEntry {
id: H256,
insertions: Vec<H256>,
deletions: Vec<H256>,
}
impl HeapSizeOf for JournalEntry {
fn heap_size_of_children(&self) -> usize {
self.insertions.heap_size_of_children() + self.deletions.heap_size_of_children()
}
}
impl Clone for OverlayRecentDB {
fn clone(&self) -> OverlayRecentDB {
OverlayRecentDB {
@@ -204,11 +197,11 @@ impl OverlayRecentDB {
for (k, v) in value.inserts {
let short_key = to_short_key(&k);
if !overlay.contains(&short_key) {
if !overlay.contains(&short_key, EMPTY_PREFIX) {
cumulative_size += v.len();
}
overlay.emplace(short_key, v);
overlay.emplace(short_key, EMPTY_PREFIX, v);
inserted_keys.push(k);
}
journal.entry(era).or_insert_with(Vec::new).push(JournalEntry {
@@ -272,12 +265,13 @@ impl JournalDB for OverlayRecentDB {
}
fn mem_used(&self) -> usize {
let mut mem = self.transaction_overlay.mem_used();
let mut ops = new_malloc_size_ops();
let mut mem = self.transaction_overlay.size_of(&mut ops);
let overlay = self.journal_overlay.read();
mem += overlay.backing_overlay.mem_used();
mem += overlay.pending_overlay.heap_size_of_children();
mem += overlay.journal.heap_size_of_children();
mem += overlay.backing_overlay.size_of(&mut ops);
mem += overlay.pending_overlay.size_of(&mut ops);
mem += overlay.journal.size_of(&mut ops);
mem
}
@@ -302,7 +296,7 @@ impl JournalDB for OverlayRecentDB {
fn state(&self, key: &H256) -> Option<Bytes> {
let journal_overlay = self.journal_overlay.read();
let key = to_short_key(key);
journal_overlay.backing_overlay.get(&key).map(|v| v.into_vec())
journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX).map(|v| v.into_vec())
.or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone().into_vec()))
.or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.into_vec()))
}
@@ -334,11 +328,11 @@ impl JournalDB for OverlayRecentDB {
for (k, v) in insertions {
let short_key = to_short_key(&k);
if !journal_overlay.backing_overlay.contains(&short_key) {
if !journal_overlay.backing_overlay.contains(&short_key, EMPTY_PREFIX) {
journal_overlay.cumulative_size += v.len();
}
journal_overlay.backing_overlay.emplace(short_key, v);
journal_overlay.backing_overlay.emplace(short_key, EMPTY_PREFIX, v);
}
let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len());
@@ -387,7 +381,7 @@ impl JournalDB for OverlayRecentDB {
{
if *canon_id == journal.id {
for h in &journal.insertions {
if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h)) {
if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h), EMPTY_PREFIX) {
if rc > 0 {
canon_insertions.push((h.clone(), d.clone())); //TODO: optimize this to avoid data copy
}
@@ -410,13 +404,13 @@ impl JournalDB for OverlayRecentDB {
}
// update the overlay
for k in overlay_deletions {
if let Some(val) = journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k)) {
if let Some(val) = journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k), EMPTY_PREFIX) {
journal_overlay.cumulative_size -= val.len();
}
}
// apply canon deletions
for k in canon_deletions {
if !journal_overlay.backing_overlay.contains(&to_short_key(&k)) {
if !journal_overlay.backing_overlay.contains(&to_short_key(&k), EMPTY_PREFIX) {
batch.delete(self.column, k.as_bytes());
}
}
@@ -458,14 +452,14 @@ impl JournalDB for OverlayRecentDB {
Ok(ops)
}
fn consolidate(&mut self, with: MemoryDB<KeccakHasher, DBValue>) {
fn consolidate(&mut self, with: super::MemoryDB) {
self.transaction_overlay.consolidate(with);
}
}
impl HashDB<KeccakHasher, DBValue> for OverlayRecentDB {
fn get(&self, key: &H256) -> Option<DBValue> {
if let Some((d, rc)) = self.transaction_overlay.raw(key) {
fn get(&self, key: &H256, prefix: Prefix) -> Option<DBValue> {
if let Some((d, rc)) = self.transaction_overlay.raw(key, prefix) {
if rc > 0 {
return Some(d.clone())
}
@@ -473,24 +467,24 @@ impl HashDB<KeccakHasher, DBValue> for OverlayRecentDB {
let v = {
let journal_overlay = self.journal_overlay.read();
let key = to_short_key(key);
journal_overlay.backing_overlay.get(&key)
journal_overlay.backing_overlay.get(&key, prefix)
.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
};
v.or_else(|| self.payload(key))
}
fn contains(&self, key: &H256) -> bool {
self.get(key).is_some()
fn contains(&self, key: &H256, prefix: Prefix) -> bool {
self.get(key, prefix).is_some()
}
fn insert(&mut self, value: &[u8]) -> H256 {
self.transaction_overlay.insert(value)
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 {
self.transaction_overlay.insert(prefix, value)
}
fn emplace(&mut self, key: H256, value: DBValue) {
self.transaction_overlay.emplace(key, value);
fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) {
self.transaction_overlay.emplace(key, prefix, value);
}
fn remove(&mut self, key: &H256) {
self.transaction_overlay.remove(key);
fn remove(&mut self, key: &H256, prefix: Prefix) {
self.transaction_overlay.remove(key, prefix);
}
}
@@ -499,7 +493,7 @@ mod tests {
use keccak::keccak;
use super::*;
use hash_db::HashDB;
use hash_db::{HashDB, EMPTY_PREFIX};
use {kvdb_memorydb, JournalDB};
fn new_db() -> OverlayRecentDB {
@@ -512,7 +506,7 @@ mod tests {
// history is 1
let mut jdb = new_db();
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
@@ -522,10 +516,10 @@ mod tests {
jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&x);
jdb.remove(&x, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
let x = jdb.insert(b"X");
let x = jdb.insert(EMPTY_PREFIX, b"X");
jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -534,30 +528,30 @@ mod tests {
jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&x));
assert!(jdb.contains(&x, EMPTY_PREFIX));
}
#[test]
fn long_history() {
// history is 3
let mut jdb = new_db();
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
jdb.remove(&h);
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.remove(&h, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&h));
assert!(!jdb.contains(&h, EMPTY_PREFIX));
}
#[test]
@@ -565,42 +559,42 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.remove(&baz, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
}
#[test]
@@ -608,31 +602,31 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
}
#[test]
@@ -640,22 +634,22 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
assert!(jdb.contains(&foo));
jdb.insert(EMPTY_PREFIX, b"foo");
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -664,23 +658,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -690,23 +684,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
@@ -716,23 +710,23 @@ mod tests {
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3a"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3b"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -751,8 +745,8 @@ mod tests {
let foo = {
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
@@ -760,18 +754,18 @@ mod tests {
{
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
}
@@ -781,19 +775,19 @@ mod tests {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
// expunge foo
@@ -807,39 +801,39 @@ mod tests {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(2, &keccak(b"2b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(4, &keccak(b"4b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
@@ -852,35 +846,35 @@ mod tests {
fn broken_assert() {
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
// foo is ancient history.
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); // BROKEN
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
#[test]
fn reopen_test() {
let mut jdb = new_db();
// history is 4
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
@@ -894,16 +888,16 @@ mod tests {
// foo is ancient history.
jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(5, &keccak(b"5"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
jdb.remove(&bar);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(6, &keccak(b"6"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.insert(b"foo");
jdb.insert(b"bar");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(7, &keccak(b"7"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
@@ -918,7 +912,7 @@ mod tests {
{
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
// history is 1
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
@@ -926,24 +920,24 @@ mod tests {
// foo is ancient history.
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
jdb.insert(b"foo");
jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(3, &keccak(b"3"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(4, &keccak(b"4"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
@@ -951,7 +945,7 @@ mod tests {
jdb.commit_batch(5, &keccak(b"5"), Some((3, keccak(b"3")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
// incantation to reopen the db
}; {
@@ -959,7 +953,7 @@ mod tests {
jdb.commit_batch(6, &keccak(b"6"), Some((4, keccak(b"4")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
}
}
@@ -970,16 +964,16 @@ mod tests {
let (foo, bar, baz) = {
let mut jdb = OverlayRecentDB::new(shared_db.clone(), None);
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.can_reconstruct_refs());
(foo, bar, baz)
@@ -989,43 +983,43 @@ mod tests {
let mut jdb = OverlayRecentDB::new(shared_db, None);
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
}
}
#[test]
fn insert_older_era() {
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0a"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
let bar = jdb.insert(b"bar");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0a")))).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(0, &keccak(b"0b"), None).unwrap();
assert!(jdb.can_reconstruct_refs());
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
}
#[test]
fn inject() {
let mut jdb = new_db();
let key = jdb.insert(b"dog");
let key = jdb.insert(EMPTY_PREFIX, b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key, EMPTY_PREFIX);
jdb.inject_batch().unwrap();
assert!(jdb.get(&key).is_none());
assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
}
#[test]
@@ -1037,7 +1031,7 @@ mod tests {
assert!(jdb.earliest_era().is_none());
// single journalled era.
let _key = jdb.insert(b"hello!");
let _key = jdb.insert(EMPTY_PREFIX, b"hello!");
let mut batch = jdb.backing().transaction();
jdb.journal_under(&mut batch, 0, &keccak(b"0")).unwrap();
jdb.backing().write_buffered(batch);

126
util/journaldb/src/refcounteddb.rs
View File

@@ -22,11 +22,10 @@ use std::sync::Arc;
use bytes::Bytes;
use ethereum_types::H256;
use hash_db::{HashDB};
use heapsize::HeapSizeOf;
use hash_db::{HashDB, Prefix, EMPTY_PREFIX};
use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops};
use keccak_hasher::KeccakHasher;
use kvdb::{KeyValueDB, DBTransaction, DBValue};
use memory_db::MemoryDB;
use overlaydb::OverlayDB;
use rlp::{encode, decode};
use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
@@ -81,11 +80,11 @@ impl RefCountedDB {
}
impl HashDB<KeccakHasher, DBValue> for RefCountedDB {
fn get(&self, key: &H256) -> Option<DBValue> { self.forward.get(key) }
fn contains(&self, key: &H256) -> bool { self.forward.contains(key) }
fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
fn emplace(&mut self, key: H256, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
fn remove(&mut self, key: &H256) { self.removes.push(key.clone()); }
fn get(&self, key: &H256, prefix: Prefix) -> Option<DBValue> { self.forward.get(key, prefix) }
fn contains(&self, key: &H256, prefix: Prefix) -> bool { self.forward.contains(key, prefix) }
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { let r = self.forward.insert(prefix, value); self.inserts.push(r.clone()); r }
fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, prefix, value); }
fn remove(&mut self, key: &H256, _prefix: Prefix) { self.removes.push(key.clone()); }
}
impl ::traits::KeyedHashDB for RefCountedDB {
@@ -105,7 +104,8 @@ impl JournalDB for RefCountedDB {
}
fn mem_used(&self) -> usize {
self.inserts.heap_size_of_children() + self.removes.heap_size_of_children()
let mut ops = new_malloc_size_ops();
self.inserts.size_of(&mut ops) + self.removes.size_of(&mut ops)
}
fn is_empty(&self) -> bool {
@@ -184,7 +184,7 @@ impl JournalDB for RefCountedDB {
}.expect("rlp read from db; qed");
trace!(target: "rcdb", "delete journal for time #{}.{}=>{}, (canon was {}): deleting {:?}", end_era, db_key.index, our_id, canon_id, to_remove);
for i in &to_remove {
self.forward.remove(i);
self.forward.remove(i, EMPTY_PREFIX);
}
batch.delete(self.column, &last);
db_key.index += 1;
@@ -197,19 +197,19 @@ impl JournalDB for RefCountedDB {
fn inject(&mut self, batch: &mut DBTransaction) -> io::Result<u32> {
self.inserts.clear();
for remove in self.removes.drain(..) {
self.forward.remove(&remove);
self.forward.remove(&remove, EMPTY_PREFIX);
}
self.forward.commit_to_batch(batch)
}
fn consolidate(&mut self, mut with: MemoryDB<KeccakHasher, DBValue>) {
fn consolidate(&mut self, mut with: super::MemoryDB) {
for (key, (value, rc)) in with.drain() {
for _ in 0..rc {
self.emplace(key, value.clone());
self.emplace(key, EMPTY_PREFIX, value.clone());
}
for _ in rc..0 {
self.remove(&key);
self.remove(&key, EMPTY_PREFIX);
}
}
}
@@ -219,7 +219,7 @@ impl JournalDB for RefCountedDB {
mod tests {
use keccak::keccak;
use hash_db::HashDB;
use hash_db::{HashDB, EMPTY_PREFIX};
use super::*;
use {JournalDB, kvdb_memorydb};
@@ -232,18 +232,18 @@ mod tests {
fn long_history() {
// history is 3
let mut jdb = new_db();
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&h));
jdb.remove(&h);
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.remove(&h, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&h));
assert!(jdb.contains(&h, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap();
assert!(!jdb.contains(&h));
assert!(!jdb.contains(&h, EMPTY_PREFIX));
}
#[test]
@@ -251,10 +251,10 @@ mod tests {
// history is 3
let mut jdb = new_db();
assert_eq!(jdb.latest_era(), None);
let h = jdb.insert(b"foo");
let h = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert_eq!(jdb.latest_era(), Some(0));
jdb.remove(&h);
jdb.remove(&h, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1"), None).unwrap();
assert_eq!(jdb.latest_era(), Some(1));
jdb.commit_batch(2, &keccak(b"2"), None).unwrap();
@@ -270,37 +270,37 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&bar);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
jdb.remove(&bar, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
let foo = jdb.insert(b"foo");
jdb.remove(&baz);
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
jdb.remove(&baz, EMPTY_PREFIX);
jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.remove(&foo);
jdb.remove(&foo, EMPTY_PREFIX);
jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap();
assert!(!jdb.contains(&foo));
assert!(!jdb.contains(&bar));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
}
#[test]
@@ -308,39 +308,39 @@ mod tests {
// history is 1
let mut jdb = new_db();
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
let foo = jdb.insert(EMPTY_PREFIX, b"foo");
let bar = jdb.insert(EMPTY_PREFIX, b"bar");
jdb.commit_batch(0, &keccak(b"0"), None).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
jdb.remove(&foo);
let baz = jdb.insert(b"baz");
jdb.remove(&foo, EMPTY_PREFIX);
let baz = jdb.insert(EMPTY_PREFIX, b"baz");
jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap();
jdb.remove(&bar);
jdb.remove(&bar, EMPTY_PREFIX);
jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap();
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
assert!(jdb.contains(&baz));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(jdb.contains(&bar, EMPTY_PREFIX));
assert!(jdb.contains(&baz, EMPTY_PREFIX));
jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap();
assert!(jdb.contains(&foo));
assert!(!jdb.contains(&baz));
assert!(!jdb.contains(&bar));
assert!(jdb.contains(&foo, EMPTY_PREFIX));
assert!(!jdb.contains(&baz, EMPTY_PREFIX));
assert!(!jdb.contains(&bar, EMPTY_PREFIX));
}
#[test]
fn inject() {
let mut jdb = new_db();
let key = jdb.insert(b"dog");
let key = jdb.insert(EMPTY_PREFIX, b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key, EMPTY_PREFIX);
jdb.inject_batch().unwrap();
assert!(jdb.get(&key).is_none());
assert!(jdb.get(&key, EMPTY_PREFIX).is_none());
}
}

2
util/journaldb/src/traits.rs
View File

@@ -93,7 +93,7 @@ pub trait JournalDB: KeyedHashDB {
fn flush(&self) {}
/// Consolidate all the insertions and deletions in the given memory overlay.
fn consolidate(&mut self, overlay: ::memory_db::MemoryDB<KeccakHasher, DBValue>);
fn consolidate(&mut self, overlay: super::MemoryDB);
/// Commit all changes in a single batch
#[cfg(test)]

2
util/keccak-hasher/Cargo.toml
View File

@@ -8,5 +8,5 @@ license = "GPL-3.0"
[dependencies]
ethereum-types = "0.6.0"
tiny-keccak = "1.4.2"
hash-db = "0.11.0"
hash-db = "0.12.4"
plain_hasher = "0.2"

2
util/memory-cache/Cargo.toml
View File

@@ -6,5 +6,5 @@ description = "An LRU-cache which operates on memory used"
license = "GPL3"
[dependencies]
heapsize = "0.4"
parity-util-mem = "0.1"
lru-cache = "0.1"

12
util/memory-cache/src/lib.rs
View File

@@ -18,10 +18,10 @@
//! crate.
// TODO: push changes upstream in a clean way.
extern crate heapsize;
extern crate parity_util_mem;
extern crate lru_cache;
use heapsize::HeapSizeOf;
use parity_util_mem::{MallocSizeOf, MallocSizeOfExt};
use lru_cache::LruCache;
use std::hash::Hash;
@@ -29,18 +29,18 @@ use std::hash::Hash;
const INITIAL_CAPACITY: usize = 4;
/// An LRU-cache which operates on memory used.
pub struct MemoryLruCache<K: Eq + Hash, V: HeapSizeOf> {
pub struct MemoryLruCache<K: Eq + Hash, V> {
inner: LruCache<K, V>,
cur_size: usize,
max_size: usize,
}
// amount of memory used when the item will be put on the heap.
fn heap_size_of<T: HeapSizeOf>(val: &T) -> usize {
::std::mem::size_of::<T>() + val.heap_size_of_children()
fn heap_size_of<T: MallocSizeOf>(val: &T) -> usize {
::std::mem::size_of::<T>() + val.malloc_size_of()
}
impl<K: Eq + Hash, V: HeapSizeOf> MemoryLruCache<K, V> {
impl<K: Eq + Hash, V: MallocSizeOf> MemoryLruCache<K, V> {
/// Create a new cache with a maximum size in bytes.
pub fn new(max_size: usize) -> Self {
MemoryLruCache {

10
util/memzero/Cargo.toml
View File

@@ -1,10 +0,0 @@
[package]
name = "memzero"
version = "0.1.0"
description = "A wrapper for zero-ing out memory when dropped"
license = "GPL-3.0"
homepage = "https://parity.io"
repository = "https://github.com/paritytech/parity-ethereum"
documentation = "https://docs.rs/crate/memzero"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"

54
util/memzero/src/lib.rs
View File

@@ -1,54 +0,0 @@
// Copyright 2015-2019 Parity Technologies (UK) Ltd.
// This file is part of Parity Ethereum.
// Parity Ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity Ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
use std::ops::{Deref, DerefMut};
use std::ptr;
/// Wrapper to zero out memory when dropped.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Memzero<T: AsMut<[u8]>> {
mem: T,
}
impl<T: AsMut<[u8]>> From<T> for Memzero<T> {
fn from(mem: T) -> Memzero<T> {
Memzero { mem }
}
}
impl<T: AsMut<[u8]>> Drop for Memzero<T> {
fn drop(&mut self) {
unsafe {
for byte_ref in self.mem.as_mut() {
ptr::write_volatile(byte_ref, 0)
}
}
}
}
impl<T: AsMut<[u8]>> Deref for Memzero<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.mem
}
}
impl<T: AsMut<[u8]>> DerefMut for Memzero<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.mem
}
}

6
util/patricia-trie-ethereum/Cargo.toml
View File

@@ -6,15 +6,15 @@ description = "Merkle-Patricia Trie (Ethereum Style)"
license = "GPL-3.0"
[dependencies]
trie-db = "0.11.0"
trie-db = "0.12.4"
keccak-hasher = { version = "0.1.1", path = "../keccak-hasher" }
hash-db = "0.11.0"
hash-db = "0.12.4"
rlp = "0.4.0"
parity-bytes = "0.1"
ethereum-types = "0.6.0"
elastic-array = "0.10"
[dev-dependencies]
memory-db = "0.11.0"
memory-db = "0.12.4"
keccak-hash = "0.2.0"
journaldb = { path = "../journaldb" }

2
util/triehash-ethereum/Cargo.toml
View File

@@ -6,6 +6,6 @@ description = "Trie-root helpers, ethereum style"
license = "GPL-3.0"
[dependencies]
triehash = "0.5.0"
triehash = "0.6.0"
ethereum-types = "0.6.0"
keccak-hasher = { path = "../keccak-hasher" }