grassrootseconomics
/
openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of MemoryDB denote (#2881)

This commit is contained in:
Arkadiy Paronyan 2016-10-26 13:53:47 +02:00 committed by Gav Wood
parent 436b7c213d
commit 5b978be034
30 changed files with 359 additions and 376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@ -222,8 +222,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "elastic-array"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
version = "0.6.0"
source = "git+https://github.com/ethcore/elastic-array#70e4012e691b732c7c4cb04e9232799e6aa268bc"
dependencies = [
"heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "env_logger"
@ -540,7 +543,7 @@ dependencies = [
"ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy 0.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
"elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)",
"env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)",
"ethcore-bigint 0.1.1",
@ -1421,7 +1424,7 @@ dependencies = [
name = "rlp"
version = "0.1.0"
dependencies = [
"elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)",
"ethcore-bigint 0.1.1",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1954,7 +1957,7 @@ dependencies = [
"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf"
"checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76"
"checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d"
"checksum elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4bc9250a632e7c001b741eb0ec6cee93c9a5b6d5f1879696a4b94d62b012210a"
"checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "<none>"
"checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5"
"checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
"checksum ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0c53453517f620847be51943db329276ae52f2e210cfc659e81182864be2f"

1
Cargo.toml
View File

@ -73,6 +73,7 @@ ipc = ["ethcore/ipc", "ethsync/ipc"]
jit = ["ethcore/jit"]
dev = ["clippy", "ethcore/dev", "ethcore-util/dev", "ethsync/dev", "ethcore-rpc/dev", "ethcore-dapps/dev", "ethcore-signer/dev"]
json-tests = ["ethcore/json-tests"]
test-heavy = ["ethcore/test-heavy"]
stratum = ["ipc"]
ethkey-cli = ["ethcore/ethkey-cli"]
ethstore-cli = ["ethcore/ethstore-cli"]

34
ethcore/src/account_db.rs
View File

@ -96,9 +96,9 @@ impl<'db> HashDB for AccountDB<'db>{
unimplemented!()
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
if key == &SHA3_NULL_RLP {
return Some(&NULL_RLP_STATIC);
return Some(DBValue::from_slice(&NULL_RLP_STATIC));
}
self.db.get(&combine_key(&self.address_hash, key))
}
@ -114,7 +114,7 @@ impl<'db> HashDB for AccountDB<'db>{
unimplemented!()
}
fn emplace(&mut self, _key: H256, _value: Bytes) {
fn emplace(&mut self, _key: H256, _value: DBValue) {
unimplemented!()
}
@ -122,7 +122,7 @@ impl<'db> HashDB for AccountDB<'db>{
unimplemented!()
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
self.db.get_aux(hash)
}
}
@ -158,9 +158,9 @@ impl<'db> HashDB for AccountDBMut<'db>{
unimplemented!()
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
if key == &SHA3_NULL_RLP {
return Some(&NULL_RLP_STATIC);
return Some(DBValue::from_slice(&NULL_RLP_STATIC));
}
self.db.get(&combine_key(&self.address_hash, key))
}
@ -178,16 +178,16 @@ impl<'db> HashDB for AccountDBMut<'db>{
}
let k = value.sha3();
let ak = combine_key(&self.address_hash, &k);
self.db.emplace(ak, value.to_vec());
self.db.emplace(ak, DBValue::from_slice(value));
k
}
fn emplace(&mut self, key: H256, value: Bytes) {
fn emplace(&mut self, key: H256, value: DBValue) {
if key == SHA3_NULL_RLP {
return;
}
let key = combine_key(&self.address_hash, &key);
self.db.emplace(key, value.to_vec())
self.db.emplace(key, value)
}
fn remove(&mut self, key: &H256) {
@ -202,7 +202,7 @@ impl<'db> HashDB for AccountDBMut<'db>{
self.db.insert_aux(hash, value);
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
self.db.get_aux(hash)
}
@ -218,9 +218,9 @@ impl<'db> HashDB for Wrapping<'db> {
unimplemented!()
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
if key == &SHA3_NULL_RLP {
return Some(&NULL_RLP_STATIC);
return Some(DBValue::from_slice(&NULL_RLP_STATIC));
}
self.0.get(key)
}
@ -236,7 +236,7 @@ impl<'db> HashDB for Wrapping<'db> {
unimplemented!()
}
fn emplace(&mut self, _key: H256, _value: Bytes) {
fn emplace(&mut self, _key: H256, _value: DBValue) {
unimplemented!()
}
@ -252,9 +252,9 @@ impl<'db> HashDB for WrappingMut<'db>{
unimplemented!()
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
if key == &SHA3_NULL_RLP {
return Some(&NULL_RLP_STATIC);
return Some(DBValue::from_slice(&NULL_RLP_STATIC));
}
self.0.get(key)
}
@ -273,7 +273,7 @@ impl<'db> HashDB for WrappingMut<'db>{
self.0.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
fn emplace(&mut self, key: H256, value: DBValue) {
if key == SHA3_NULL_RLP {
return;
}
@ -286,4 +286,4 @@ impl<'db> HashDB for WrappingMut<'db>{
}
self.0.remove(key)
}
}
}

2
ethcore/src/migrations/state/v7.rs
View File

@ -154,7 +154,7 @@ impl OverlayRecentV7 {
// and commit the altered entries.
fn migrate_journal(&self, source: Arc<Database>, mut batch: Batch, dest: &mut Database) -> Result<(), Error> {
if let Some(val) = try!(source.get(None, V7_LATEST_ERA_KEY).map_err(Error::Custom)) {
try!(batch.insert(V7_LATEST_ERA_KEY.into(), val.to_owned(), dest));
try!(batch.insert(V7_LATEST_ERA_KEY.into(), val.clone().to_vec(), dest));
let mut era = decode::<u64>(&val);
loop {

12
ethcore/src/snapshot/account.rs
View File

@ -19,7 +19,7 @@
use account_db::{AccountDB, AccountDBMut};
use snapshot::Error;
use util::{U256, FixedHash, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP};
use util::{U256, FixedHash, H256, Bytes, HashDB, DBValue, SHA3_EMPTY, SHA3_NULL_RLP};
use util::trie::{TrieDB, Trie};
use rlp::{Rlp, RlpStream, Stream, UntrustedRlp, View};
@ -112,7 +112,7 @@ impl Account {
let mut stream = RlpStream::new_list(pairs.len());
for (k, v) in pairs {
stream.begin_list(2).append(&k).append(&v);
stream.begin_list(2).append(&k).append(&&*v);
}
let pairs_rlp = stream.out();
@ -130,7 +130,7 @@ impl Account {
match acct_db.get(&self.code_hash) {
Some(c) => {
used_code.insert(self.code_hash.clone());
account_stream.append(&CodeState::Inline.raw()).append(&c);
account_stream.append(&CodeState::Inline.raw()).append(&&*c);
}
None => {
warn!("code lookup failed during snapshot");
@ -178,7 +178,7 @@ impl Account {
CodeState::Hash => {
let code_hash = try!(rlp.val_at(3));
if let Some(code) = code_map.get(&code_hash) {
acct_db.emplace(code_hash.clone(), code.clone());
acct_db.emplace(code_hash.clone(), DBValue::from_slice(&code));
}
(code_hash, None)
@ -226,7 +226,7 @@ mod tests {
use snapshot::tests::helpers::fill_storage;
use util::sha3::{SHA3_EMPTY, SHA3_NULL_RLP};
use util::{Address, FixedHash, H256, HashDB};
use util::{Address, FixedHash, H256, HashDB, DBValue};
use rlp::{UntrustedRlp, View};
use std::collections::{HashSet, HashMap};
@ -292,7 +292,7 @@ mod tests {
{
let mut acct_db = AccountDBMut::new(db.as_hashdb_mut(), &addr2);
acct_db.emplace(code_hash.clone(), b"this is definitely code".to_vec());
acct_db.emplace(code_hash.clone(), DBValue::from_slice(b"this is definitely code"));
}
let account1 = Account {

6
ethcore/src/snapshot/mod.rs
View File

@ -29,7 +29,7 @@ use engines::Engine;
use ids::BlockID;
use views::BlockView;
use util::{Bytes, Hashable, HashDB, snappy, U256, Uint};
use util::{Bytes, Hashable, HashDB, DBValue, snappy, U256, Uint};
use util::memorydb::MemoryDB;
use util::Mutex;
use util::hash::{FixedHash, H256};
@ -369,7 +369,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
// account_key here is the address' hash.
for item in try!(account_trie.iter()) {
let (account_key, account_data) = try!(item);
let account = Account::from_thin_rlp(account_data);
let account = Account::from_thin_rlp(&*account_data);
let account_key_hash = H256::from_slice(&account_key);
let account_db = AccountDB::from_hash(db, account_key_hash);
@ -457,7 +457,7 @@ impl StateRebuilder {
for (code_hash, code) in chunk_code {
for addr_hash in self.missing_code.remove(&code_hash).unwrap_or_else(Vec::new) {
let mut db = AccountDBMut::from_hash(self.db.as_hashdb_mut(), addr_hash);
db.emplace(code_hash, code.clone());
db.emplace(code_hash, DBValue::from_slice(&code));
}
self.code_map.insert(code_hash, code);

3
ethcore/src/snapshot/tests/helpers.rs
View File

@ -21,6 +21,7 @@ use account_db::AccountDBMut;
use rand::Rng;
use snapshot::account::Account;
use util::DBValue;
use util::hash::{FixedHash, H256};
use util::hashdb::HashDB;
use util::trie::{Alphabet, StandardMap, SecTrieDBMut, TrieMut, ValueMode};
@ -66,7 +67,7 @@ impl StateProducer {
let mut account = Account::from_thin_rlp(&*account_data);
let acct_db = AccountDBMut::from_hash(db, *address_hash);
fill_storage(acct_db, account.storage_root_mut(), &mut self.storage_seed);
*account_data = account.to_thin_rlp();
*account_data = DBValue::from_vec(account.to_thin_rlp());
}
// sweep again to alter account trie.

6
ethcore/src/state/account.rs
View File

@ -172,7 +172,7 @@ impl Account {
using it will not fail.");
let item: U256 = match db.get(key){
Ok(x) => x.map_or_else(U256::zero, decode),
Ok(x) => x.map_or_else(U256::zero, |v| decode(&*v)),
Err(e) => panic!("Encountered potential DB corruption: {}", e),
};
let value: H256 = item.into();
@ -253,8 +253,8 @@ impl Account {
self.is_cached() ||
match db.get(&self.code_hash) {
Some(x) => {
self.code_cache = Arc::new(x.to_vec());
self.code_size = Some(x.len());
self.code_cache = Arc::new(x.to_vec());
true
},
_ => {
@ -351,7 +351,7 @@ impl Account {
self.code_filth = Filth::Clean;
},
(true, false) => {
db.emplace(self.code_hash.clone(), (*self.code_cache).clone());
db.emplace(self.code_hash.clone(), DBValue::from_slice(&*self.code_cache));
self.code_size = Some(self.code_cache.len());
self.code_filth = Filth::Clean;
},

6
ethcore/src/state/mod.rs
View File

@ -408,7 +408,7 @@ impl State {
// account is not found in the global cache, get from the DB and insert into local
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
let maybe_acc = match db.get(address) {
Ok(acc) => acc.map(Account::from_rlp),
Ok(acc) => acc.map(|v| Account::from_rlp(&v)),
Err(e) => panic!("Potential DB corruption encountered: {}", e),
};
let r = maybe_acc.as_ref().map_or(H256::new(), |a| {
@ -648,7 +648,7 @@ impl State {
// not found in the global cache, get from the DB and insert into local
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
let mut maybe_acc = match db.get(a) {
Ok(acc) => acc.map(Account::from_rlp),
Ok(acc) => acc.map(|v| Account::from_rlp(&v)),
Err(e) => panic!("Potential DB corruption encountered: {}", e),
};
if let Some(ref mut account) = maybe_acc.as_mut() {
@ -680,7 +680,7 @@ impl State {
let maybe_acc = if self.db.check_account_bloom(a) {
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
let maybe_acc = match db.get(a) {
Ok(Some(acc)) => AccountEntry::new_clean(Some(Account::from_rlp(acc))),
Ok(Some(acc)) => AccountEntry::new_clean(Some(Account::from_rlp(&acc))),
Ok(None) => AccountEntry::new_clean(None),
Err(e) => panic!("Potential DB corruption encountered: {}", e),
};

2
util/Cargo.toml
View File

@ -18,7 +18,7 @@ rocksdb = { git = "https://github.com/ethcore/rust-rocksdb" }
lazy_static = "0.2"
eth-secp256k1 = { git = "https://github.com/ethcore/rust-secp256k1" }
rust-crypto = "0.2.34"
elastic-array = "0.5"
elastic-array = { git = "https://github.com/ethcore/elastic-array" }
rlp = { path = "rlp" }
heapsize = { version = "0.3", features = ["unstable"] }
itertools = "0.4"

4
util/rlp/Cargo.toml
View File

@ -6,7 +6,7 @@ version = "0.1.0"
authors = ["Ethcore <admin@ethcore.io>"]
[dependencies]
elastic-array = "0.5"
elastic-array = { git = "https://github.com/ethcore/elastic-array" }
ethcore-bigint = { path = "../bigint" }
lazy_static = "0.2"
rustc-serialize = "0.3"
rustc-serialize = "0.3"

13
util/src/hashdb.rs
View File

@ -16,8 +16,11 @@
//! Database of byte-slices keyed to their Keccak hash.
use hash::*;
use bytes::*;
use std::collections::HashMap;
use elastic_array::ElasticArray256;
/// `HashDB` value type.
pub type DBValue = ElasticArray256<u8>;
/// Trait modelling datastore keyed by a 32-byte Keccak hash.
pub trait HashDB: AsHashDB + Send + Sync {
@ -39,7 +42,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
/// assert_eq!(m.get(&hash).unwrap(), hello_bytes);
/// }
/// ```
fn get(&self, key: &H256) -> Option<&[u8]>;
fn get(&self, key: &H256) -> Option<DBValue>;
/// Check for the existance of a hash-key.
///
@ -80,7 +83,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
fn insert(&mut self, value: &[u8]) -> H256;
/// Like `insert()` , except you provide the key and the data is all moved.
fn emplace(&mut self, key: H256, value: Bytes);
fn emplace(&mut self, key: H256, value: DBValue);
/// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may
/// happen without the data being eventually being inserted into the DB.
@ -111,7 +114,7 @@ pub trait HashDB: AsHashDB + Send + Sync {
}
/// Get auxiliary data from hashdb.
fn get_aux(&self, _hash: &[u8]) -> Option<Vec<u8>> {
fn get_aux(&self, _hash: &[u8]) -> Option<DBValue> {
unimplemented!();
}
@ -136,4 +139,4 @@ impl<T: HashDB> AsHashDB for T {
fn as_hashdb_mut(&mut self) -> &mut HashDB {
self
}
}
}

28
util/src/journaldb/archivedb.rs
View File

@ -65,8 +65,8 @@ impl ArchiveDB {
Self::new(backing, None)
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
fn payload(&self, key: &H256) -> Option<DBValue> {
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
}
}
@ -85,19 +85,12 @@ impl HashDB for ArchiveDB {
ret
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
let k = self.overlay.raw(key);
match k {
Some((d, rc)) if rc > 0 => Some(d),
_ => {
if let Some(x) = self.payload(key) {
Some(self.overlay.denote(key, x).0)
}
else {
None
}
}
if let Some((d, rc)) = k {
if rc > 0 { return Some(d); }
}
self.payload(key)
}
fn contains(&self, key: &H256) -> bool {
@ -108,7 +101,7 @@ impl HashDB for ArchiveDB {
self.overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
fn emplace(&mut self, key: H256, value: DBValue) {
self.overlay.emplace(key, value);
}
@ -120,7 +113,7 @@ impl HashDB for ArchiveDB {
self.overlay.insert_aux(hash, value);
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
if let Some(res) = self.overlay.get_aux(hash) {
return Some(res)
}
@ -130,7 +123,6 @@ impl HashDB for ArchiveDB {
self.backing.get(self.column, &db_hash)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|v| v.to_vec())
}
fn remove_aux(&mut self, hash: &[u8]) {
@ -396,7 +388,7 @@ mod tests {
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
foo
};
@ -497,7 +489,7 @@ mod tests {
let key = jdb.insert(b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), b"dog");
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
jdb.inject_batch().unwrap();

29
util/src/journaldb/earlymergedb.rs
View File

@ -150,7 +150,7 @@ impl EarlyMergeDB {
backing.get(col, &Self::morph_key(key, 0)).expect("Low-level database error. Some issue with your hard disk?").is_some()
}
fn insert_keys(inserts: &[(H256, Bytes)], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>, batch: &mut DBTransaction, trace: bool) {
fn insert_keys(inserts: &[(H256, DBValue)], backing: &Database, col: Option<u32>, refs: &mut HashMap<H256, RefInfo>, batch: &mut DBTransaction, trace: bool) {
for &(ref h, ref d) in inserts {
if let Some(c) = refs.get_mut(h) {
// already counting. increment.
@ -268,8 +268,8 @@ impl EarlyMergeDB {
}
}
fn payload(&self, key: &H256) -> Option<Bytes> {
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
fn payload(&self, key: &H256) -> Option<DBValue> {
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
}
fn read_refs(db: &Database, col: Option<u32>) -> (Option<u64>, HashMap<H256, RefInfo>) {
@ -317,19 +317,12 @@ impl HashDB for EarlyMergeDB {
ret
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
let k = self.overlay.raw(key);
match k {
Some((d, rc)) if rc > 0 => Some(d),
_ => {
if let Some(x) = self.payload(key) {
Some(self.overlay.denote(key, x).0)
}
else {
None
}
}
if let Some((d, rc)) = k {
if rc > 0 { return Some(d) }
}
self.payload(key)
}
fn contains(&self, key: &H256) -> bool {
@ -339,7 +332,7 @@ impl HashDB for EarlyMergeDB {
fn insert(&mut self, value: &[u8]) -> H256 {
self.overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
fn emplace(&mut self, key: H256, value: DBValue) {
self.overlay.emplace(key, value);
}
fn remove(&mut self, key: &H256) {
@ -413,7 +406,7 @@ impl JournalDB for EarlyMergeDB {
.iter()
.filter_map(|(k, &(_, c))| if c < 0 {Some(k.clone())} else {None})
.collect();
let inserts: Vec<(H256, Bytes)> = drained
let inserts: Vec<(H256, _)> = drained
.into_iter()
.filter_map(|(k, (v, r))| if r > 0 { assert!(r == 1); Some((k, v)) } else { assert!(r >= -1); None })
.collect();
@ -832,7 +825,7 @@ mod tests {
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
@ -1088,7 +1081,7 @@ mod tests {
let key = jdb.insert(b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), b"dog");
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
jdb.inject_batch().unwrap();

56
util/src/journaldb/overlayrecentdb.rs
View File

@ -67,7 +67,7 @@ pub struct OverlayRecentDB {
#[derive(PartialEq)]
struct JournalOverlay {
backing_overlay: MemoryDB, // Nodes added in the history period
pending_overlay: H256FastMap<Bytes>, // Nodes being transfered from backing_overlay to backing db
pending_overlay: H256FastMap<DBValue>, // Nodes being transfered from backing_overlay to backing db
journal: HashMap<u64, Vec<JournalEntry>>,
latest_era: Option<u64>,
earliest_era: Option<u64>,
@ -130,7 +130,7 @@ impl OverlayRecentDB {
journal_overlay.latest_era == reconstructed.latest_era
}
fn payload(&self, key: &H256) -> Option<Bytes> {
fn payload(&self, key: &H256) -> Option<DBValue> {
self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?")
}
@ -160,8 +160,8 @@ impl OverlayRecentDB {
let mut inserted_keys = Vec::new();
for r in insertions.iter() {
let k: H256 = r.val_at(0);
let v: Bytes = r.val_at(1);
overlay.emplace(to_short_key(&k), v);
let v = r.at(1).data();
overlay.emplace(to_short_key(&k), DBValue::from_slice(v));
inserted_keys.push(k);
count += 1;
}
@ -229,7 +229,7 @@ impl JournalDB for OverlayRecentDB {
let journal_overlay = self.journal_overlay.read();
let key = to_short_key(key);
journal_overlay.backing_overlay.get(&key).map(|v| v.to_vec())
.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
.or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone().to_vec()))
.or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.to_vec()))
}
@ -255,7 +255,7 @@ impl JournalDB for OverlayRecentDB {
for (k, v) in insertions {
r.begin_list(2);
r.append(&k);
r.append(&v);
r.append(&&*v);
journal_overlay.backing_overlay.emplace(to_short_key(&k), v);
}
r.append(&removed_keys);
@ -284,7 +284,7 @@ impl JournalDB for OverlayRecentDB {
let mut ops = 0;
// apply old commits' details
if let Some(ref mut records) = journal_overlay.journal.get_mut(&end_era) {
let mut canon_insertions: Vec<(H256, Bytes)> = Vec::new();
let mut canon_insertions: Vec<(H256, DBValue)> = Vec::new();
let mut canon_deletions: Vec<H256> = Vec::new();
let mut overlay_deletions: Vec<H256> = Vec::new();
let mut index = 0usize;
@ -301,7 +301,7 @@ impl JournalDB for OverlayRecentDB {
for h in &journal.insertions {
if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h)) {
if rc > 0 {
canon_insertions.push((h.clone(), d.to_owned())); //TODO: optimize this to avoid data copy
canon_insertions.push((h.clone(), d)); //TODO: optimize this to avoid data copy
}
}
}
@ -386,32 +386,18 @@ impl HashDB for OverlayRecentDB {
ret
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
let k = self.transaction_overlay.raw(key);
match k {
Some((d, rc)) if rc > 0 => Some(d),
_ => {
let v = {
let journal_overlay = self.journal_overlay.read();
let key = to_short_key(key);
journal_overlay.backing_overlay.get(&key).map(|v| v.to_vec())
.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
};
match v {
Some(x) => {
Some(self.transaction_overlay.denote(key, x).0)
}
_ => {
if let Some(x) = self.payload(key) {
Some(self.transaction_overlay.denote(key, x).0)
}
else {
None
}
}
}
}
if let Some((d, rc)) = k {
if rc > 0 { return Some(d) }
}
let v = {
let journal_overlay = self.journal_overlay.read();
let key = to_short_key(key);
journal_overlay.backing_overlay.get(&key)
.or_else(|| journal_overlay.pending_overlay.get(&key).cloned())
};
v.or_else(|| self.payload(key))
}
fn contains(&self, key: &H256) -> bool {
@ -421,7 +407,7 @@ impl HashDB for OverlayRecentDB {
fn insert(&mut self, value: &[u8]) -> H256 {
self.transaction_overlay.insert(value)
}
fn emplace(&mut self, key: H256, value: Bytes) {
fn emplace(&mut self, key: H256, value: DBValue) {
self.transaction_overlay.emplace(key, value);
}
fn remove(&mut self, key: &H256) {
@ -692,7 +678,7 @@ mod tests {
let mut jdb = new_db(&dir);
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
jdb.emplace(bar.clone(), DBValue::from_slice(b"bar"));
jdb.commit_batch(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
foo
@ -965,7 +951,7 @@ mod tests {
let key = jdb.insert(b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), b"dog");
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
jdb.inject_batch().unwrap();

6
util/src/journaldb/refcounteddb.rs
View File

@ -83,10 +83,10 @@ impl RefCountedDB {
impl HashDB for RefCountedDB {
fn keys(&self) -> HashMap<H256, i32> { self.forward.keys() }
fn get(&self, key: &H256) -> Option<&[u8]> { self.forward.get(key) }
fn get(&self, key: &H256) -> Option<DBValue> { self.forward.get(key) }
fn contains(&self, key: &H256) -> bool { self.forward.contains(key) }
fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
fn emplace(&mut self, key: H256, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
fn remove(&mut self, key: &H256) { self.removes.push(key.clone()); }
}
@ -326,7 +326,7 @@ mod tests {
let key = jdb.insert(b"dog");
jdb.inject_batch().unwrap();
assert_eq!(jdb.get(&key).unwrap(), b"dog");
assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog"));
jdb.remove(&key);
jdb.inject_batch().unwrap();

21
util/src/kvdb.rs
View File

@ -21,6 +21,7 @@ use common::*;
use elastic_array::*;
use std::default::Default;
use std::path::PathBuf;
use hashdb::DBValue;
use rlp::{UntrustedRlp, RlpType, View, Compressible};
use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions};
@ -43,12 +44,12 @@ enum DBOp {
Insert {
col: Option<u32>,
key: ElasticArray32<u8>,
value: Bytes,
value: DBValue,
},
InsertCompressed {
col: Option<u32>,
key: ElasticArray32<u8>,
value: Bytes,
value: DBValue,
},
Delete {
col: Option<u32>,
@ -71,7 +72,7 @@ impl DBTransaction {
self.ops.push(DBOp::Insert {
col: col,
key: ekey,
value: value.to_vec(),
value: DBValue::from_slice(value),
});
}
@ -82,7 +83,7 @@ impl DBTransaction {
self.ops.push(DBOp::Insert {
col: col,
key: ekey,
value: value,
value: DBValue::from_vec(value),
});
}
@ -94,7 +95,7 @@ impl DBTransaction {
self.ops.push(DBOp::InsertCompressed {
col: col,
key: ekey,
value: value,
value: DBValue::from_vec(value),
});
}
@ -110,8 +111,8 @@ impl DBTransaction {
}
enum KeyState {
Insert(Bytes),
InsertCompressed(Bytes),
Insert(DBValue),
InsertCompressed(DBValue),
Delete,
}
@ -507,7 +508,7 @@ impl Database {
}
/// Get value by key.
pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<Bytes>, String> {
pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<DBValue>, String> {
match *self.db.read() {
Some(DBAndColumns { ref db, ref cfs }) => {
let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
@ -521,8 +522,8 @@ impl Database {
Some(&KeyState::Delete) => Ok(None),
None => {
col.map_or_else(
|| db.get_opt(key, &self.read_opts).map(|r| r.map(|v| v.to_vec())),
|c| db.get_cf_opt(cfs[c as usize], key, &self.read_opts).map(|r| r.map(|v| v.to_vec())))
|| db.get_opt(key, &self.read_opts).map(|r| r.map(|v| DBValue::from_slice(&v))),
|c| db.get_cf_opt(cfs[c as usize], key, &self.read_opts).map(|r| r.map(|v| DBValue::from_slice(&v))))
},
}
},

2
util/src/lib.rs
View File

@ -148,7 +148,7 @@ mod timer;
pub use common::*;
pub use misc::*;
pub use hashdb::*;
pub use memorydb::*;
pub use memorydb::MemoryDB;
pub use overlaydb::*;
pub use journaldb::JournalDB;
pub use triehash::*;

78
util/src/memorydb.rs
View File

@ -24,8 +24,6 @@ use hashdb::*;
use heapsize::*;
use std::mem;
use std::collections::HashMap;
const STATIC_NULL_RLP: (&'static [u8], i32) = (&[0x80; 1], 1);
use std::collections::hash_map::Entry;
/// Reference-counted memory-based `HashDB` implementation.
@ -73,8 +71,8 @@ use std::collections::hash_map::Entry;
/// ```
#[derive(Default, Clone, PartialEq)]
pub struct MemoryDB {
data: H256FastMap<(Bytes, i32)>,
aux: HashMap<Bytes, Bytes>,
data: H256FastMap<(DBValue, i32)>,
aux: HashMap<Bytes, DBValue>,
}
impl MemoryDB {
@ -116,12 +114,12 @@ impl MemoryDB {
}
/// Return the internal map of hashes to data, clearing the current state.
pub fn drain(&mut self) -> H256FastMap<(Bytes, i32)> {
pub fn drain(&mut self) -> H256FastMap<(DBValue, i32)> {
mem::replace(&mut self.data, H256FastMap::default())
}
/// Return the internal map of auxiliary data, clearing the current state.
pub fn drain_aux(&mut self) -> HashMap<Bytes, Bytes> {
pub fn drain_aux(&mut self) -> HashMap<Bytes, DBValue> {
mem::replace(&mut self.aux, HashMap::new())
}
@ -130,25 +128,11 @@ impl MemoryDB {
///
/// Even when Some is returned, the data is only guaranteed to be useful
/// when the refs > 0.
pub fn raw(&self, key: &H256) -> Option<(&[u8], i32)> {
pub fn raw(&self, key: &H256) -> Option<(DBValue, i32)> {
if key == &SHA3_NULL_RLP {
return Some(STATIC_NULL_RLP.clone());
return Some((DBValue::from_slice(&NULL_RLP_STATIC), 1));
}
self.data.get(key).map(|&(ref val, rc)| (&val[..], rc))
}
/// Denote than an existing value has the given key. Used when a key gets removed without
/// a prior insert and thus has a negative reference with no value.
///
/// May safely be called even if the key's value is known, in which case it will be a no-op.
pub fn denote(&self, key: &H256, value: Bytes) -> (&[u8], i32) {
if self.raw(key) == None {
unsafe {
let p = &self.data as *const H256FastMap<(Bytes, i32)> as *mut H256FastMap<(Bytes, i32)>;
(*p).insert(key.clone(), (value, 0));
}
}
self.raw(key).expect("entry just inserted into data; qed")
self.data.get(key).cloned()
}
/// Returns the size of allocated heap memory
@ -170,7 +154,7 @@ impl MemoryDB {
entry.get_mut().1 -= 1;
},
Entry::Vacant(entry) => {
entry.insert((Bytes::new(), -1));
entry.insert((DBValue::new(), -1));
}
}
}
@ -197,13 +181,13 @@ impl MemoryDB {
static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];
impl HashDB for MemoryDB {
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
if key == &SHA3_NULL_RLP {
return Some(&NULL_RLP_STATIC);
return Some(DBValue::from_slice(&NULL_RLP_STATIC));
}
match self.data.get(key) {
Some(&(ref d, rc)) if rc > 0 => Some(d),
Some(&(ref d, rc)) if rc > 0 => Some(d.clone()),
_ => None
}
}
@ -230,20 +214,20 @@ impl HashDB for MemoryDB {
let key = value.sha3();
if match self.data.get_mut(&key) {
Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => {
*old_value = value.into();
*old_value = DBValue::from_slice(value);
*rc += 1;
false
},
Some(&mut (_, ref mut x)) => { *x += 1; false } ,
None => true,
}{ // ... None falls through into...
self.data.insert(key.clone(), (value.into(), 1));
self.data.insert(key.clone(), (DBValue::from_slice(value), 1));
}
key
}
fn emplace(&mut self, key: H256, value: Bytes) {
if value == &NULL_RLP {
fn emplace(&mut self, key: H256, value: DBValue) {
if &*value == &NULL_RLP {
return;
}
@ -269,15 +253,15 @@ impl HashDB for MemoryDB {
Some(&mut (_, ref mut x)) => { *x -= 1; false }
None => true
}{ // ... None falls through into...
self.data.insert(key.clone(), (Bytes::new(), -1));
self.data.insert(key.clone(), (DBValue::new(), -1));
}
}
fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
self.aux.insert(hash, value);
self.aux.insert(hash, DBValue::from_vec(value));
}
fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
self.aux.get(hash).cloned()
}
@ -286,24 +270,6 @@ impl HashDB for MemoryDB {
}
}
#[test]
fn memorydb_denote() {
let mut m = MemoryDB::new();
let hello_bytes = b"Hello world!";
let hash = m.insert(hello_bytes);
assert_eq!(m.get(&hash).unwrap(), b"Hello world!");
for _ in 0..1000 {
let r = H256::random();
let k = r.sha3();
let (v, rc) = m.denote(&k, r.to_vec());
assert_eq!(v, &*r);
assert_eq!(rc, 0);
}
assert_eq!(m.get(&hash).unwrap(), b"Hello world!");
}
#[test]
fn memorydb_remove_and_purge() {
let hello_bytes = b"Hello world!";
@ -337,12 +303,12 @@ fn consolidate() {
main.remove(&remove_key);
let insert_key = other.insert(b"arf");
main.emplace(insert_key, b"arf".to_vec());
main.emplace(insert_key, DBValue::from_slice(b"arf"));
main.consolidate(other);
let overlay = main.drain();
assert_eq!(overlay.get(&remove_key).unwrap(), &(b"doggo".to_vec(), 0));
assert_eq!(overlay.get(&insert_key).unwrap(), &(b"arf".to_vec(), 2));
}
assert_eq!(overlay.get(&remove_key).unwrap(), &(DBValue::from_slice(b"doggo"), 0));
assert_eq!(overlay.get(&insert_key).unwrap(), &(DBValue::from_slice(b"arf"), 2));
}

19
util/src/nibbleslice.rs
View File

@ -17,7 +17,7 @@
//! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
use std::cmp::*;
use std::fmt;
use bytes::*;
use elastic_array::ElasticArray36;
/// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
///
@ -149,9 +149,9 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view {
}
/// Encode while nibble slice in prefixed hex notation, noting whether it `is_leaf`.
pub fn encoded(&self, is_leaf: bool) -> Bytes {
pub fn encoded(&self, is_leaf: bool) -> ElasticArray36<u8> {
let l = self.len();
let mut r = Bytes::with_capacity(l / 2 + 1);
let mut r = ElasticArray36::new();
let mut i = l % 2;
r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0});
while i < l {
@ -163,9 +163,9 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view {
/// Encode only the leftmost `n` bytes of the nibble slice in prefixed hex notation,
/// noting whether it `is_leaf`.
pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> Bytes {
pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> ElasticArray36<u8> {
let l = min(self.len(), n);
let mut r = Bytes::with_capacity(l / 2 + 1);
let mut r = ElasticArray36::new();
let mut i = l % 2;
r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0});
while i < l {
@ -212,6 +212,7 @@ impl<'a> fmt::Debug for NibbleSlice<'a> {
#[cfg(test)]
mod tests {
use super::NibbleSlice;
use elastic_array::ElasticArray36;
static D: &'static [u8;3] = &[0x01u8, 0x23, 0x45];
#[test]
@ -254,10 +255,10 @@ mod tests {
#[test]
fn encoded() {
let n = NibbleSlice::new(D);
assert_eq!(n.encoded(false), &[0x00, 0x01, 0x23, 0x45]);
assert_eq!(n.encoded(true), &[0x20, 0x01, 0x23, 0x45]);
assert_eq!(n.mid(1).encoded(false), &[0x11, 0x23, 0x45]);
assert_eq!(n.mid(1).encoded(true), &[0x31, 0x23, 0x45]);
assert_eq!(n.encoded(false), ElasticArray36::from_slice(&[0x00, 0x01, 0x23, 0x45]));
assert_eq!(n.encoded(true), ElasticArray36::from_slice(&[0x20, 0x01, 0x23, 0x45]));
assert_eq!(n.mid(1).encoded(false), ElasticArray36::from_slice(&[0x11, 0x23, 0x45]));
assert_eq!(n.mid(1).encoded(true), ElasticArray36::from_slice(&[0x31, 0x23, 0x45]));
}
#[test]

81
util/src/overlaydb.rs
View File

@ -18,7 +18,6 @@
use error::*;
use hash::*;
use bytes::*;
use rlp::*;
use hashdb::*;
use memorydb::*;
@ -101,21 +100,21 @@ impl OverlayDB {
pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key).map_or(0, |(_, refs)| refs) }
/// Get the refs and value of the given key.
fn payload(&self, key: &H256) -> Option<(Bytes, u32)> {
fn payload(&self, key: &H256) -> Option<(DBValue, u32)> {
self.backing.get(self.column, key)
.expect("Low-level database error. Some issue with your hard disk?")
.map(|d| {
let r = Rlp::new(&d);
(r.at(1).as_val(), r.at(0).as_val())
(DBValue::from_slice(r.at(1).data()), r.at(0).as_val())
})
}
/// Put the refs and value of the given key, possibly deleting it from the db.
fn put_payload_in_batch(&self, batch: &mut DBTransaction, key: &H256, payload: (Bytes, u32)) -> bool {
fn put_payload_in_batch(&self, batch: &mut DBTransaction, key: &H256, payload: (DBValue, u32)) -> bool {
if payload.1 > 0 {
let mut s = RlpStream::new_list(2);
s.append(&payload.1);
s.append(&payload.0);
s.append(&&*payload.0);
batch.put(self.column, key, s.as_raw());
false
} else {
@ -140,29 +139,31 @@ impl HashDB for OverlayDB {
}
ret
}
fn get(&self, key: &H256) -> Option<&[u8]> {
fn get(&self, key: &H256) -> Option<DBValue> {
// return ok if positive; if negative, check backing - might be enough references there to make
// it positive again.
let k = self.overlay.raw(key);
match k {
Some((d, rc)) if rc > 0 => Some(d),
_ => {
let memrc = k.map_or(0, |(_, rc)| rc);
match self.payload(key) {
Some(x) => {
let (d, rc) = x;
if rc as i32 + memrc > 0 {
Some(self.overlay.denote(key, d).0)
}
else {
None
}
}
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => Some(d),
_ => None,
let memrc = {
if let Some((d, rc)) = k {
if rc > 0 { return Some(d); }
rc
} else {
0
}
};
match self.payload(key) {
Some(x) => {
let (d, rc) = x;
if rc as i32 + memrc > 0 {
Some(d)
}
else {
None
}
}
// Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done.
//Some((d, rc)) if rc + memrc > 0 => Some(d),
_ => None,
}
}
fn contains(&self, key: &H256) -> bool {
@ -186,7 +187,7 @@ impl HashDB for OverlayDB {
}
}
fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) }
fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); }
fn emplace(&mut self, key: H256, value: DBValue) { self.overlay.emplace(key, value); }
fn remove(&mut self, key: &H256) { self.overlay.remove(key); }
}
@ -211,7 +212,7 @@ fn overlaydb_revert() {
fn overlaydb_overlay_insert_and_remove() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
trie.remove(&h);
assert_eq!(trie.get(&h), None);
}
@ -220,11 +221,11 @@ fn overlaydb_overlay_insert_and_remove() {
fn overlaydb_backing_insert_revert() {
let mut trie = OverlayDB::new_temp();
let h = trie.insert(b"hello world");
assert_eq!(trie.get(&h).unwrap(), b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
trie.commit().unwrap();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
trie.revert();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
}
#[test]
@ -248,7 +249,7 @@ fn overlaydb_backing_remove_revert() {
trie.remove(&h);
assert_eq!(trie.get(&h), None);
trie.revert();
assert_eq!(trie.get(&h).unwrap(), b"hello world");
assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world"));
}
#[test]
@ -266,29 +267,29 @@ fn overlaydb_negative() {
fn overlaydb_complex() {
let mut trie = OverlayDB::new_temp();
let hfoo = trie.insert(b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
let hbar = trie.insert(b"bar");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.insert(b"foo"); // two refs
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hbar).unwrap(), b"bar");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar"));
trie.remove(&hbar); // zero refs - delete
assert_eq!(trie.get(&hbar), None);
trie.remove(&hfoo); // one ref - keep
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - would delete, but...
assert_eq!(trie.get(&hfoo), None);
trie.insert(b"foo"); // one ref - keep after all.
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.commit().unwrap();
assert_eq!(trie.get(&hfoo).unwrap(), b"foo");
assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo"));
trie.remove(&hfoo); // zero ref - delete
assert_eq!(trie.get(&hfoo), None);
trie.commit().unwrap(); //

10
util/src/trie/fatdb.rs
View File

@ -16,7 +16,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use hashdb::{HashDB, DBValue};
use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder};
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@ -58,7 +58,7 @@ impl<'db> Trie for FatDB<'db> {
self.raw.contains(&key.sha3())
}
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
where 'a: 'b, R: Recorder
{
self.raw.get_recorded(&key.sha3(), rec)
@ -88,7 +88,7 @@ impl<'db> Iterator for FatDBIterator<'db> {
self.trie_iterator.next()
.map(|res|
res.map(|(hash, value)| {
(self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value)
(self.trie.db().get_aux(&hash).expect("Missing fatdb hash").to_vec(), value)
})
)
}
@ -106,6 +106,6 @@ fn fatdb_to_trie() {
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
}
let t = FatDB::new(&memdb, &root).unwrap();
assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.iter().unwrap().map(Result::unwrap).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]);
assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
assert_eq!(t.iter().unwrap().map(Result::unwrap).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]);
}

6
util/src/trie/fatdbmut.rs
View File

@ -16,7 +16,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use hashdb::{HashDB, DBValue};
use super::{TrieDBMut, TrieMut};
/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
@ -66,7 +66,7 @@ impl<'db> TrieMut for FatDBMut<'db> {
self.raw.contains(&key.sha3())
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<&'a [u8]>>
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<DBValue>>
where 'a: 'key
{
self.raw.get(&key.sha3())
@ -98,5 +98,5 @@ fn fatdb_to_trie() {
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
}
let t = TrieDB::new(&memdb, &root).unwrap();
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
}

8
util/src/trie/journal.rs
View File

@ -24,7 +24,7 @@ use hashdb::*;
/// Type of operation for the backing database - either a new node or a node deletion.
#[derive(Debug)]
enum Operation {
New(H256, Bytes),
New(H256, DBValue),
Delete(H256),
}
@ -52,16 +52,16 @@ impl Journal {
/// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal`
/// such that the reference is valid, once applied.
pub fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) {
pub fn new_node(&mut self, rlp: DBValue, out: &mut RlpStream) {
if rlp.len() >= 32 {
let rlp_sha3 = rlp.sha3();
trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty());
trace!("new_node: reference node {:?} => {:?}", rlp_sha3, &*rlp);
out.append(&rlp_sha3);
self.0.push(Operation::New(rlp_sha3, rlp));
}
else {
trace!("new_node: inline node {:?}", rlp.pretty());
trace!("new_node: inline node {:?}", &*rlp);
out.append_raw(&rlp, 1);
}
}

12
util/src/trie/mod.rs
View File

@ -18,7 +18,7 @@
use std::fmt;
use hash::H256;
use hashdb::HashDB;
use hashdb::{HashDB, DBValue};
/// Export the standardmap module.
pub mod standardmap;
@ -76,7 +76,7 @@ impl fmt::Display for TrieError {
pub type Result<T> = ::std::result::Result<T, Box<TrieError>>;
/// Trie-Item type.
pub type TrieItem<'a> = Result<(Vec<u8>, &'a [u8])>;
pub type TrieItem<'a> = Result<(Vec<u8>, DBValue)>;
/// A key-value datastore implemented as a database-backed modified Merkle tree.
pub trait Trie {
@ -92,13 +92,13 @@ pub trait Trie {
}
/// What is the value of the given key in this trie?
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<&'a [u8]>> where 'a: 'key {
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key {
self.get_recorded(key, &mut recorder::NoOp)
}
/// Query the value of the given key in this trie while recording visited nodes
/// to the given recorder. If the query fails, the nodes passed to the recorder are unspecified.
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<&'a [u8]>>
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<DBValue>>
where 'a: 'b, R: Recorder;
/// Returns an iterator over elements of trie.
@ -119,7 +119,7 @@ pub trait TrieMut {
}
/// What is the value of the given key in this trie?
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<&'a [u8]>> where 'a: 'key;
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key;
/// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing
/// `key` from the trie.
@ -188,7 +188,7 @@ impl<'db> Trie for TrieKinds<'db> {
wrapper!(self, contains, key)
}
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<&'a [u8]>>
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<DBValue>>
where 'a: 'b, R: Recorder {
wrapper!(self, get_recorded, key, r)
}

84
util/src/trie/node.rs
View File

@ -14,27 +14,51 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use elastic_array::ElasticArray36;
use nibbleslice::*;
use bytes::*;
use rlp::*;
use super::journal::*;
use hashdb::DBValue;
/// Partial node key type.
pub type NodeKey = ElasticArray36<u8>;
/// Type of node in the trie and essential information thereof.
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum Node<'a> {
#[derive(Eq, PartialEq, Debug)]
pub enum Node {
/// Null trie node; could be an empty root or an empty branch entry.
Empty,
/// Leaf node; has key slice and value. Value may not be empty.
Leaf(NibbleSlice<'a>, &'a [u8]),
Leaf(NodeKey, DBValue),
/// Extension node; has key slice and node data. Data may not be null.
Extension(NibbleSlice<'a>, &'a [u8]),
Extension(NodeKey, DBValue),
/// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data.
Branch([&'a [u8]; 16], Option<&'a [u8]>)
Branch([NodeKey; 16], Option<DBValue>)
}
impl<'a> Node<'a> {
impl Clone for Node {
fn clone(&self) -> Node {
match *self {
Node::Empty => Node::Empty,
Node::Leaf(ref k, ref v) => Node::Leaf(k.clone(), v.clone()),
Node::Extension(ref k, ref v) => Node::Extension(k.clone(), v.clone()),
Node::Branch(ref k, ref v) => {
let mut branch = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
for i in 0 .. 16 {
branch[i] = k[i].clone();
}
Node::Branch(branch, v.clone())
}
}
}
}
impl Node {
/// Decode the `node_rlp` and return the Node.
pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> {
pub fn decoded(node_rlp: &[u8]) -> Node {
let r = Rlp::new(node_rlp);
match r.prototype() {
// either leaf or extension - decode first item with NibbleSlice::???
@ -43,16 +67,18 @@ impl<'a> Node<'a> {
// if extension, second item is a node (either SHA3 to be looked up and
// fed back into this function or inline RLP which can be fed back into this function).
Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
(slice, true) => Node::Leaf(slice, r.at(1).data()),
(slice, false) => Node::Extension(slice, r.at(1).as_raw()),
(slice, true) => Node::Leaf(slice.encoded(true), DBValue::from_slice(r.at(1).data())),
(slice, false) => Node::Extension(slice.encoded(false), DBValue::from_slice(r.at(1).as_raw())),
},
// branch - first 16 are nodes, 17th is a value (or empty).
Prototype::List(17) => {
let mut nodes: [&'a [u8]; 16] = [&[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[], &[]];
let mut nodes: [NodeKey; 16] = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
for i in 0..16 {
nodes[i] = r.at(i).as_raw();
nodes[i] = NodeKey::from_slice(r.at(i).as_raw());
}
Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(DBValue::from_slice(r.at(16).data())) })
},
// an empty branch index.
Prototype::Data(0) => Node::Empty,
@ -69,23 +95,23 @@ impl<'a> Node<'a> {
match *self {
Node::Leaf(ref slice, ref value) => {
let mut stream = RlpStream::new_list(2);
stream.append(&slice.encoded(true));
stream.append(value);
stream.append(&&**slice);
stream.append(&&**value);
stream.out()
},
Node::Extension(ref slice, raw_rlp) => {
Node::Extension(ref slice, ref raw_rlp) => {
let mut stream = RlpStream::new_list(2);
stream.append(&slice.encoded(false));
stream.append_raw(raw_rlp, 1);
stream.append(&&**slice);
stream.append_raw(&&*raw_rlp, 1);
stream.out()
},
Node::Branch(ref nodes, ref value) => {
let mut stream = RlpStream::new_list(17);
for i in 0..16 {
stream.append_raw(nodes[i], 1);
stream.append_raw(&*nodes[i], 1);
}
match *value {
Some(n) => { stream.append(&n); },
Some(ref n) => { stream.append(&&**n); },
None => { stream.append_empty_data(); },
}
stream.out()
@ -100,26 +126,26 @@ impl<'a> Node<'a> {
/// Encode the node, adding it to `journal` if necessary and return the RLP valid for
/// insertion into a parent node.
pub fn encoded_and_added(&self, journal: &mut Journal) -> Bytes {
pub fn encoded_and_added(&self, journal: &mut Journal) -> DBValue {
let mut stream = RlpStream::new();
match *self {
Node::Leaf(ref slice, ref value) => {
stream.begin_list(2);
stream.append(&slice.encoded(true));
stream.append(value);
stream.append(&&**slice);
stream.append(&&**value);
},
Node::Extension(ref slice, raw_rlp) => {
Node::Extension(ref slice, ref raw_rlp) => {
stream.begin_list(2);
stream.append(&slice.encoded(false));
stream.append_raw(raw_rlp, 1);
stream.append(&&**slice);
stream.append_raw(&&**raw_rlp, 1);
},
Node::Branch(ref nodes, ref value) => {
stream.begin_list(17);
for i in 0..16 {
stream.append_raw(nodes[i], 1);
stream.append_raw(&*nodes[i], 1);
}
match *value {
Some(n) => { stream.append(&n); },
Some(ref n) => { stream.append(&&**n); },
None => { stream.append_empty_data(); },
}
},
@ -127,13 +153,13 @@ impl<'a> Node<'a> {
stream.append_empty_data();
}
}
let node = stream.out();
let node = DBValue::from_slice(stream.as_raw());
match node.len() {
0 ... 31 => node,
_ => {
let mut stream = RlpStream::new();
journal.new_node(node, &mut stream);
stream.out()
DBValue::from_slice(stream.as_raw())
}
}
}

6
util/src/trie/sectriedb.rs
View File

@ -16,7 +16,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use hashdb::{HashDB, DBValue};
use super::triedb::TrieDB;
use super::{Trie, TrieItem, Recorder};
@ -59,7 +59,7 @@ impl<'db> Trie for SecTrieDB<'db> {
self.raw.contains(&key.sha3())
}
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
where 'a: 'b, R: Recorder
{
self.raw.get_recorded(&key.sha3(), rec)
@ -79,5 +79,5 @@ fn trie_to_sectrie() {
t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]).unwrap();
}
let t = SecTrieDB::new(&memdb, &root).unwrap();
assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
}

6
util/src/trie/sectriedbmut.rs
View File

@ -16,7 +16,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::HashDB;
use hashdb::{HashDB, DBValue};
use super::triedbmut::TrieDBMut;
use super::TrieMut;
@ -62,7 +62,7 @@ impl<'db> TrieMut for SecTrieDBMut<'db> {
self.raw.contains(&key.sha3())
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<&'a [u8]>>
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result<Option<DBValue>>
where 'a: 'key
{
self.raw.get(&key.sha3())
@ -90,5 +90,5 @@ fn sectrie_to_trie() {
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
}
let t = TrieDB::new(&memdb, &root).unwrap();
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
}

85
util/src/trie/triedb.rs
View File

@ -44,7 +44,7 @@ use super::{Trie, TrieItem, TrieError};
/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap();
/// let t = TrieDB::new(&memdb, &root).unwrap();
/// assert!(t.contains(b"foo").unwrap());
/// assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar");
/// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
/// assert!(t.db_items_remaining().unwrap().is_empty());
/// }
/// ```
@ -119,8 +119,8 @@ impl<'db> TrieDB<'db> {
};
match node {
Node::Extension(_, payload) => try!(handle_payload(payload)),
Node::Branch(payloads, _) => for payload in &payloads { try!(handle_payload(payload)) },
Node::Extension(_, ref payload) => try!(handle_payload(payload)),
Node::Branch(ref payloads, _) => for payload in payloads { try!(handle_payload(payload)) },
_ => {},
}
@ -129,18 +129,18 @@ impl<'db> TrieDB<'db> {
/// Get the root node's RLP.
fn root_node<R: Recorder>(&self, r: &mut R) -> super::Result<Node> {
self.root_data(r).map(Node::decoded)
self.root_data(r).map(|d| Node::decoded(&d))
}
/// Get the data of the root node.
fn root_data<'a, R: 'a + Recorder>(&self, r: &'a mut R) -> super::Result<&[u8]> {
fn root_data<'a, R: 'a + Recorder>(&self, r: &'a mut R) -> super::Result<DBValue> {
self.db.get(self.root).ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root)))
.map(|node| { r.record(self.root, node, 0); node })
.map(|node| { r.record(self.root, &*node, 0); node })
}
/// Get the root node as a `Node`.
fn get_node<'a, R: 'a + Recorder>(&'db self, node: &'db [u8], r: &'a mut R, depth: u32) -> super::Result<Node> {
self.get_raw_or_lookup(node, r, depth).map(Node::decoded)
self.get_raw_or_lookup(node, r, depth).map(|n| Node::decoded(&n))
}
/// Indentation helper for `formal_all`.
@ -155,20 +155,20 @@ impl<'db> TrieDB<'db> {
fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result {
match node {
Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())),
Node::Extension(ref slice, item) => {
Node::Extension(ref slice, ref item) => {
try!(write!(f, "'{:?} ", slice));
if let Ok(node) = self.get_node(item, &mut NoOp, 0) {
if let Ok(node) = self.get_node(&*item, &mut NoOp, 0) {
try!(self.fmt_all(node, f, deepness));
}
},
Node::Branch(ref nodes, ref value) => {
try!(writeln!(f, ""));
if let Some(v) = *value {
if let Some(ref v) = *value {
try!(self.fmt_indent(f, deepness + 1));
try!(writeln!(f, "=: {:?}", v.pretty()))
}
for i in 0..16 {
match self.get_node(nodes[i], &mut NoOp, 0) {
match self.get_node(&*nodes[i], &mut NoOp, 0) {
Ok(Node::Empty) => {},
Ok(n) => {
try!(self.fmt_indent(f, deepness + 1));
@ -190,11 +190,11 @@ impl<'db> TrieDB<'db> {
}
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<&'db [u8]>>
fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<DBValue>>
where 'db: 'key, R: Recorder
{
let root_rlp = try!(self.root_data(r));
self.get_from_node(root_rlp, key, r, 1)
self.get_from_node(&root_rlp, key, r, 1)
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
@ -207,18 +207,23 @@ impl<'db> TrieDB<'db> {
key: &NibbleSlice<'key>,
r: &'key mut R,
d: u32
) -> super::Result<Option<&'db [u8]>> where 'db: 'key, R: Recorder {
) -> super::Result<Option<DBValue>> where 'db: 'key, R: Recorder {
match Node::decoded(node) {
Node::Leaf(ref slice, value) if key == slice => Ok(Some(value)),
Node::Extension(ref slice, item) if key.starts_with(slice) => {
let data = try!(self.get_raw_or_lookup(item, r, d));
self.get_from_node(data, &key.mid(slice.len()), r, d + 1)
Node::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == *key => Ok(Some(value.clone())),
Node::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) {
let data = try!(self.get_raw_or_lookup(&*item, r, d));
self.get_from_node(&data, &key.mid(slice.len()), r, d + 1)
} else {
Ok(None)
}
},
Node::Branch(ref nodes, value) => match key.is_empty() {
true => Ok(value),
Node::Branch(ref nodes, ref value) => match key.is_empty() {
true => Ok(value.clone()),
false => {
let node = try!(self.get_raw_or_lookup(nodes[key.at(0) as usize], r, d));
self.get_from_node(node, &key.mid(1), r, d + 1)
let node = try!(self.get_raw_or_lookup(&*nodes[key.at(0) as usize], r, d));
self.get_from_node(&node, &key.mid(1), r, d + 1)
}
},
_ => Ok(None)
@ -228,16 +233,16 @@ impl<'db> TrieDB<'db> {
/// Given some node-describing data `node`, return the actual node RLP.
/// This could be a simple identity operation in the case that the node is sufficiently small, but
/// may require a database lookup.
fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<&'db [u8]> {
fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<DBValue> {
// check if its sha3 + len
let r = Rlp::new(node);
match r.is_data() && r.size() == 32 {
true => {
let key = r.as_val::<H256>();
self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
.map(|raw| { rec.record(&key, raw, d); raw })
.map(|raw| { rec.record(&key, &raw, d); raw })
}
false => Ok(node)
false => Ok(DBValue::from_slice(node))
}
}
}
@ -251,12 +256,12 @@ enum Status {
}
#[derive(Clone, Eq, PartialEq)]
struct Crumb<'a> {
node: Node<'a>,
struct Crumb {
node: Node,
status: Status,
}
impl<'a> Crumb<'a> {
impl Crumb {
/// Move on to next status in the node's sequence.
fn increment(&mut self) {
self.status = match (&self.status, &self.node) {
@ -273,7 +278,7 @@ impl<'a> Crumb<'a> {
#[derive(Clone)]
pub struct TrieDBIterator<'a> {
db: &'a TrieDB<'a>,
trail: Vec<Crumb<'a>>,
trail: Vec<Crumb>,
key_nibbles: Bytes,
}
@ -286,18 +291,18 @@ impl<'a> TrieDBIterator<'a> {
key_nibbles: Vec::new(),
};
try!(db.root_data(&mut NoOp).and_then(|root| r.descend(root)));
try!(db.root_data(&mut NoOp).and_then(|root| r.descend(&root)));
Ok(r)
}
/// Descend into a payload.
fn descend(&mut self, d: &'a [u8]) -> super::Result<()> {
fn descend(&mut self, d: &[u8]) -> super::Result<()> {
self.trail.push(Crumb {
status: Status::Entering,
node: try!(self.db.get_node(d, &mut NoOp, 0)),
});
match self.trail.last().expect("just pushed item; qed").node {
Node::Leaf(n, _) | Node::Extension(n, _) => { self.key_nibbles.extend(n.iter()); },
Node::Leaf(ref n, _) | Node::Extension(ref n, _) => { self.key_nibbles.extend(NibbleSlice::from_encoded(n).0.iter()); },
_ => {}
}
@ -325,7 +330,7 @@ impl<'a> Iterator for TrieDBIterator<'a> {
match n {
Node::Leaf(n, _) | Node::Extension(n, _) => {
let l = self.key_nibbles.len();
self.key_nibbles.truncate(l - n.len());
self.key_nibbles.truncate(l - NibbleSlice::from_encoded(&*n).0.len());
},
Node::Branch(_, _) => { self.key_nibbles.pop(); },
_ => {}
@ -337,19 +342,19 @@ impl<'a> Iterator for TrieDBIterator<'a> {
return Some(Ok((self.key(), v)));
},
(Status::At, Node::Extension(_, d)) => {
if let Err(e) = self.descend(d) {
if let Err(e) = self.descend(&*d) {
return Some(Err(e));
}
// continue
},
(Status::At, Node::Branch(_, _)) => {},
(Status::AtChild(i), Node::Branch(children, _)) if children[i].len() > 0 => {
(Status::AtChild(i), Node::Branch(ref children, _)) if children[i].len() > 0 => {
match i {
0 => self.key_nibbles.push(0),
i => *self.key_nibbles.last_mut()
.expect("pushed as 0; moves sequentially; removed afterwards; qed") = i as u8,
}
if let Err(e) = self.descend(children[i]) {
if let Err(e) = self.descend(&*children[i]) {
return Some(Err(e));
}
// continue
@ -373,7 +378,7 @@ impl<'db> Trie for TrieDB<'db> {
fn root(&self) -> &H256 { self.root }
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<&'a [u8]>>
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
where 'a: 'b, R: Recorder
{
self.do_lookup(&NibbleSlice::new(key), rec)
@ -384,7 +389,7 @@ impl<'db> fmt::Debug for TrieDB<'db> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
try!(writeln!(f, "c={:?} [", self.hash_count));
let root_rlp = self.db.get(self.root).expect("Trie root not found!");
try!(self.fmt_all(Node::decoded(root_rlp), f, 0));
try!(self.fmt_all(Node::decoded(&root_rlp), f, 0));
writeln!(f, "]")
}
}
@ -395,7 +400,7 @@ fn iterator() {
use super::TrieMut;
use super::triedbmut::*;
let d = vec![ &b"A"[..], &b"AA"[..], &b"AB"[..], &b"B"[..] ];
let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ];
let mut memdb = MemoryDB::new();
let mut root = H256::new();
@ -407,6 +412,6 @@ fn iterator() {
}
let t = TrieDB::new(&memdb, &root).unwrap();
assert_eq!(d.iter().map(|i|i.to_vec()).collect::<Vec<_>>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::<Vec<_>>());
assert_eq!(d.iter().map(|i| i.clone().to_vec()).collect::<Vec<_>>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::<Vec<_>>());
assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::<Vec<_>>());
}

96
util/src/trie/triedbmut.rs
View File

@ -18,12 +18,14 @@
use super::{TrieError, TrieMut};
use super::node::Node as RlpNode;
use super::node::NodeKey;
use ::{Bytes, HashDB, H256};
use ::{HashDB, H256};
use ::bytes::ToPretty;
use ::nibbleslice::NibbleSlice;
use ::rlp::{Rlp, RlpStream, View, Stream};
use ::sha3::SHA3_NULL_RLP;
use hashdb::DBValue;
use elastic_array::ElasticArray1024;
@ -72,14 +74,14 @@ enum Node {
/// A leaf node contains the end of a key and a value.
/// This key is encoded from a `NibbleSlice`, meaning it contains
/// a flag indicating it is a leaf.
Leaf(Bytes, Bytes),
Leaf(NodeKey, DBValue),
/// An extension contains a shared portion of a key and a child node.
/// The shared portion is encoded from a `NibbleSlice` meaning it contains
/// a flag indicating it is an extension.
/// The child node is always a branch.
Extension(Bytes, NodeHandle),
Extension(NodeKey, NodeHandle),
/// A branch has up to 16 children and an optional value.
Branch(Box<[Option<NodeHandle>; 16]>, Option<Bytes>)
Branch(Box<[Option<NodeHandle>; 16]>, Option<DBValue>)
}
impl Node {
@ -98,21 +100,18 @@ impl Node {
fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self {
match RlpNode::decoded(rlp) {
RlpNode::Empty => Node::Empty,
RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), v.to_owned()),
RlpNode::Extension(partial, cb) => {
let key = partial.encoded(false);
Node::Extension(key, Self::inline_or_hash(cb, db, storage))
RlpNode::Leaf(k, v) => Node::Leaf(k, v),
RlpNode::Extension(key, cb) => {
Node::Extension(key, Self::inline_or_hash(&*cb, db, storage))
}
RlpNode::Branch(children_rlp, v) => {
let val = v.map(|x| x.to_owned());
RlpNode::Branch(children_rlp, val) => {
let mut children = empty_children();
for i in 0..16 {
let raw = children_rlp[i];
let child_rlp = Rlp::new(raw);
let raw = &children_rlp[i];
let child_rlp = Rlp::new(&*raw);
if !child_rlp.is_empty() {
children[i] = Some(Self::inline_or_hash(raw, db, storage));
children[i] = Some(Self::inline_or_hash(&*raw, db, storage));
}
}
@ -134,13 +133,13 @@ impl Node {
}
Node::Leaf(partial, value) => {
let mut stream = RlpStream::new_list(2);
stream.append(&partial);
stream.append(&value);
stream.append(&&*partial);
stream.append(&&*value);
stream.drain()
}
Node::Extension(partial, child) => {
let mut stream = RlpStream::new_list(2);
stream.append(&partial);
stream.append(&&*partial);
child_cb(child, &mut stream);
stream.drain()
}
@ -154,7 +153,7 @@ impl Node {
}
}
if let Some(value) = value {
stream.append(&value);
stream.append(&&*value);
} else {
stream.append_empty_data();
}
@ -276,7 +275,7 @@ impl<'a> Index<&'a StorageHandle> for NodeStorage {
/// assert_eq!(*t.root(), ::util::sha3::SHA3_NULL_RLP);
/// t.insert(b"foo", b"bar").unwrap();
/// assert!(t.contains(b"foo").unwrap());
/// assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar");
/// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
/// t.remove(b"foo").unwrap();
/// assert!(!t.contains(b"foo").unwrap());
/// }
@ -338,7 +337,7 @@ impl<'a> TrieDBMut<'a> {
// cache a node by hash
fn cache(&mut self, hash: H256) -> super::Result<StorageHandle> {
let node_rlp = try!(self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash))));
let node = Node::from_rlp(node_rlp, &*self.db, &mut self.storage);
let node = Node::from_rlp(&node_rlp, &*self.db, &mut self.storage);
Ok(self.storage.alloc(Stored::Cached(node, hash)))
}
@ -367,7 +366,7 @@ impl<'a> TrieDBMut<'a> {
}
// walk the trie, attempting to find the key's node.
fn lookup<'x, 'key>(&'x self, partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result<Option<&'x [u8]>>
fn lookup<'x, 'key>(&'x self, partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result<Option<DBValue>>
where 'x: 'key
{
match *handle {
@ -376,7 +375,7 @@ impl<'a> TrieDBMut<'a> {
Node::Empty => Ok(None),
Node::Leaf(ref key, ref value) => {
if NibbleSlice::from_encoded(key).0 == partial {
Ok(Some(value))
Ok(Some(DBValue::from_slice(value)))
} else {
Ok(None)
}
@ -391,7 +390,7 @@ impl<'a> TrieDBMut<'a> {
}
Node::Branch(ref children, ref value) => {
if partial.is_empty() {
Ok(value.as_ref().map(|v| &v[..]))
Ok(value.as_ref().map(|v| DBValue::from_slice(v)))
} else {
let idx = partial.at(0);
match children[idx as usize].as_ref() {
@ -405,28 +404,33 @@ impl<'a> TrieDBMut<'a> {
}
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<&'x [u8]>>
fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
where 'x: 'key
{
self.db.get(hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(*hash)))
.and_then(|node_rlp| self.get_from_db_node(node_rlp, key))
.and_then(|node_rlp| self.get_from_db_node(&node_rlp, key))
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
/// value exists for the key.
///
/// Note: Not a public API; use Trie trait functions.
fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<&'x [u8]>>
fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
where 'x: 'key
{
match RlpNode::decoded(node) {
RlpNode::Leaf(ref slice, value) if &key == slice => Ok(Some(value)),
RlpNode::Extension(ref slice, item) if key.starts_with(slice) => {
self.get_from_db_node(try!(self.get_raw_or_lookup(item)), key.mid(slice.len()))
RlpNode::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == key => Ok(Some(value.clone())),
RlpNode::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) {
self.get_from_db_node(&try!(self.get_raw_or_lookup(&*item)), key.mid(slice.len()))
} else {
Ok(None)
}
},
RlpNode::Branch(ref nodes, value) => match key.is_empty() {
true => Ok(value),
false => self.get_from_db_node(try!(self.get_raw_or_lookup(nodes[key.at(0) as usize])), key.mid(1))
RlpNode::Branch(ref nodes, ref value) => match key.is_empty() {
true => Ok(value.clone()),
false => self.get_from_db_node(&try!(self.get_raw_or_lookup(&*nodes[key.at(0) as usize])), key.mid(1))
},
_ => Ok(None),
}
@ -435,7 +439,7 @@ impl<'a> TrieDBMut<'a> {
/// Given some node-describing data `node`, return the actual node RLP.
/// This could be a simple identity operation in the case that the node is sufficiently small, but
/// may require a database lookup.
fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<&'x [u8]> {
fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<DBValue> {
// check if its sha3 + len
let r = Rlp::new(node);
match r.is_data() && r.size() == 32 {
@ -443,12 +447,12 @@ impl<'a> TrieDBMut<'a> {
let key = r.as_val::<H256>();
self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
}
false => Ok(node)
false => Ok(DBValue::from_slice(node))
}
}
/// insert a key, value pair into the trie, creating new nodes if necessary.
fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: Bytes) -> super::Result<(StorageHandle, bool)> {
fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue) -> super::Result<(StorageHandle, bool)> {
let h = match handle {
NodeHandle::InMemory(h) => h,
NodeHandle::Hash(h) => try!(self.cache(h)),
@ -463,7 +467,7 @@ impl<'a> TrieDBMut<'a> {
/// the insertion inspector.
#[cfg_attr(feature = "dev", allow(cyclomatic_complexity))]
fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: Bytes) -> super::Result<InsertAction> {
fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue) -> super::Result<InsertAction> {
trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty());
Ok(match node {
@ -898,7 +902,7 @@ impl<'a> TrieMut for TrieDBMut<'a> {
}
}
fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result<Option<&'x [u8]>> where 'x: 'key {
fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result<Option<DBValue>> where 'x: 'key {
self.lookup(NibbleSlice::new(key), &self.root_handle)
}
@ -911,7 +915,7 @@ impl<'a> TrieMut for TrieDBMut<'a> {
trace!(target: "trie", "insert: key={:?}, value={:?}", key.pretty(), value.pretty());
let root_handle = self.root_handle();
let (new_handle, changed) = try!(self.insert_at(root_handle, NibbleSlice::new(key), value.to_owned()));
let (new_handle, changed) = try!(self.insert_at(root_handle, NibbleSlice::new(key), DBValue::from_slice(value)));
trace!(target: "trie", "insert: altered trie={}", changed);
self.root_handle = NodeHandle::InMemory(new_handle);
@ -1180,9 +1184,9 @@ mod tests {
let mut root = H256::new();
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), &[0x1u8, 0x23]);
assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23]));
t.commit();
assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), &[0x1u8, 0x23]);
assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23]));
}
#[test]
@ -1193,14 +1197,14 @@ mod tests {
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap();
t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap();
t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap();
assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), &[0xf1u8, 0x23]);
assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), &[0x81u8, 0x23]);
assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23]));
assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23]));
assert_eq!(t.get(&[0x82, 0x23]), Ok(None));
t.commit();
assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), &[0x01u8, 0x23]);
assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), &[0xf1u8, 0x23]);
assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), &[0x81u8, 0x23]);
assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23]));
assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23]));
assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23]));
assert_eq!(t.get(&[0x82, 0x23]), Ok(None));
}