grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2516 from ethcore/canon-cache-size

Browse Source 
Mostly configurable canonical cache size
This commit is contained in:
Robert Habermeier
2016-10-12 21:48:29 +02:00
committed by GitHub
parent c9ce25c8f3 4276ab8a5f
commit 4bcc9e3b49
19 changed files with 230 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ethcore/Cargo.toml
View File

@@ -37,7 +37,7 @@ ethkey = { path = "../ethkey" }
ethcore-ipc-nano = { path = "../ipc/nano" }
rlp = { path = "../util/rlp" }
rand = "0.3"
lru-cache = "0.0.7"
lru-cache = { git = "https://github.com/contain-rs/lru-cache" }
ethcore-bloom-journal = { path = "../util/bloom" }
byteorder = "0.5"

7
ethcore/src/client/client.rs
View File

@@ -170,7 +170,7 @@ impl Client {
};
let journal_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
let mut state_db = StateDB::new(journal_db);
let mut state_db = StateDB::new(journal_db, config.state_cache_size);
if state_db.journal_db().is_empty() && try!(spec.ensure_db_good(&mut state_db)) {
let mut batch = DBTransaction::new(&db);
try!(state_db.commit(&mut batch, 0, &spec.genesis_header().hash(), None));
@@ -190,7 +190,7 @@ impl Client {
let awake = match config.mode { Mode::Dark(..) => false, _ => true };
let factories = Factories {
vm: EvmFactory::new(config.vm_type.clone()),
vm: EvmFactory::new(config.vm_type.clone(), config.jump_table_size),
trie: TrieFactory::new(trie_spec),
accountdb: Default::default(),
};
@@ -682,7 +682,8 @@ impl snapshot::DatabaseRestore for Client {
let db = self.db.write();
try!(db.restore(new_db));
*state_db = StateDB::new(journaldb::new(db.clone(), self.pruning, ::db::COL_STATE));
let cache_size = state_db.cache_size();
*state_db = StateDB::new(journaldb::new(db.clone(), self.pruning, ::db::COL_STATE), cache_size);
*chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone()));
*tracedb = TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone());
Ok(())

6
ethcore/src/client/config.rs
View File

@@ -96,7 +96,7 @@ pub struct ClientConfig {
pub pruning: journaldb::Algorithm,
/// The name of the client instance.
pub name: String,
/// State db cache-size if not default
/// RocksDB state column cache-size if not default
pub db_cache_size: Option<usize>,
/// State db compaction profile
pub db_compaction: DatabaseCompactionProfile,
@@ -106,6 +106,10 @@ pub struct ClientConfig {
pub mode: Mode,
/// Type of block verifier used by client.
pub verifier_type: VerifierType,
/// State db cache-size.
pub state_cache_size: usize,
/// EVM jump-tables cache size.
pub jump_table_size: usize,
}
#[cfg(test)]

4
ethcore/src/client/test_client.rs
View File

@@ -130,7 +130,7 @@ impl TestBlockChainClient {
queue_size: AtomicUsize::new(0),
miner: Arc::new(Miner::with_spec(&spec)),
spec: spec,
vm_factory: EvmFactory::new(VMType::Interpreter),
vm_factory: EvmFactory::new(VMType::Interpreter, 1024 * 1024),
latest_block_timestamp: RwLock::new(10_000_000),
};
client.add_blocks(1, EachBlockWith::Nothing); // add genesis block
@@ -298,7 +298,7 @@ pub fn get_temp_state_db() -> GuardedTempResult<StateDB> {
let temp = RandomTempPath::new();
let db = Database::open(&DatabaseConfig::with_columns(NUM_COLUMNS), temp.as_str()).unwrap();
let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, COL_STATE);
let state_db = StateDB::new(journal_db);
let state_db = StateDB::new(journal_db, 1024 * 1024);
GuardedTempResult {
_temp: temp,
result: Some(state_db)

19
ethcore/src/evm/factory.rs
View File

@@ -118,11 +118,12 @@ impl Factory {
}
}
/// Create new instance of specific `VMType` factory
pub fn new(evm: VMType) -> Self {
/// Create new instance of specific `VMType` factory, with a size in bytes
/// for caching jump destinations.
pub fn new(evm: VMType, cache_size: usize) -> Self {
Factory {
evm: evm,
evm_cache: Arc::new(SharedCache::default()),
evm_cache: Arc::new(SharedCache::new(cache_size)),
}
}
@@ -164,22 +165,22 @@ macro_rules! evm_test(
#[ignore]
#[cfg(feature = "jit")]
fn $name_jit() {
$name_test(Factory::new(VMType::Jit));
$name_test(Factory::new(VMType::Jit, 1024 * 32));
}
#[test]
fn $name_int() {
$name_test(Factory::new(VMType::Interpreter));
$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
}
};
($name_test: ident: $name_jit: ident, $name_int: ident) => {
#[test]
#[cfg(feature = "jit")]
fn $name_jit() {
$name_test(Factory::new(VMType::Jit));
$name_test(Factory::new(VMType::Jit, 1024 * 32));
}
#[test]
fn $name_int() {
$name_test(Factory::new(VMType::Interpreter));
$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
}
}
);
@@ -193,13 +194,13 @@ macro_rules! evm_test_ignore(
#[cfg(feature = "jit")]
#[cfg(feature = "ignored-tests")]
fn $name_jit() {
$name_test(Factory::new(VMType::Jit));
$name_test(Factory::new(VMType::Jit, 1024 * 32));
}
#[test]
#[ignore]
#[cfg(feature = "ignored-tests")]
fn $name_int() {
$name_test(Factory::new(VMType::Interpreter));
$name_test(Factory::new(VMType::Interpreter, 1024 * 32));
}
}
);

59
ethcore/src/evm/interpreter/shared_cache.rs
View File

@@ -21,25 +21,66 @@ use util::sha3::*;
use bit_set::BitSet;
use super::super::instructions;
const CACHE_CODE_ITEMS: usize = 65536;
const INITIAL_CAPACITY: usize = 32;
const DEFAULT_CACHE_SIZE: usize = 4 * 1024 * 1024;
/// GLobal cache for EVM interpreter
/// Global cache for EVM interpreter
pub struct SharedCache {
jump_destinations: Mutex<LruCache<H256, Arc<BitSet>>>
jump_destinations: Mutex<LruCache<H256, Arc<BitSet>>>,
max_size: usize,
cur_size: Mutex<usize>,
}
impl SharedCache {
/// Get jump destincations bitmap for a contract.
/// Create a jump destinations cache with a maximum size in bytes
/// to cache.
pub fn new(max_size: usize) -> Self {
SharedCache {
jump_destinations: Mutex::new(LruCache::new(INITIAL_CAPACITY)),
max_size: max_size * 8, // dealing with bits here.
cur_size: Mutex::new(0),
}
}
/// Get jump destinations bitmap for a contract.
pub fn jump_destinations(&self, code_hash: &H256, code: &[u8]) -> Arc<BitSet> {
if code_hash == &SHA3_EMPTY {
return Self::find_jump_destinations(code);
}
if let Some(d) = self.jump_destinations.lock().get_mut(code_hash) {
return d.clone();
}
let d = Self::find_jump_destinations(code);
self.jump_destinations.lock().insert(code_hash.clone(), d.clone());
{
let mut cur_size = self.cur_size.lock();
*cur_size += d.capacity();
let mut jump_dests = self.jump_destinations.lock();
let cap = jump_dests.capacity();
// grow the cache as necessary; it operates on amount of items
// but we're working based on memory usage.
if jump_dests.len() == cap && *cur_size < self.max_size {
jump_dests.set_capacity(cap * 2);
}
// account for any element displaced from the cache.
if let Some(lru) = jump_dests.insert(code_hash.clone(), d.clone()) {
*cur_size -= lru.capacity();
}
// remove elements until we are below the memory target.
while *cur_size > self.max_size {
match jump_dests.remove_lru() {
Some((_, v)) => *cur_size -= v.capacity(),
_ => break,
}
}
}
d
}
@@ -57,15 +98,15 @@ impl SharedCache {
}
position += 1;
}
jump_dests.shrink_to_fit();
Arc::new(jump_dests)
}
}
impl Default for SharedCache {
fn default() -> SharedCache {
SharedCache {
jump_destinations: Mutex::new(LruCache::new(CACHE_CODE_ITEMS)),
}
fn default() -> Self {
SharedCache::new(DEFAULT_CACHE_SIZE)
}
}

2
ethcore/src/evm/tests.rs
View File

@@ -817,7 +817,7 @@ fn test_signextend(factory: super::Factory) {
#[test] // JIT just returns out of gas
fn test_badinstruction_int() {
let factory = super::Factory::new(VMType::Interpreter);
let factory = super::Factory::new(VMType::Interpreter, 1024 * 32);
let code = "af".from_hex().unwrap();
let mut params = ActionParams::default();

4
ethcore/src/executive.rs
View File

@@ -598,7 +598,7 @@ mod tests {
#[test]
// Tracing is not suported in JIT
fn test_call_to_create() {
let factory = Factory::new(VMType::Interpreter);
let factory = Factory::new(VMType::Interpreter, 1024 * 32);
// code:
//
@@ -724,7 +724,7 @@ mod tests {
#[test]
fn test_create_contract() {
// Tracing is not supported in JIT
let factory = Factory::new(VMType::Interpreter);
let factory = Factory::new(VMType::Interpreter, 1024 * 32);
// code:
//
// 60 10 - push 16

2
ethcore/src/json_tests/executive.rs
View File

@@ -191,7 +191,7 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
state.populate_from(From::from(vm.pre_state.clone()));
let info = From::from(vm.env);
let engine = TestEngine::new(1);
let vm_factory = Factory::new(vm_type.clone());
let vm_factory = Factory::new(vm_type.clone(), 1024 * 32);
let params = ActionParams::from(vm.transaction);
let mut substate = Substate::new();

178
ethcore/src/state_db.rs
View File

@@ -26,17 +26,19 @@ use bloom_journal::{Bloom, BloomJournal};
use db::COL_ACCOUNT_BLOOM;
use byteorder::{LittleEndian, ByteOrder};
const STATE_CACHE_ITEMS: usize = 256000;
const STATE_CACHE_BLOCKS: usize = 8;
pub const ACCOUNT_BLOOM_SPACE: usize = 1048576;
pub const DEFAULT_ACCOUNT_PRESET: usize = 1000000;
pub const ACCOUNT_BLOOM_HASHCOUNT_KEY: &'static [u8] = b"account_hash_count";
const STATE_CACHE_BLOCKS: usize = 8;
/// Shared canonical state cache.
struct AccountCache {
/// DB Account cache. `None` indicates that account is known to be missing.
// When changing the type of the values here, be sure to update `mem_used` and
// `new`.
accounts: LruCache<Address, Option<Account>>,
/// Information on the modifications in recently committed blocks; specifically which addresses
/// changed in which block. Ordered by block number.
@@ -92,6 +94,7 @@ pub struct StateDB {
local_cache: Vec<CacheQueueItem>,
/// Shared account bloom. Does not handle chain reorganizations.
account_bloom: Arc<Mutex<Bloom>>,
cache_size: usize,
/// Hash of the block on top of which this instance was created or
/// `None` if cache is disabled
parent_hash: Option<H256>,
@@ -102,6 +105,30 @@ pub struct StateDB {
}
impl StateDB {
/// Create a new instance wrapping `JournalDB` and the maximum allowed size
/// of the LRU cache in bytes. Actual used memory may (read: will) be higher due to bookkeeping.
// TODO: make the cache size actually accurate by moving the account storage cache
// into the `AccountCache` structure as its own `LruCache<(Address, H256), H256>`.
pub fn new(db: Box<JournalDB>, cache_size: usize) -> StateDB {
let bloom = Self::load_bloom(db.backing());
let cache_items = cache_size / ::std::mem::size_of::<Option<Account>>();
StateDB {
db: db,
account_cache: Arc::new(Mutex::new(AccountCache {
accounts: LruCache::new(cache_items),
modifications: VecDeque::new(),
})),
local_cache: Vec::new(),
account_bloom: Arc::new(Mutex::new(bloom)),
cache_size: cache_size,
parent_hash: None,
commit_hash: None,
commit_number: None,
}
}
/// Loads accounts bloom from the database
/// This bloom is used to handle request for the non-existant account fast
pub fn load_bloom(db: &Database) -> Bloom {
@@ -129,23 +156,6 @@ impl StateDB {
bloom
}
/// Create a new instance wrapping `JournalDB`
pub fn new(db: Box<JournalDB>) -> StateDB {
let bloom = Self::load_bloom(db.backing());
StateDB {
db: db,
account_cache: Arc::new(Mutex::new(AccountCache {
accounts: LruCache::new(STATE_CACHE_ITEMS),
modifications: VecDeque::new(),
})),
local_cache: Vec::new(),
account_bloom: Arc::new(Mutex::new(bloom)),
parent_hash: None,
commit_hash: None,
commit_number: None,
}
}
pub fn check_account_bloom(&self, address: &Address) -> bool {
trace!(target: "account_bloom", "Check account bloom: {:?}", address);
let bloom = self.account_bloom.lock();
@@ -298,6 +308,7 @@ impl StateDB {
account_cache: self.account_cache.clone(),
local_cache: Vec::new(),
account_bloom: self.account_bloom.clone(),
cache_size: self.cache_size,
parent_hash: None,
commit_hash: None,
commit_number: None,
@@ -311,6 +322,7 @@ impl StateDB {
account_cache: self.account_cache.clone(),
local_cache: Vec::new(),
account_bloom: self.account_bloom.clone(),
cache_size: self.cache_size,
parent_hash: Some(parent.clone()),
commit_hash: None,
commit_number: None,
@@ -324,7 +336,8 @@ impl StateDB {
/// Heap size used.
pub fn mem_used(&self) -> usize {
self.db.mem_used() //TODO: + self.account_cache.lock().heap_size_of_children()
// TODO: account for LRU-cache overhead; this is a close approximation.
self.db.mem_used() + self.account_cache.lock().accounts.len() * ::std::mem::size_of::<Option<Account>>()
}
/// Returns underlying `JournalDB`.
@@ -365,6 +378,11 @@ impl StateDB {
cache.accounts.get_mut(a).map(|c| f(c.as_mut()))
}
/// Query how much memory is set aside for the accounts cache (in bytes).
pub fn cache_size(&self) -> usize {
self.cache_size
}
/// Check if the account can be returned from cache by matching current block parent hash against canonical
/// state and filtering out account modified in later blocks.
fn is_allowed(addr: &Address, parent_hash: &Option<H256>, modifications: &VecDeque<BlockChanges>) -> bool {
@@ -404,77 +422,77 @@ impl StateDB {
#[cfg(test)]
mod tests {
use util::{U256, H256, FixedHash, Address, DBTransaction};
use tests::helpers::*;
use state::Account;
use util::log::init_log;
use util::{U256, H256, FixedHash, Address, DBTransaction};
use tests::helpers::*;
use state::Account;
use util::log::init_log;
#[test]
fn state_db_smoke() {
init_log();
#[test]
fn state_db_smoke() {
init_log();
let mut state_db_result = get_temp_state_db();
let state_db = state_db_result.take();
let root_parent = H256::random();
let address = Address::random();
let h0 = H256::random();
let h1a = H256::random();
let h1b = H256::random();
let h2a = H256::random();
let h2b = H256::random();
let h3a = H256::random();
let h3b = H256::random();
let mut batch = DBTransaction::new(state_db.journal_db().backing());
let mut state_db_result = get_temp_state_db();
let state_db = state_db_result.take();
let root_parent = H256::random();
let address = Address::random();
let h0 = H256::random();
let h1a = H256::random();
let h1b = H256::random();
let h2a = H256::random();
let h2b = H256::random();
let h3a = H256::random();
let h3b = H256::random();
let mut batch = DBTransaction::new(state_db.journal_db().backing());
// blocks [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ]
// balance [ 5 5 4 3 2 2 ]
let mut s = state_db.boxed_clone_canon(&root_parent);
s.add_to_account_cache(address, Some(Account::new_basic(2.into(), 0.into())), false);
s.commit(&mut batch, 0, &h0, None).unwrap();
s.sync_cache(&[], &[], true);
// blocks [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ]
// balance [ 5 5 4 3 2 2 ]
let mut s = state_db.boxed_clone_canon(&root_parent);
s.add_to_account_cache(address, Some(Account::new_basic(2.into(), 0.into())), false);
s.commit(&mut batch, 0, &h0, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h0);
s.commit(&mut batch, 1, &h1a, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h0);
s.commit(&mut batch, 1, &h1a, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h0);
s.add_to_account_cache(address, Some(Account::new_basic(3.into(), 0.into())), true);
s.commit(&mut batch, 1, &h1b, None).unwrap();
s.sync_cache(&[], &[], false);
let mut s = state_db.boxed_clone_canon(&h0);
s.add_to_account_cache(address, Some(Account::new_basic(3.into(), 0.into())), true);
s.commit(&mut batch, 1, &h1b, None).unwrap();
s.sync_cache(&[], &[], false);
let mut s = state_db.boxed_clone_canon(&h1b);
s.add_to_account_cache(address, Some(Account::new_basic(4.into(), 0.into())), true);
s.commit(&mut batch, 2, &h2b, None).unwrap();
s.sync_cache(&[], &[], false);
let mut s = state_db.boxed_clone_canon(&h1b);
s.add_to_account_cache(address, Some(Account::new_basic(4.into(), 0.into())), true);
s.commit(&mut batch, 2, &h2b, None).unwrap();
s.sync_cache(&[], &[], false);
let mut s = state_db.boxed_clone_canon(&h1a);
s.add_to_account_cache(address, Some(Account::new_basic(5.into(), 0.into())), true);
s.commit(&mut batch, 2, &h2a, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h1a);
s.add_to_account_cache(address, Some(Account::new_basic(5.into(), 0.into())), true);
s.commit(&mut batch, 2, &h2a, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h2a);
s.commit(&mut batch, 3, &h3a, None).unwrap();
s.sync_cache(&[], &[], true);
let mut s = state_db.boxed_clone_canon(&h2a);
s.commit(&mut batch, 3, &h3a, None).unwrap();
s.sync_cache(&[], &[], true);
let s = state_db.boxed_clone_canon(&h3a);
assert_eq!(s.get_cached_account(&address).unwrap().unwrap().balance(), &U256::from(5));
let s = state_db.boxed_clone_canon(&h3a);
assert_eq!(s.get_cached_account(&address).unwrap().unwrap().balance(), &U256::from(5));
let s = state_db.boxed_clone_canon(&h1a);
assert!(s.get_cached_account(&address).is_none());
let s = state_db.boxed_clone_canon(&h1a);
assert!(s.get_cached_account(&address).is_none());
let s = state_db.boxed_clone_canon(&h2b);
assert!(s.get_cached_account(&address).is_none());
let s = state_db.boxed_clone_canon(&h2b);
assert!(s.get_cached_account(&address).is_none());
let s = state_db.boxed_clone_canon(&h1b);
assert!(s.get_cached_account(&address).is_none());
let s = state_db.boxed_clone_canon(&h1b);
assert!(s.get_cached_account(&address).is_none());
// reorg to 3b
// blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ]
let mut s = state_db.boxed_clone_canon(&h2b);
s.commit(&mut batch, 3, &h3b, None).unwrap();
s.sync_cache(&[h1b.clone(), h2b.clone(), h3b.clone()], &[h1a.clone(), h2a.clone(), h3a.clone()], true);
let s = state_db.boxed_clone_canon(&h3a);
assert!(s.get_cached_account(&address).is_none());
}
// reorg to 3b
// blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ]
let mut s = state_db.boxed_clone_canon(&h2b);
s.commit(&mut batch, 3, &h3b, None).unwrap();
s.sync_cache(&[h1b.clone(), h2b.clone(), h3b.clone()], &[h1a.clone(), h2a.clone(), h3a.clone()], true);
let s = state_db.boxed_clone_canon(&h3a);
assert!(s.get_cached_account(&address).is_none());
}
}

2
ethcore/src/tests/helpers.rs
View File

@@ -346,7 +346,7 @@ pub fn get_temp_state() -> GuardedTempResult<State> {
pub fn get_temp_state_db_in(path: &Path) -> StateDB {
let db = new_db(path.to_str().expect("Only valid utf8 paths for tests."));
let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, COL_STATE);
StateDB::new(journal_db)
StateDB::new(journal_db, 5 * 1024 * 1024)
}
pub fn get_temp_state_in(path: &Path) -> State {