grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into miner-improvements

Browse Source
This commit is contained in:
Gav Wood 2016-06-28 10:02:33 +02:00
commit af935df553
14 changed files with 182 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@ -1077,7 +1077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rocksdb"
version = "0.4.5"
source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541"
source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6"
dependencies = [
"libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
"rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)",
@ -1086,7 +1086,7 @@ dependencies = [
[[package]]
name = "rocksdb-sys"
version = "0.3.0"
source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541"
source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6"
dependencies = [
"gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",

13
ethcore/src/client/client.rs
View File

@ -38,7 +38,7 @@ use filter::Filter;
use log_entry::LocalizedLogEntry;
use block_queue::{BlockQueue, BlockQueueInfo};
use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute};
use client::{BlockID, TransactionID, UncleID, TraceId, ClientConfig, BlockChainClient, MiningBlockChainClient, TraceFilter, CallAnalytics};
use client::{BlockID, TransactionID, UncleID, TraceId, ClientConfig, DatabaseCompactionProfile, BlockChainClient, MiningBlockChainClient, TraceFilter, CallAnalytics};
use client::Error as ClientError;
use env_info::EnvInfo;
use executive::{Executive, Executed, TransactOptions, contract_address};
@ -146,10 +146,19 @@ impl<V> Client<V> where V: Verifier {
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone())));
let mut state_db_config = match config.db_cache_size {
None => DatabaseConfig::default(),
Some(cache_size) => DatabaseConfig::with_cache(cache_size),
};
if config.db_compaction == DatabaseCompactionProfile::HDD {
state_db_config = state_db_config.compaction(CompactionProfile::hdd());
}
let mut state_db = journaldb::new(
&append_path(&path, "state"),
config.pruning,
config.db_cache_size
state_db_config
);
if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {

15
ethcore/src/client/config.rs
View File

@ -20,6 +20,19 @@ pub use trace::{Config as TraceConfig, Switch};
pub use evm::VMType;
use util::journaldb;
/// Client state db compaction profile
#[derive(Debug, PartialEq)]
pub enum DatabaseCompactionProfile {
/// Default compaction profile
Default,
/// HDD or other slow storage io compaction profile
HDD,
}
impl Default for DatabaseCompactionProfile {
fn default() -> Self { DatabaseCompactionProfile::Default }
}
/// Client configuration. Includes configs for all sub-systems.
#[derive(Debug, Default)]
pub struct ClientConfig {
@ -37,4 +50,6 @@ pub struct ClientConfig {
pub name: String,
/// State db cache-size if not default
pub db_cache_size: Option<usize>,
/// State db compaction profile
pub db_compaction: DatabaseCompactionProfile,
}

2
ethcore/src/client/mod.rs
View File

@ -23,7 +23,7 @@ mod test_client;
mod trace;
pub use self::client::*;
pub use self::config::{ClientConfig, BlockQueueConfig, BlockChainConfig, Switch, VMType};
pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockQueueConfig, BlockChainConfig, Switch, VMType};
pub use self::error::Error;
pub use types::ids::*;
pub use self::test_client::{TestBlockChainClient, EachBlockWith};

4
ethcore/src/tests/helpers.rs
View File

@ -303,7 +303,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult<BlockChain> {
pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
let temp = RandomTempPath::new();
let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, None);
let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default());
GuardedTempResult {
_temp: temp,
result: Some(journal_db)
@ -320,7 +320,7 @@ pub fn get_temp_state() -> GuardedTempResult<State> {
}
pub fn get_temp_journal_db_in(path: &Path) -> Box<JournalDB> {
journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge, None)
journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default())
}
pub fn get_temp_state_in(path: &Path) -> State {

8
parity/cli.rs
View File

@ -186,7 +186,12 @@ Footprint Options:
--cache MEGABYTES Set total amount of discretionary memory to use for
the entire system, overrides other cache and queue
options.
--db-cache-size MB Database cache size.
Database Options:
--db-cache-size MB Override RocksDB database cache size.
--db-compaction TYPE Database compaction type. TYPE may be one of:
ssd - suitable for SSDs and fast HDDs;
hdd - suitable for slow HDDs [default: ssd].
Import/Export Options:
--from BLOCK Export from block BLOCK, which may be an index or
@ -342,6 +347,7 @@ pub struct Args {
pub flag_ipcpath: Option<String>,
pub flag_ipcapi: Option<String>,
pub flag_db_cache_size: Option<usize>,
pub flag_db_compaction: String,
}
pub fn print_version() {

11
parity/configuration.rs
View File

@ -26,7 +26,7 @@ use die::*;
use util::*;
use ethcore::account_provider::AccountProvider;
use util::network_settings::NetworkSettings;
use ethcore::client::{append_path, get_db_path, ClientConfig, Switch, VMType};
use ethcore::client::{append_path, get_db_path, ClientConfig, DatabaseCompactionProfile, Switch, VMType};
use ethcore::miner::{MinerOptions, PendingSet};
use ethcore::ethereum;
use ethcore::spec::Spec;
@ -259,7 +259,7 @@ impl Configuration {
let mut latest_era = None;
let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
for i in jdb_types.into_iter() {
let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, None);
let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, kvdb::DatabaseConfig::default());
trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era());
match (latest_era, db.latest_era()) {
(Some(best), Some(this)) if best >= this => {}
@ -310,6 +310,13 @@ impl Configuration {
// forced state db cache size if provided
client_config.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
// compaction profile
client_config.db_compaction = match self.args.flag_db_compaction.as_str() {
"ssd" => DatabaseCompactionProfile::Default,
"hdd" => DatabaseCompactionProfile::HDD,
_ => { die!("Invalid compaction profile given (--db-compaction argument), expected hdd/default."); }
};
if self.args.flag_jitvm {
client_config.vm_type = VMType::jit().unwrap_or_else(|| die!("Parity built without jit vm."))
}

3
parity/migration.rs
View File

@ -20,7 +20,7 @@ use std::io::{Read, Write, Error as IoError, ErrorKind};
use std::path::PathBuf;
use std::fmt::{Display, Formatter, Error as FmtError};
use util::migration::{Manager as MigrationManager, Config as MigrationConfig, MigrationIterator};
use util::kvdb::{Database, DatabaseConfig};
use util::kvdb::{Database, DatabaseConfig, CompactionProfile};
use ethcore::migrations;
/// Database is assumed to be at default version, when no version file is found.
@ -163,6 +163,7 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -
prefix_size: None,
max_open_files: 64,
cache_size: None,
compaction: CompactionProfile::default(),
};
// open old database

30
util/src/journaldb/archivedb.rs
View File

@ -43,13 +43,8 @@ const DB_VERSION : u32 = 0x103;
impl ArchiveDB {
/// Create a new instance from file
pub fn new(path: &str, cache_size: Option<usize>) -> ArchiveDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
pub fn new(path: &str, config: DatabaseConfig) -> ArchiveDB {
let opts = config.prefix(DB_PREFIX_LEN);
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
@ -75,7 +70,7 @@ impl ArchiveDB {
fn new_temp() -> ArchiveDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), None)
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
}
fn payload(&self, key: &H256) -> Option<Bytes> {
@ -187,6 +182,7 @@ mod tests {
use super::*;
use hashdb::*;
use journaldb::traits::JournalDB;
use kvdb::DatabaseConfig;
#[test]
fn insert_same_in_fork() {
@ -328,7 +324,7 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
@ -337,13 +333,13 @@ mod tests {
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
}
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@ -356,7 +352,7 @@ mod tests {
dir.push(H32::random().hex());
let foo = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
@ -370,7 +366,7 @@ mod tests {
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
assert!(jdb.contains(&foo));
@ -385,7 +381,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, _, _) = {
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
@ -400,7 +396,7 @@ mod tests {
};
{
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.contains(&foo));
}
@ -411,14 +407,14 @@ mod tests {
let temp = ::devtools::RandomTempPath::new();
let key = {
let mut jdb = ArchiveDB::new(temp.as_str(), None);
let mut jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
let key = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
key
};
{
let jdb = ArchiveDB::new(temp.as_str(), None);
let jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
let state = jdb.state(&key);
assert!(state.is_some());
}

44
util/src/journaldb/earlymergedb.rs
View File

@ -73,13 +73,8 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl EarlyMergeDB {
/// Create a new instance from file
pub fn new(path: &str, cache_size: Option<usize>) -> EarlyMergeDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
pub fn new(path: &str, config: DatabaseConfig) -> EarlyMergeDB {
let opts = config.prefix(DB_PREFIX_LEN);
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
@ -107,7 +102,7 @@ impl EarlyMergeDB {
fn new_temp() -> EarlyMergeDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), None)
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
}
fn morph_key(key: &H256, index: u8) -> Bytes {
@ -537,6 +532,7 @@ mod tests {
use super::super::traits::JournalDB;
use hashdb::*;
use log::init_log;
use kvdb::DatabaseConfig;
#[test]
fn insert_same_in_fork() {
@ -714,7 +710,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -742,7 +738,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -770,7 +766,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -808,7 +804,7 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
@ -818,14 +814,14 @@ mod tests {
};
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@ -840,7 +836,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
@ -869,7 +865,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
@ -918,7 +914,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
@ -949,7 +945,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
@ -989,7 +985,7 @@ mod tests {
let foo = b"foo".sha3();
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
@ -1010,7 +1006,7 @@ mod tests {
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
@ -1018,14 +1014,14 @@ mod tests {
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -1038,7 +1034,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, bar, baz) = {
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
@ -1056,7 +1052,7 @@ mod tests {
};
{
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));

11
util/src/journaldb/mod.rs
View File

@ -17,6 +17,7 @@
//! `JournalDB` interface and implementation.
use common::*;
use kvdb::DatabaseConfig;
/// Export the journaldb module.
pub mod traits;
@ -71,12 +72,12 @@ impl fmt::Display for Algorithm {
}
/// Create a new `JournalDB` trait object.
pub fn new(path: &str, algorithm: Algorithm, cache_size: Option<usize>) -> Box<JournalDB> {
pub fn new(path: &str, algorithm: Algorithm, config: DatabaseConfig) -> Box<JournalDB> {
match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, cache_size)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, cache_size)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, cache_size)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, cache_size)),
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, config)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, config)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, config)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, config)),
}
}

48
util/src/journaldb/overlayrecentdb.rs
View File

@ -98,18 +98,13 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB {
/// Create a new instance from file
pub fn new(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
Self::from_prefs(path, cache_size)
pub fn new(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
Self::from_prefs(path, config)
}
/// Create a new instance from file
pub fn from_prefs(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
pub fn from_prefs(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
let opts = config.prefix(DB_PREFIX_LEN);
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
@ -135,7 +130,7 @@ impl OverlayRecentDB {
pub fn new_temp() -> OverlayRecentDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), None)
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
}
#[cfg(test)]
@ -369,6 +364,7 @@ mod tests {
use hashdb::*;
use log::init_log;
use journaldb::JournalDB;
use kvdb::DatabaseConfig;
#[test]
fn insert_same_in_fork() {
@ -526,7 +522,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -554,7 +550,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -582,7 +578,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(0, &b"0".sha3(), None).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -620,7 +616,7 @@ mod tests {
let bar = H256::random();
let foo = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.emplace(bar.clone(), b"bar".to_vec());
@ -630,14 +626,14 @@ mod tests {
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
}
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
assert!(jdb.contains(&foo));
assert!(jdb.contains(&bar));
jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@ -652,7 +648,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
@ -681,7 +677,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
@ -730,7 +726,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
@ -761,7 +757,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 4
let foo = jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
@ -801,7 +797,7 @@ mod tests {
let foo = b"foo".sha3();
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
jdb.insert(b"foo");
jdb.commit(0, &b"0".sha3(), None).unwrap();
@ -822,7 +818,7 @@ mod tests {
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.remove(&foo);
jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
@ -830,14 +826,14 @@ mod tests {
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));
// incantation to reopen the db
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
@ -850,7 +846,7 @@ mod tests {
let mut dir = ::std::env::temp_dir();
dir.push(H32::random().hex());
let (foo, bar, baz) = {
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
// history is 1
let foo = jdb.insert(b"foo");
let bar = jdb.insert(b"bar");
@ -868,7 +864,7 @@ mod tests {
};
{
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
assert!(jdb.can_reconstruct_refs());
assert!(jdb.contains(&foo));

11
util/src/journaldb/refcounteddb.rs
View File

@ -46,13 +46,8 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB {
/// Create a new instance given a `backing` database.
pub fn new(path: &str, cache_size: Option<usize>) -> RefCountedDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
pub fn new(path: &str, config: DatabaseConfig) -> RefCountedDB {
let opts = config.prefix(DB_PREFIX_LEN);
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);
});
@ -82,7 +77,7 @@ impl RefCountedDB {
fn new_temp() -> RefCountedDB {
let mut dir = env::temp_dir();
dir.push(H32::random().hex());
Self::new(dir.to_str().unwrap(), None)
Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
}
}

78
util/src/kvdb.rs
View File

@ -20,8 +20,6 @@ use std::default::Default;
use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator,
IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction};
const DB_FILE_SIZE_BASE: u64 = 32 * 1024 * 1024;
const DB_FILE_SIZE_MULTIPLIER: i32 = 2;
const DB_BACKGROUND_FLUSHES: i32 = 2;
const DB_BACKGROUND_COMPACTIONS: i32 = 2;
@ -53,6 +51,36 @@ impl DBTransaction {
}
}
/// Compaction profile for the database settings
pub struct CompactionProfile {
/// L0-L1 target file size
pub initial_file_size: u64,
/// L2-LN target file size multiplier
pub file_size_multiplier: i32,
/// rate limiter for background flushes and compactions, bytes/sec, if any
pub write_rate_limit: Option<u64>,
}
impl CompactionProfile {
/// Default profile suitable for most storage
pub fn default() -> CompactionProfile {
CompactionProfile {
initial_file_size: 32 * 1024 * 1024,
file_size_multiplier: 2,
write_rate_limit: None,
}
}
/// Slow hdd compaction profile
pub fn hdd() -> CompactionProfile {
CompactionProfile {
initial_file_size: 192 * 1024 * 1024,
file_size_multiplier: 1,
write_rate_limit: Some(8 * 1024 * 1024),
}
}
}
/// Database configuration
pub struct DatabaseConfig {
/// Optional prefix size in bytes. Allows lookup by partial key.
@ -61,6 +89,8 @@ pub struct DatabaseConfig {
pub max_open_files: i32,
/// Cache-size
pub cache_size: Option<usize>,
/// Compaction profile
pub compaction: CompactionProfile,
}
impl DatabaseConfig {
@ -70,8 +100,21 @@ impl DatabaseConfig {
cache_size: Some(cache_size),
prefix_size: None,
max_open_files: -1,
compaction: CompactionProfile::default(),
}
}
/// Modify the compaction profile
pub fn compaction(mut self, profile: CompactionProfile) -> Self {
self.compaction = profile;
self
}
/// Modify the prefix of the db
pub fn prefix(mut self, prefix_size: usize) -> Self {
self.prefix_size = Some(prefix_size);
self
}
}
impl Default for DatabaseConfig {
@ -80,6 +123,7 @@ impl Default for DatabaseConfig {
cache_size: None,
prefix_size: None,
max_open_files: -1,
compaction: CompactionProfile::default(),
}
}
}
@ -111,13 +155,18 @@ impl Database {
/// Open database file. Creates if it does not exist.
pub fn open(config: &DatabaseConfig, path: &str) -> Result<Database, String> {
let mut opts = Options::new();
try!(opts.set_parsed_options("rate_limiter_bytes_per_sec=256000000"));
if let Some(rate_limit) = config.compaction.write_rate_limit {
try!(opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit)));
}
opts.set_max_open_files(config.max_open_files);
opts.create_if_missing(true);
opts.set_use_fsync(false);
// compaction settings
opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
opts.set_target_file_size_base(DB_FILE_SIZE_BASE);
opts.set_target_file_size_multiplier(DB_FILE_SIZE_MULTIPLIER);
opts.set_target_file_size_base(config.compaction.initial_file_size);
opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier);
opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES);
opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS);
if let Some(cache_size) = config.cache_size {
@ -150,7 +199,16 @@ impl Database {
opts.set_block_based_table_factory(&block_opts);
opts.set_prefix_extractor_fixed_size(size);
}
let db = try!(DB::open(&opts, path));
let db = match DB::open(&opts, path) {
Ok(db) => db,
Err(ref s) if s.starts_with("Corruption:") => {
info!("{}", s);
info!("Attempting DB repair for {}", path);
try!(DB::repair(&opts, path));
try!(DB::open(&opts, path))
},
Err(s) => { return Err(s); }
};
Ok(Database { db: db })
}
@ -244,10 +302,10 @@ mod tests {
let path = RandomTempPath::create_dir();
let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
assert!(smoke.is_empty());
test_db(&DatabaseConfig { prefix_size: None, max_open_files: 256, cache_size: None, });
test_db(&DatabaseConfig { prefix_size: Some(1), max_open_files: 256, cache_size: None, });
test_db(&DatabaseConfig { prefix_size: Some(8), max_open_files: 256, cache_size: None, });
test_db(&DatabaseConfig { prefix_size: Some(32), max_open_files: 256, cache_size: None, });
test_db(&DatabaseConfig::default());
test_db(&DatabaseConfig::default().prefix(12));
test_db(&DatabaseConfig::default().prefix(22));
test_db(&DatabaseConfig::default().prefix(8));
}
}