grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

configurable cache size

Browse Source
This commit is contained in:
Nikolay Volf 2016-06-20 13:42:04 +03:00
parent bf6308312e
commit 7e452ab2e0
15 changed files with 88 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ethcore/src/blockchain/blockchain.rs
View File

@ -253,12 +253,22 @@ impl BlockChain {
// open extras db
let mut extras_path = path.to_path_buf();
extras_path.push("extras");
let extras_db = Database::open_default(extras_path.to_str().unwrap()).unwrap();
let extras_db = match config.db_cache_size {
None => Database::open_default(extras_path.to_str().unwrap()).unwrap(),
Some(cache_size) => Database::open(
&DatabaseConfig::with_cache(cache_size/2),
extras_path.to_str().unwrap()).unwrap(),
};
// open blocks db
let mut blocks_path = path.to_path_buf();
blocks_path.push("blocks");
let blocks_db = Database::open_default(blocks_path.to_str().unwrap()).unwrap();
let blocks_db = match config.db_cache_size {
None => Database::open_default(blocks_path.to_str().unwrap()).unwrap(),
Some(cache_size) => Database::open(
&DatabaseConfig::with_cache(cache_size/2),
blocks_path.to_str().unwrap()).unwrap(),
};
let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()};
(0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new()));

3
ethcore/src/blockchain/config.rs
View File

@ -23,6 +23,8 @@ pub struct Config {
pub pref_cache_size: usize,
/// Maximum cache size in bytes.
pub max_cache_size: usize,
/// Backing db cache_size
pub db_cache_size: Option<usize>,
}
impl Default for Config {
@ -30,6 +32,7 @@ impl Default for Config {
Config {
pref_cache_size: 1 << 14,
max_cache_size: 1 << 20,
db_cache_size: None,
}
}
}

5
ethcore/src/client/client.rs
View File

@ -141,7 +141,10 @@ impl<V> Client<V> where V: Verifier {
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone())));
let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning);
let mut state_db = journaldb::new(
&append_path(&path, "state"),
config.pruning,
config.db_cache_size);
if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {
state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB");

2
ethcore/src/client/config.rs
View File

@ -35,4 +35,6 @@ pub struct ClientConfig {
pub pruning: journaldb::Algorithm,
/// The name of the client instance.
pub name: String,
/// State db cache-size if not default
pub db_cache_size: Option<usize>,
}

5
ethcore/src/trace/config.rs
View File

@ -48,6 +48,8 @@ pub struct Config {
pub enabled: Switch,
/// Traces blooms configuration.
pub blooms: BloomConfig,
/// Database cache-size if not default
pub db_cache_size: Option<usize>,
}
impl Default for Config {
@ -57,7 +59,8 @@ impl Default for Config {
blooms: BloomConfig {
levels: 3,
elements_per_index: 16,
}
},
db_cache_size: None,
}
}
}

9
ethcore/src/trace/db.rs
View File

@ -22,7 +22,7 @@ use std::sync::{RwLock, Arc};
use std::path::Path;
use bloomchain::{Number, Config as BloomConfig};
use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
use util::{H256, H264, Database, DBTransaction};
use util::{H256, H264, Database, DatabaseConfig, DBTransaction};
use header::BlockNumber;
use trace::{BlockTraces, LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest,
DatabaseExtras, Error};
@ -118,7 +118,12 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
pub fn new(config: Config, path: &Path, extras: Arc<T>) -> Result<Self, Error> {
let mut tracedb_path = path.to_path_buf();
tracedb_path.push("tracedb");
let tracesdb = Database::open_default(tracedb_path.to_str().unwrap()).unwrap();
let tracesdb = match config.db_cache_size {
None => Database::open_default(tracedb_path.to_str().unwrap()).unwrap(),
Some(db_cache) => Database::open(
&DatabaseConfig::with_cache(db_cache),
tracedb_path.to_str().unwrap()).unwrap(),
};
// check if in previously tracing was enabled
let old_tracing = match tracesdb.get(b"enabled").unwrap() {

2
parity/cli.rs
View File

@ -155,6 +155,7 @@ Footprint Options:
--cache MEGABYTES Set total amount of discretionary memory to use for
the entire system, overrides other cache and queue
options.
--db-cache-size MEGABYTE Database cache size.
Import/Export Options:
--from BLOCK Export from block BLOCK, which may be an index or
@ -292,6 +293,7 @@ pub struct Args {
pub flag_ipcdisable: bool,
pub flag_ipcpath: Option<String>,
pub flag_ipcapi: Option<String>,
pub flag_db_cache_size: Option<usize>,
}
pub fn print_version() {

9
parity/configuration.rs
View File

@ -187,7 +187,7 @@ impl Configuration {
let mut latest_era = None;
let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
for i in jdb_types.into_iter() {
let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i);
let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, None);
trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era());
match (latest_era, db.latest_era()) {
(Some(best), Some(this)) if best >= this => {}
@ -214,6 +214,8 @@ impl Configuration {
client_config.blockchain.max_cache_size = self.args.flag_cache_max_size;
}
}
// forced blockchain (blocks + extras) db cache size if provided
client_config.blockchain.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 2));
client_config.tracing.enabled = match self.args.flag_tracing.as_str() {
"auto" => Switch::Auto,
@ -221,6 +223,8 @@ impl Configuration {
"off" => Switch::Off,
_ => { die!("Invalid tracing method given!") }
};
// forced trace db cache size if provided
client_config.tracing.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
client_config.pruning = match self.args.flag_pruning.as_str() {
"archive" => journaldb::Algorithm::Archive,
@ -231,6 +235,9 @@ impl Configuration {
_ => { die!("Invalid pruning method given."); }
};
// forced state db cache size if provided
client_config.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
if self.args.flag_jitvm {
client_config.vm_type = VMType::jit().unwrap_or_else(|| die!("Parity built without jit vm."))
}

1
parity/migration.rs
View File

@ -163,6 +163,7 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -
let db_config = DatabaseConfig {
prefix_size: None,
max_open_files: 64,
cache_size: None,
};
// open old database

3
util/src/journaldb/archivedb.rs
View File

@ -43,11 +43,12 @@ const DB_VERSION : u32 = 0x103;
impl ArchiveDB {
/// Create a new instance from file
pub fn new(path: &str) -> ArchiveDB {
pub fn new(path: &str, cache_size: Option<usize>) -> ArchiveDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);

3
util/src/journaldb/earlymergedb.rs
View File

@ -73,11 +73,12 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl EarlyMergeDB {
/// Create a new instance from file
pub fn new(path: &str) -> EarlyMergeDB {
pub fn new(path: &str, cache_size: Option<usize>) -> EarlyMergeDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);

10
util/src/journaldb/mod.rs
View File

@ -71,12 +71,12 @@ impl fmt::Display for Algorithm {
}
/// Create a new `JournalDB` trait object.
pub fn new(path: &str, algorithm: Algorithm) -> Box<JournalDB> {
pub fn new(path: &str, algorithm: Algorithm, cache_size: Option<usize>) -> Box<JournalDB> {
match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path)),
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, cache_size)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, cache_size)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, cache_size)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, cache_size)),
}
}

7
util/src/journaldb/overlayrecentdb.rs
View File

@ -98,16 +98,17 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB {
/// Create a new instance from file
pub fn new(path: &str) -> OverlayRecentDB {
Self::from_prefs(path)
pub fn new(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
Self::from_prefs(path, cache_size)
}
/// Create a new instance from file
pub fn from_prefs(path: &str) -> OverlayRecentDB {
pub fn from_prefs(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);

3
util/src/journaldb/refcounteddb.rs
View File

@ -46,11 +46,12 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB {
/// Create a new instance given a `backing` database.
pub fn new(path: &str) -> RefCountedDB {
pub fn new(path: &str, cache_size: Option<usize>) -> RefCountedDB {
let opts = DatabaseConfig {
// this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256,
cache_size: cache_size,
};
let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e);

31
util/src/kvdb.rs
View File

@ -54,6 +54,29 @@ pub struct DatabaseConfig {
pub prefix_size: Option<usize>,
/// Max number of open files.
pub max_open_files: i32,
/// Cache-size
pub cache_size: Option<usize>,
}
impl DatabaseConfig {
/// Database with default settings and specified cache size
pub fn with_cache(cache_size: usize) -> DatabaseConfig {
DatabaseConfig {
cache_size: Some(cache_size),
prefix_size: None,
max_open_files: 256
}
}
}
impl Default for DatabaseConfig {
fn default() -> DatabaseConfig {
DatabaseConfig {
cache_size: None,
prefix_size: None,
max_open_files: 256
}
}
}
/// Database iterator
@ -77,7 +100,7 @@ pub struct Database {
impl Database {
/// Open database with default settings.
pub fn open_default(path: &str) -> Result<Database, String> {
Database::open(&DatabaseConfig { prefix_size: None, max_open_files: 256 }, path)
Database::open(&DatabaseConfig::default(), path)
}
/// Open database file. Creates if it does not exist.
@ -87,6 +110,12 @@ impl Database {
opts.create_if_missing(true);
opts.set_use_fsync(false);
opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
if let Some(cache_size) = config.cache_size {
// half goes to read cache
opts.set_block_cache_size_mb(cache_size as u64 / 2);
// quarter goes to each of the two write buffers
opts.set_write_buffer_size(cache_size * 1024 * 256);
}
/*
opts.set_bytes_per_sync(8388608);
opts.set_disable_data_sync(false);