configurable cache size

This commit is contained in:
Nikolay Volf 2016-06-20 13:42:04 +03:00
parent bf6308312e
commit 7e452ab2e0
15 changed files with 88 additions and 19 deletions

View File

@ -253,12 +253,22 @@ impl BlockChain {
// open extras db // open extras db
let mut extras_path = path.to_path_buf(); let mut extras_path = path.to_path_buf();
extras_path.push("extras"); extras_path.push("extras");
let extras_db = Database::open_default(extras_path.to_str().unwrap()).unwrap(); let extras_db = match config.db_cache_size {
None => Database::open_default(extras_path.to_str().unwrap()).unwrap(),
Some(cache_size) => Database::open(
&DatabaseConfig::with_cache(cache_size/2),
extras_path.to_str().unwrap()).unwrap(),
};
// open blocks db // open blocks db
let mut blocks_path = path.to_path_buf(); let mut blocks_path = path.to_path_buf();
blocks_path.push("blocks"); blocks_path.push("blocks");
let blocks_db = Database::open_default(blocks_path.to_str().unwrap()).unwrap(); let blocks_db = match config.db_cache_size {
None => Database::open_default(blocks_path.to_str().unwrap()).unwrap(),
Some(cache_size) => Database::open(
&DatabaseConfig::with_cache(cache_size/2),
blocks_path.to_str().unwrap()).unwrap(),
};
let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()}; let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()};
(0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new()));

View File

@ -23,6 +23,8 @@ pub struct Config {
pub pref_cache_size: usize, pub pref_cache_size: usize,
/// Maximum cache size in bytes. /// Maximum cache size in bytes.
pub max_cache_size: usize, pub max_cache_size: usize,
/// Backing db cache_size
pub db_cache_size: Option<usize>,
} }
impl Default for Config { impl Default for Config {
@ -30,6 +32,7 @@ impl Default for Config {
Config { Config {
pref_cache_size: 1 << 14, pref_cache_size: 1 << 14,
max_cache_size: 1 << 20, max_cache_size: 1 << 20,
db_cache_size: None,
} }
} }
} }

View File

@ -141,7 +141,10 @@ impl<V> Client<V> where V: Verifier {
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path)); let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone()))); let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone())));
let mut state_db = journaldb::new(&append_path(&path, "state"), config.pruning); let mut state_db = journaldb::new(
&append_path(&path, "state"),
config.pruning,
config.db_cache_size);
if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) { if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {
state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); state_db.commit(0, &spec.genesis_header().hash(), None).expect("Error commiting genesis state to state DB");

View File

@ -35,4 +35,6 @@ pub struct ClientConfig {
pub pruning: journaldb::Algorithm, pub pruning: journaldb::Algorithm,
/// The name of the client instance. /// The name of the client instance.
pub name: String, pub name: String,
/// State db cache-size if not default
pub db_cache_size: Option<usize>,
} }

View File

@ -48,6 +48,8 @@ pub struct Config {
pub enabled: Switch, pub enabled: Switch,
/// Traces blooms configuration. /// Traces blooms configuration.
pub blooms: BloomConfig, pub blooms: BloomConfig,
/// Database cache-size if not default
pub db_cache_size: Option<usize>,
} }
impl Default for Config { impl Default for Config {
@ -57,7 +59,8 @@ impl Default for Config {
blooms: BloomConfig { blooms: BloomConfig {
levels: 3, levels: 3,
elements_per_index: 16, elements_per_index: 16,
} },
db_cache_size: None,
} }
} }
} }

View File

@ -22,7 +22,7 @@ use std::sync::{RwLock, Arc};
use std::path::Path; use std::path::Path;
use bloomchain::{Number, Config as BloomConfig}; use bloomchain::{Number, Config as BloomConfig};
use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup}; use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
use util::{H256, H264, Database, DBTransaction}; use util::{H256, H264, Database, DatabaseConfig, DBTransaction};
use header::BlockNumber; use header::BlockNumber;
use trace::{BlockTraces, LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest, use trace::{BlockTraces, LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest,
DatabaseExtras, Error}; DatabaseExtras, Error};
@ -118,7 +118,12 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
pub fn new(config: Config, path: &Path, extras: Arc<T>) -> Result<Self, Error> { pub fn new(config: Config, path: &Path, extras: Arc<T>) -> Result<Self, Error> {
let mut tracedb_path = path.to_path_buf(); let mut tracedb_path = path.to_path_buf();
tracedb_path.push("tracedb"); tracedb_path.push("tracedb");
let tracesdb = Database::open_default(tracedb_path.to_str().unwrap()).unwrap(); let tracesdb = match config.db_cache_size {
None => Database::open_default(tracedb_path.to_str().unwrap()).unwrap(),
Some(db_cache) => Database::open(
&DatabaseConfig::with_cache(db_cache),
tracedb_path.to_str().unwrap()).unwrap(),
};
// check if in previously tracing was enabled // check if in previously tracing was enabled
let old_tracing = match tracesdb.get(b"enabled").unwrap() { let old_tracing = match tracesdb.get(b"enabled").unwrap() {

View File

@ -155,6 +155,7 @@ Footprint Options:
--cache MEGABYTES Set total amount of discretionary memory to use for --cache MEGABYTES Set total amount of discretionary memory to use for
the entire system, overrides other cache and queue the entire system, overrides other cache and queue
options. options.
--db-cache-size MEGABYTE Database cache size.
Import/Export Options: Import/Export Options:
--from BLOCK Export from block BLOCK, which may be an index or --from BLOCK Export from block BLOCK, which may be an index or
@ -292,6 +293,7 @@ pub struct Args {
pub flag_ipcdisable: bool, pub flag_ipcdisable: bool,
pub flag_ipcpath: Option<String>, pub flag_ipcpath: Option<String>,
pub flag_ipcapi: Option<String>, pub flag_ipcapi: Option<String>,
pub flag_db_cache_size: Option<usize>,
} }
pub fn print_version() { pub fn print_version() {

View File

@ -187,7 +187,7 @@ impl Configuration {
let mut latest_era = None; let mut latest_era = None;
let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted]; let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
for i in jdb_types.into_iter() { for i in jdb_types.into_iter() {
let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i); let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, None);
trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era()); trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era());
match (latest_era, db.latest_era()) { match (latest_era, db.latest_era()) {
(Some(best), Some(this)) if best >= this => {} (Some(best), Some(this)) if best >= this => {}
@ -214,6 +214,8 @@ impl Configuration {
client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; client_config.blockchain.max_cache_size = self.args.flag_cache_max_size;
} }
} }
// forced blockchain (blocks + extras) db cache size if provided
client_config.blockchain.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 2));
client_config.tracing.enabled = match self.args.flag_tracing.as_str() { client_config.tracing.enabled = match self.args.flag_tracing.as_str() {
"auto" => Switch::Auto, "auto" => Switch::Auto,
@ -221,6 +223,8 @@ impl Configuration {
"off" => Switch::Off, "off" => Switch::Off,
_ => { die!("Invalid tracing method given!") } _ => { die!("Invalid tracing method given!") }
}; };
// forced trace db cache size if provided
client_config.tracing.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
client_config.pruning = match self.args.flag_pruning.as_str() { client_config.pruning = match self.args.flag_pruning.as_str() {
"archive" => journaldb::Algorithm::Archive, "archive" => journaldb::Algorithm::Archive,
@ -231,6 +235,9 @@ impl Configuration {
_ => { die!("Invalid pruning method given."); } _ => { die!("Invalid pruning method given."); }
}; };
// forced state db cache size if provided
client_config.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
if self.args.flag_jitvm { if self.args.flag_jitvm {
client_config.vm_type = VMType::jit().unwrap_or_else(|| die!("Parity built without jit vm.")) client_config.vm_type = VMType::jit().unwrap_or_else(|| die!("Parity built without jit vm."))
} }

View File

@ -163,6 +163,7 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -
let db_config = DatabaseConfig { let db_config = DatabaseConfig {
prefix_size: None, prefix_size: None,
max_open_files: 64, max_open_files: 64,
cache_size: None,
}; };
// open old database // open old database

View File

@ -43,11 +43,12 @@ const DB_VERSION : u32 = 0x103;
impl ArchiveDB { impl ArchiveDB {
/// Create a new instance from file /// Create a new instance from file
pub fn new(path: &str) -> ArchiveDB { pub fn new(path: &str, cache_size: Option<usize>) -> ArchiveDB {
let opts = DatabaseConfig { let opts = DatabaseConfig {
// this must match account_db prefix // this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN), prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256, max_open_files: 256,
cache_size: cache_size,
}; };
let backing = Database::open(&opts, path).unwrap_or_else(|e| { let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e); panic!("Error opening state db: {}", e);

View File

@ -73,11 +73,12 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl EarlyMergeDB { impl EarlyMergeDB {
/// Create a new instance from file /// Create a new instance from file
pub fn new(path: &str) -> EarlyMergeDB { pub fn new(path: &str, cache_size: Option<usize>) -> EarlyMergeDB {
let opts = DatabaseConfig { let opts = DatabaseConfig {
// this must match account_db prefix // this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN), prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256, max_open_files: 256,
cache_size: cache_size,
}; };
let backing = Database::open(&opts, path).unwrap_or_else(|e| { let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e); panic!("Error opening state db: {}", e);

View File

@ -71,12 +71,12 @@ impl fmt::Display for Algorithm {
} }
/// Create a new `JournalDB` trait object. /// Create a new `JournalDB` trait object.
pub fn new(path: &str, algorithm: Algorithm) -> Box<JournalDB> { pub fn new(path: &str, algorithm: Algorithm, cache_size: Option<usize>) -> Box<JournalDB> {
match algorithm { match algorithm {
Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path)), Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, cache_size)),
Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path)), Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, cache_size)),
Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path)), Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, cache_size)),
Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path)), Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, cache_size)),
} }
} }

View File

@ -98,16 +98,17 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl OverlayRecentDB { impl OverlayRecentDB {
/// Create a new instance from file /// Create a new instance from file
pub fn new(path: &str) -> OverlayRecentDB { pub fn new(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
Self::from_prefs(path) Self::from_prefs(path, cache_size)
} }
/// Create a new instance from file /// Create a new instance from file
pub fn from_prefs(path: &str) -> OverlayRecentDB { pub fn from_prefs(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
let opts = DatabaseConfig { let opts = DatabaseConfig {
// this must match account_db prefix // this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN), prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256, max_open_files: 256,
cache_size: cache_size,
}; };
let backing = Database::open(&opts, path).unwrap_or_else(|e| { let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e); panic!("Error opening state db: {}", e);

View File

@ -46,11 +46,12 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
impl RefCountedDB { impl RefCountedDB {
/// Create a new instance given a `backing` database. /// Create a new instance given a `backing` database.
pub fn new(path: &str) -> RefCountedDB { pub fn new(path: &str, cache_size: Option<usize>) -> RefCountedDB {
let opts = DatabaseConfig { let opts = DatabaseConfig {
// this must match account_db prefix // this must match account_db prefix
prefix_size: Some(DB_PREFIX_LEN), prefix_size: Some(DB_PREFIX_LEN),
max_open_files: 256, max_open_files: 256,
cache_size: cache_size,
}; };
let backing = Database::open(&opts, path).unwrap_or_else(|e| { let backing = Database::open(&opts, path).unwrap_or_else(|e| {
panic!("Error opening state db: {}", e); panic!("Error opening state db: {}", e);

View File

@ -54,6 +54,29 @@ pub struct DatabaseConfig {
pub prefix_size: Option<usize>, pub prefix_size: Option<usize>,
/// Max number of open files. /// Max number of open files.
pub max_open_files: i32, pub max_open_files: i32,
/// Cache-size
pub cache_size: Option<usize>,
}
impl DatabaseConfig {
/// Database with default settings and specified cache size
pub fn with_cache(cache_size: usize) -> DatabaseConfig {
DatabaseConfig {
cache_size: Some(cache_size),
prefix_size: None,
max_open_files: 256
}
}
}
impl Default for DatabaseConfig {
fn default() -> DatabaseConfig {
DatabaseConfig {
cache_size: None,
prefix_size: None,
max_open_files: 256
}
}
} }
/// Database iterator /// Database iterator
@ -77,7 +100,7 @@ pub struct Database {
impl Database { impl Database {
/// Open database with default settings. /// Open database with default settings.
pub fn open_default(path: &str) -> Result<Database, String> { pub fn open_default(path: &str) -> Result<Database, String> {
Database::open(&DatabaseConfig { prefix_size: None, max_open_files: 256 }, path) Database::open(&DatabaseConfig::default(), path)
} }
/// Open database file. Creates if it does not exist. /// Open database file. Creates if it does not exist.
@ -87,6 +110,12 @@ impl Database {
opts.create_if_missing(true); opts.create_if_missing(true);
opts.set_use_fsync(false); opts.set_use_fsync(false);
opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
if let Some(cache_size) = config.cache_size {
// half goes to read cache
opts.set_block_cache_size_mb(cache_size as u64 / 2);
// quarter goes to each of the two write buffers
opts.set_write_buffer_size(cache_size * 1024 * 256);
}
/* /*
opts.set_bytes_per_sync(8388608); opts.set_bytes_per_sync(8388608);
opts.set_disable_data_sync(false); opts.set_disable_data_sync(false);