grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade to RocksDB 5.8.8 and tune settings to reduce space amplification (#7348)

Browse Source 
* kvdb-rocksdb: update to RocksDB 5.8.8

* kvdb-rocksdb: tune RocksDB options

* Switch to level-style compaction
* Increase default block size (16K), and use bigger blocks for HDDs (64K)
* Increase default file size base (64MB SSDs, 256MB HDDs)
* Create a single block cache shared across all column families
* Tune compaction settings using RocksDB helper functions, taking into account
  memory budget spread across all columns
* Configure backgrounds jobs based on the number of CPUs
* Set some default recommended settings

* ethcore: remove unused config blockchain.db_cache_size

* parity: increase default value for db_cache_size

* kvdb-rocksdb: enable compression on all levels

* kvdb-rocksdb: set global db_write_bufer_size

* kvdb-rocksdb: reduce db_write_bufer_size to force earlier flushing

* kvdb-rocksdb: use master branch for rust-rocksdb dependency
This commit is contained in:
André Silva
2018-01-03 10:00:37 +00:00
committed by Tomasz Drwięga
parent 8405edab41
commit e114b0b28d
13 changed files with 93 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
parity/cache.rs
View File

@@ -17,8 +17,10 @@
use std::cmp::max;
const MIN_BC_CACHE_MB: u32 = 4;
const MIN_DB_CACHE_MB: u32 = 2;
const MIN_DB_CACHE_MB: u32 = 8;
const MIN_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 16;
const DEFAULT_DB_CACHE_SIZE: u32 = 128;
const DEFAULT_BC_CACHE_SIZE: u32 = 8;
const DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 40;
const DEFAULT_TRACE_CACHE_SIZE: u32 = 20;
const DEFAULT_STATE_CACHE_SIZE: u32 = 25;
@@ -41,7 +43,11 @@ pub struct CacheConfig {
impl Default for CacheConfig {
fn default() -> Self {
CacheConfig::new(32, 8, DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, DEFAULT_STATE_CACHE_SIZE)
CacheConfig::new(
DEFAULT_DB_CACHE_SIZE,
DEFAULT_BC_CACHE_SIZE,
DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB,
DEFAULT_STATE_CACHE_SIZE)
}
}
@@ -68,14 +74,9 @@ impl CacheConfig {
}
}
/// Size of db cache for blockchain.
pub fn db_blockchain_cache_size(&self) -> u32 {
max(MIN_DB_CACHE_MB, self.db / 4)
}
/// Size of db cache for state.
pub fn db_state_cache_size(&self) -> u32 {
max(MIN_DB_CACHE_MB, self.db * 3 / 4)
/// Size of db cache.
pub fn db_cache_size(&self) -> u32 {
max(MIN_DB_CACHE_MB, self.db)
}
/// Size of block queue size limit
@@ -122,13 +123,16 @@ mod tests {
fn test_cache_config_db_cache_sizes() {
let config = CacheConfig::new_with_total_cache_size(400);
assert_eq!(config.db, 280);
assert_eq!(config.db_blockchain_cache_size(), 70);
assert_eq!(config.db_state_cache_size(), 210);
assert_eq!(config.db_cache_size(), 280);
}
#[test]
fn test_cache_config_default() {
assert_eq!(CacheConfig::default(),
CacheConfig::new(32, 8, super::DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, super::DEFAULT_STATE_CACHE_SIZE));
CacheConfig::new(
super::DEFAULT_DB_CACHE_SIZE,
super::DEFAULT_BC_CACHE_SIZE,
super::DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB,
super::DEFAULT_STATE_CACHE_SIZE));
}
}

4
parity/cli/mod.rs
View File

@@ -779,7 +779,7 @@ usage! {
"--pruning-memory=[MB]",
"The ideal amount of memory in megabytes to use to store recent states. As many states as possible will be kept within this limit, and at least --pruning-history states will always be kept.",
ARG arg_cache_size_db: (u32) = 32u32, or |c: &Config| otry!(c.footprint).cache_size_db.clone(),
ARG arg_cache_size_db: (u32) = 128u32, or |c: &Config| otry!(c.footprint).cache_size_db.clone(),
"--cache-size-db=[MB]",
"Override database cache size.",
@@ -1797,7 +1797,7 @@ mod tests {
pruning_memory: None,
fast_and_loose: None,
cache_size: None,
cache_size_db: Some(128),
cache_size_db: Some(256),
cache_size_blocks: Some(16),
cache_size_queue: Some(100),
cache_size_state: Some(25),

2
parity/cli/tests/config.toml
View File

@@ -63,7 +63,7 @@ tx_queue_gas = "off"
tracing = "on"
pruning = "fast"
pruning_history = 64
cache_size_db = 128
cache_size_db = 256
cache_size_blocks = 16
cache_size_queue = 100
cache_size_state = 25

6
parity/helpers.rs
View File

@@ -227,10 +227,8 @@ pub fn to_client_config(
client_config.blockchain.max_cache_size = cache_config.blockchain() as usize * mb;
// in bytes
client_config.blockchain.pref_cache_size = cache_config.blockchain() as usize * 3 / 4 * mb;
// db blockchain cache size, in megabytes
client_config.blockchain.db_cache_size = Some(cache_config.db_blockchain_cache_size() as usize);
// db state cache size, in megabytes
client_config.db_cache_size = Some(cache_config.db_state_cache_size() as usize);
// db cache size, in megabytes
client_config.db_cache_size = Some(cache_config.db_cache_size() as usize);
// db queue cache size, in bytes
client_config.queue.max_mem_use = cache_config.queue() as usize * mb;
// in bytes

2
parity/migration.rs
View File

@@ -168,7 +168,7 @@ fn consolidate_database(
let config = default_migration_settings(compaction_profile);
let mut db_config = DatabaseConfig {
max_open_files: 64,
cache_sizes: Default::default(),
memory_budget: None,
compaction: config.compaction_profile,
columns: None,
wal: true,