From d874555310db1708ee2e9475b808ccb85b2cf61b Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 20 Jun 2016 22:45:24 +0300 Subject: [PATCH 1/5] make default 100mb file size --- Cargo.lock | 4 ++-- util/src/kvdb.rs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03ea96029..1bae73b40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,7 +1105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#9140e37ce0fdb748097f85653c01b0f7e3736ea9" +source = "git+https://github.com/ethcore/rust-rocksdb#8f6e43b2d8799578a378648bcb17c01fb431dbec" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#9140e37ce0fdb748097f85653c01b0f7e3736ea9" +source = "git+https://github.com/ethcore/rust-rocksdb#8f6e43b2d8799578a378648bcb17c01fb431dbec" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 7d581fdbc..c71934cc3 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -20,6 +20,9 @@ use std::default::Default; use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; +const DB_FILE_SIZE_BASE: u64 = 100 * 1024 * 1024; +const DB_FILE_SIZE_MULTIPLIER: usize = 10; + /// Write transaction. Batches a sequence of put/delete operations for efficiency. pub struct DBTransaction { batch: WriteBatch, @@ -110,6 +113,8 @@ impl Database { opts.create_if_missing(true); opts.set_use_fsync(false); opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); + opts.set_target_file_size_base(DB_FILE_SIZE_BASE); + opts.set_target_file_size_multiplier(DB_FILE_SIZE_MULTIPLIER); if let Some(cache_size) = config.cache_size { // half goes to read cache opts.set_block_cache_size_mb(cache_size as u64 / 2); From f4f842f48e51b9f4bb534d613cce843bc6e54020 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 20 Jun 2016 22:52:31 +0300 Subject: [PATCH 2/5] update again --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bae73b40..7f901eb45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,7 +1105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#8f6e43b2d8799578a378648bcb17c01fb431dbec" +source = "git+https://github.com/ethcore/rust-rocksdb#e0e6c099d8cd156fe446009fce241d57b00cd8f4" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#8f6e43b2d8799578a378648bcb17c01fb431dbec" +source = "git+https://github.com/ethcore/rust-rocksdb#e0e6c099d8cd156fe446009fce241d57b00cd8f4" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", From 1f599ac6f9814c7fd65920ac89a3cddd7345a695 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 20 Jun 2016 23:01:09 +0300 Subject: [PATCH 3/5] fix type --- util/src/kvdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index c71934cc3..1633a6f88 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -21,7 +21,7 @@ use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; const DB_FILE_SIZE_BASE: u64 = 100 * 1024 * 1024; -const DB_FILE_SIZE_MULTIPLIER: usize = 10; +const DB_FILE_SIZE_MULTIPLIER: i32 = 10; /// Write transaction. Batches a sequence of put/delete operations for efficiency. pub struct DBTransaction { From d42f8eac1e980961a94924970df0044b6258fb26 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 21 Jun 2016 12:07:07 +0300 Subject: [PATCH 4/5] little less extreme file sizes --- util/src/kvdb.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 1633a6f88..0911b3471 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -20,8 +20,8 @@ use std::default::Default; use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; -const DB_FILE_SIZE_BASE: u64 = 100 * 1024 * 1024; -const DB_FILE_SIZE_MULTIPLIER: i32 = 10; +const DB_FILE_SIZE_BASE: u64 = 10 * 1024 * 1024; +const DB_FILE_SIZE_MULTIPLIER: i32 = 5; /// Write transaction. Batches a sequence of put/delete operations for efficiency. pub struct DBTransaction { @@ -67,7 +67,7 @@ impl DatabaseConfig { DatabaseConfig { cache_size: Some(cache_size), prefix_size: None, - max_open_files: 256 + max_open_files: -1, } } } @@ -77,7 +77,7 @@ impl Default for DatabaseConfig { DatabaseConfig { cache_size: None, prefix_size: None, - max_open_files: 256 + max_open_files: -1, } } } From 9132895d0e0cbd3180ed68a34b94ba5c635449d7 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 21 Jun 2016 17:20:47 +0300 Subject: [PATCH 5/5] limit flush rate --- Cargo.lock | 4 ++-- util/src/kvdb.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f901eb45..1902f076b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,7 +1105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#e0e6c099d8cd156fe446009fce241d57b00cd8f4" +source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#e0e6c099d8cd156fe446009fce241d57b00cd8f4" +source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 0911b3471..66a4a02ee 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -109,6 +109,7 @@ impl Database { /// Open database file. Creates if it does not exist. pub fn open(config: &DatabaseConfig, path: &str) -> Result { let mut opts = Options::new(); + try!(opts.set_parsed_options("rate_limiter_bytes_per_sec=256000000")); opts.set_max_open_files(config.max_open_files); opts.create_if_missing(true); opts.set_use_fsync(false);