From f94f061f6aae7625b9ba46bd3fee4c975ce06d46 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 27 Jun 2016 22:12:23 +0300 Subject: [PATCH 1/3] disable wal when commiting transactions --- Cargo.lock | 4 ++-- util/src/kvdb.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e75e1bc76..bcfdd7325 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,7 +1103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541" +source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1112,7 +1112,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541" +source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index bc0acc2dd..1aa2fc1eb 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -18,7 +18,7 @@ use std::default::Default; use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, - IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; + IndexType, Options, WriteOptions, DBCompactionStyle, BlockBasedOptions, Direction}; const DB_BACKGROUND_FLUSHES: i32 = 2; const DB_BACKGROUND_COMPACTIONS: i32 = 2; @@ -169,6 +169,7 @@ impl Database { opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES); opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS); + if let Some(cache_size) = config.cache_size { // half goes to read cache opts.set_block_cache_size_mb(cache_size as u64 / 2); @@ -200,7 +201,7 @@ impl Database { opts.set_prefix_extractor_fixed_size(size); } let db = try!(DB::open(&opts, path)); - Ok(Database { db: db }) + Ok(Database { db: db, }) } /// Insert a key-value pair in the transaction. Any existing value value will be overwritten. @@ -215,7 +216,9 @@ impl Database { /// Commit transaction to database. pub fn write(&self, tr: DBTransaction) -> Result<(), String> { - self.db.write(tr.batch) + let mut write_opts = WriteOptions::new(); + write_opts.disable_wal(true); + self.db.write_opt(tr.batch, &write_opts) } /// Get value by key. From 3ac33ceda036d674be6987662092db2b79f53d4b Mon Sep 17 00:00:00 2001 From: NikVolf Date: Wed, 13 Jul 2016 18:43:24 +0200 Subject: [PATCH 2/3] block options for prefixed --- Cargo.lock | 4 ++-- util/src/kvdb.rs | 34 ++++++++++------------------------ 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d952b7b98..6cad02e5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1048,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" +source = "git+https://github.com/ethcore/rust-rocksdb#6472a9dce16c267a3acec2ee6fd01d1bf8de4913" dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)", @@ -1057,7 +1057,7 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6" +source = "git+https://github.com/ethcore/rust-rocksdb#6472a9dce16c267a3acec2ee6fd01d1bf8de4913" dependencies = [ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 8a19e48bf..8185a24a5 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -18,7 +18,7 @@ use std::default::Default; use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, - IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; + IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache}; const DB_BACKGROUND_FLUSHES: i32 = 2; const DB_BACKGROUND_COMPACTIONS: i32 = 2; @@ -169,35 +169,21 @@ impl Database { opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES); opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS); - if let Some(cache_size) = config.cache_size { - // half goes to read cache - opts.set_block_cache_size_mb(cache_size as u64 / 2); - // quarter goes to each of the two write buffers - opts.set_write_buffer_size(cache_size * 1024 * 256); - } - /* - opts.set_bytes_per_sync(8388608); - opts.set_disable_data_sync(false); - opts.set_block_cache_size_mb(1024); - opts.set_table_cache_num_shard_bits(6); - opts.set_max_write_buffer_number(32); - opts.set_write_buffer_size(536870912); - opts.set_target_file_size_base(1073741824); - opts.set_min_write_buffer_number_to_merge(4); - opts.set_level_zero_stop_writes_trigger(2000); - opts.set_level_zero_slowdown_writes_trigger(0); - opts.set_compaction_style(DBUniversalCompaction); - opts.set_max_background_compactions(4); - opts.set_max_background_flushes(4); - opts.set_filter_deletes(false); - opts.set_disable_auto_compactions(false); - */ if let Some(size) = config.prefix_size { let mut block_opts = BlockBasedOptions::new(); block_opts.set_index_type(IndexType::HashSearch); opts.set_block_based_table_factory(&block_opts); opts.set_prefix_extractor_fixed_size(size); + if let Some(cache_size) = config.cache_size { + block_opts.set_cache(Cache::new(cache_size * 1024 * 256)); + opts.set_write_buffer_size(cache_size * 1024 * 256); + } + } else if let Some(cache_size) = config.cache_size { + // half goes to read cache + opts.set_block_cache_size_mb(cache_size as u64 / 2); + // quarter goes to each of the two write buffers + opts.set_write_buffer_size(cache_size * 1024 * 256); } let db = match DB::open(&opts, path) { Ok(db) => db, From 9fd95e66943a711b7a141b82c404e9a9d9b86dac Mon Sep 17 00:00:00 2001 From: NikVolf Date: Wed, 13 Jul 2016 19:51:03 +0200 Subject: [PATCH 3/3] wiping also for non-prefixed db --- util/src/kvdb.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 39df15b35..f6d25ae51 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -181,8 +181,10 @@ impl Database { opts.set_write_buffer_size(cache_size * 1024 * 256); } } else if let Some(cache_size) = config.cache_size { + let mut block_opts = BlockBasedOptions::new(); // half goes to read cache - opts.set_block_cache_size_mb(cache_size as u64 / 2); + block_opts.set_cache(Cache::new(cache_size * 1024 * 256)); + opts.set_block_based_table_factory(&block_opts); // quarter goes to each of the two write buffers opts.set_write_buffer_size(cache_size * 1024 * 256); }