diff --git a/Cargo.lock b/Cargo.lock index b2b08d29c..531ec4ce8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,9 @@ dependencies = [ name = "cc" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "cfg-if" @@ -598,7 +601,7 @@ dependencies = [ "migration 0.1.0", "native-contracts 0.1.0", "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-machine 0.1.0", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "patricia_trie 0.1.0", @@ -1135,7 +1138,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1270,7 +1273,7 @@ dependencies = [ "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1528,6 +1531,7 @@ dependencies = [ "ethcore-devtools 1.8.0", "kvdb 0.1.0", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.0", @@ -1984,7 +1988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num_cpus" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2105,7 +2109,7 @@ dependencies = [ "migration 0.1.0", "node-filter 1.8.0", "node-health 0.1.0", - "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "number_prefix 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", "parity-dapps 1.8.0", @@ -2718,7 +2722,7 @@ dependencies = [ "futures 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2805,7 +2809,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/paritytech/rust-rocksdb#4364caec4dd5da1a1d78c39276774ee65bf55c7d" +source = "git+https://github.com/paritytech/rust-rocksdb#166e14ed63cbd2e44b51267b8b98e4b89b0f236f" dependencies = [ "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2815,10 +2819,11 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/paritytech/rust-rocksdb#4364caec4dd5da1a1d78c39276774ee65bf55c7d" +source = "git+https://github.com/paritytech/rust-rocksdb#166e14ed63cbd2e44b51267b8b98e4b89b0f236f" dependencies = [ - "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)", ] [[package]] @@ -3115,6 +3120,15 @@ dependencies = [ "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "snappy-sys" +version = "0.1.0" +source = "git+https://github.com/paritytech/rust-snappy#858eac97192ea25d18d3f3626a8cc13ca0b175bb" +dependencies = [ + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "spmc" version = "0.2.2" @@ -3828,7 +3842,7 @@ dependencies = [ "checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01" "checksum num-rational 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "288629c76fac4b33556f4b7ab57ba21ae202da65ba8b77466e6d598e31990790" "checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0" -"checksum num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aec53c34f2d0247c5ca5d32cca1478762f301740468ee9ee6dcb7a0dd7a0c584" +"checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d" "checksum number_prefix 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "59a14be9c211cb9c602bad35ac99f41e9a84b44d71b8cbd3040e3bd02a214902" "checksum odds 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "c3df9b730298cea3a1c3faa90b7e2f9df3a9c400d0936d6015e6165734eefcba" "checksum ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c" @@ -3913,6 +3927,7 @@ dependencies = [ "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" "checksum smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8fcd03faf178110ab0334d74ca9631d77f94c8c11cc77fcb59538abf0025695d" +"checksum snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)" = "" "checksum spmc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cd1f11d1fb5fd41834e55ce0b85a186efbf2f2afd9fdb09e2c8d72f9bff1ad1a" "checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" diff --git a/ethcore/light/src/client/service.rs b/ethcore/light/src/client/service.rs index 2ab3e530a..11d6a8e77 100644 --- a/ethcore/light/src/client/service.rs +++ b/ethcore/light/src/client/service.rs @@ -63,11 +63,7 @@ impl Service { // initialize database. let mut db_config = DatabaseConfig::with_columns(db::NUM_COLUMNS); - // give all rocksdb cache to the header chain column. - if let Some(size) = config.db_cache_size { - db_config.set_cache(db::COL_LIGHT_CHAIN, size); - } - + db_config.memory_budget = config.db_cache_size; db_config.compaction = config.db_compaction; db_config.wal = config.db_wal; diff --git a/ethcore/src/blockchain/config.rs b/ethcore/src/blockchain/config.rs index 4be606b33..312289b06 100644 --- a/ethcore/src/blockchain/config.rs +++ b/ethcore/src/blockchain/config.rs @@ -23,8 +23,6 @@ pub struct Config { pub pref_cache_size: usize, /// Maximum cache size in bytes. pub max_cache_size: usize, - /// Backing db cache_size - pub db_cache_size: Option, } impl Default for Config { @@ -32,8 +30,6 @@ impl Default for Config { Config { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, - db_cache_size: None, } } } - diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 14efb3ec4..e5385ffee 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -141,7 +141,7 @@ pub struct ClientConfig { pub pruning: journaldb::Algorithm, /// The name of the client instance. pub name: String, - /// RocksDB state column cache-size if not default + /// RocksDB column cache-size if not default pub db_cache_size: Option, /// State db compaction profile pub db_compaction: DatabaseCompactionProfile, diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 0ceabce84..56cc534f1 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -83,12 +83,7 @@ impl ClientService { let mut db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); - // give all rocksdb cache to state column; everything else has its - // own caches. - if let Some(size) = config.db_cache_size { - db_config.set_cache(::db::COL_STATE, size); - } - + db_config.memory_budget = config.db_cache_size; db_config.compaction = config.db_compaction.compaction_profile(client_path); db_config.wal = config.db_wal; diff --git a/parity/cache.rs b/parity/cache.rs index 8784ffa3d..0bf0717a3 100644 --- a/parity/cache.rs +++ b/parity/cache.rs @@ -17,8 +17,10 @@ use std::cmp::max; const MIN_BC_CACHE_MB: u32 = 4; -const MIN_DB_CACHE_MB: u32 = 2; +const MIN_DB_CACHE_MB: u32 = 8; const MIN_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 16; +const DEFAULT_DB_CACHE_SIZE: u32 = 128; +const DEFAULT_BC_CACHE_SIZE: u32 = 8; const DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 40; const DEFAULT_TRACE_CACHE_SIZE: u32 = 20; const DEFAULT_STATE_CACHE_SIZE: u32 = 25; @@ -41,7 +43,11 @@ pub struct CacheConfig { impl Default for CacheConfig { fn default() -> Self { - CacheConfig::new(32, 8, DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, DEFAULT_STATE_CACHE_SIZE) + CacheConfig::new( + DEFAULT_DB_CACHE_SIZE, + DEFAULT_BC_CACHE_SIZE, + DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, + DEFAULT_STATE_CACHE_SIZE) } } @@ -68,14 +74,9 @@ impl CacheConfig { } } - /// Size of db cache for blockchain. - pub fn db_blockchain_cache_size(&self) -> u32 { - max(MIN_DB_CACHE_MB, self.db / 4) - } - - /// Size of db cache for state. - pub fn db_state_cache_size(&self) -> u32 { - max(MIN_DB_CACHE_MB, self.db * 3 / 4) + /// Size of db cache. + pub fn db_cache_size(&self) -> u32 { + max(MIN_DB_CACHE_MB, self.db) } /// Size of block queue size limit @@ -122,13 +123,16 @@ mod tests { fn test_cache_config_db_cache_sizes() { let config = CacheConfig::new_with_total_cache_size(400); assert_eq!(config.db, 280); - assert_eq!(config.db_blockchain_cache_size(), 70); - assert_eq!(config.db_state_cache_size(), 210); + assert_eq!(config.db_cache_size(), 280); } #[test] fn test_cache_config_default() { assert_eq!(CacheConfig::default(), - CacheConfig::new(32, 8, super::DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, super::DEFAULT_STATE_CACHE_SIZE)); + CacheConfig::new( + super::DEFAULT_DB_CACHE_SIZE, + super::DEFAULT_BC_CACHE_SIZE, + super::DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, + super::DEFAULT_STATE_CACHE_SIZE)); } } diff --git a/parity/cli/mod.rs b/parity/cli/mod.rs index 3f5099b61..f9f7e28d0 100644 --- a/parity/cli/mod.rs +++ b/parity/cli/mod.rs @@ -767,7 +767,7 @@ usage! { "--pruning-memory=[MB]", "The ideal amount of memory in megabytes to use to store recent states. As many states as possible will be kept within this limit, and at least --pruning-history states will always be kept.", - ARG arg_cache_size_db: (u32) = 32u32, or |c: &Config| otry!(c.footprint).cache_size_db.clone(), + ARG arg_cache_size_db: (u32) = 128u32, or |c: &Config| otry!(c.footprint).cache_size_db.clone(), "--cache-size-db=[MB]", "Override database cache size.", @@ -1776,7 +1776,7 @@ mod tests { pruning_memory: None, fast_and_loose: None, cache_size: None, - cache_size_db: Some(128), + cache_size_db: Some(256), cache_size_blocks: Some(16), cache_size_queue: Some(100), cache_size_state: Some(25), diff --git a/parity/cli/tests/config.toml b/parity/cli/tests/config.toml index 08da653de..abdf3e0c7 100644 --- a/parity/cli/tests/config.toml +++ b/parity/cli/tests/config.toml @@ -63,7 +63,7 @@ tx_queue_gas = "off" tracing = "on" pruning = "fast" pruning_history = 64 -cache_size_db = 128 +cache_size_db = 256 cache_size_blocks = 16 cache_size_queue = 100 cache_size_state = 25 diff --git a/parity/helpers.rs b/parity/helpers.rs index 1319f45d3..bbb23abc5 100644 --- a/parity/helpers.rs +++ b/parity/helpers.rs @@ -239,10 +239,8 @@ pub fn to_client_config( client_config.blockchain.max_cache_size = cache_config.blockchain() as usize * mb; // in bytes client_config.blockchain.pref_cache_size = cache_config.blockchain() as usize * 3 / 4 * mb; - // db blockchain cache size, in megabytes - client_config.blockchain.db_cache_size = Some(cache_config.db_blockchain_cache_size() as usize); - // db state cache size, in megabytes - client_config.db_cache_size = Some(cache_config.db_state_cache_size() as usize); + // db cache size, in megabytes + client_config.db_cache_size = Some(cache_config.db_cache_size() as usize); // db queue cache size, in bytes client_config.queue.max_mem_use = cache_config.queue() as usize * mb; // in bytes diff --git a/parity/migration.rs b/parity/migration.rs index 1e8cf2693..f5f7a1981 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -167,7 +167,7 @@ fn consolidate_database( let config = default_migration_settings(compaction_profile); let mut db_config = DatabaseConfig { max_open_files: 64, - cache_sizes: Default::default(), + memory_budget: None, compaction: config.compaction_profile, columns: None, wal: true, diff --git a/util/kvdb-rocksdb/Cargo.toml b/util/kvdb-rocksdb/Cargo.toml index 0f1809d14..a3361af98 100644 --- a/util/kvdb-rocksdb/Cargo.toml +++ b/util/kvdb-rocksdb/Cargo.toml @@ -4,12 +4,13 @@ version = "0.1.0" authors = ["Parity Technologies "] [dependencies] -log = "0.3" elastic-array = "0.9" ethcore-bigint = { path = "../bigint" } ethcore-devtools = { path = "../../devtools" } +kvdb = { path = "../kvdb" } +log = "0.3" +num_cpus = "1.0" parking_lot = "0.4" regex = "0.2" rlp = { path = "../rlp" } rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" } -kvdb = { path = "../kvdb" } diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index f17be8fb3..901ed9f80 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -18,6 +18,7 @@ extern crate log; extern crate elastic_array; +extern crate num_cpus; extern crate parking_lot; extern crate regex; extern crate rocksdb; @@ -27,6 +28,7 @@ extern crate ethcore_devtools as devtools; extern crate kvdb; extern crate rlp; +use std::cmp; use std::collections::HashMap; use std::marker::PhantomData; use std::path::{PathBuf, Path}; @@ -35,7 +37,7 @@ use std::{mem, fs, io}; use parking_lot::{Mutex, MutexGuard, RwLock}; use rocksdb::{ DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator, - Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions + Options, BlockBasedOptions, Direction, Cache, Column, ReadOptions }; use elastic_array::ElasticArray32; @@ -49,9 +51,7 @@ use std::process::Command; #[cfg(target_os = "linux")] use std::fs::File; -const DB_BACKGROUND_FLUSHES: i32 = 2; -const DB_BACKGROUND_COMPACTIONS: i32 = 2; -const DB_WRITE_BUFFER_SIZE: usize = 2048 * 1000; +const DB_DEFAULT_MEMORY_BUDGET_MB: usize = 128; enum KeyState { Insert(DBValue), @@ -64,8 +64,8 @@ enum KeyState { pub struct CompactionProfile { /// L0-L1 target file size pub initial_file_size: u64, - /// L2-LN target file size multiplier - pub file_size_multiplier: i32, + /// block size + pub block_size: usize, /// rate limiter for background flushes and compactions, bytes/sec, if any pub write_rate_limit: Option, } @@ -135,8 +135,8 @@ impl CompactionProfile { /// Default profile suitable for SSD storage pub fn ssd() -> CompactionProfile { CompactionProfile { - initial_file_size: 32 * 1024 * 1024, - file_size_multiplier: 2, + initial_file_size: 64 * 1024 * 1024, + block_size: 16 * 1024, write_rate_limit: None, } } @@ -144,9 +144,9 @@ impl CompactionProfile { /// Slow HDD compaction profile pub fn hdd() -> CompactionProfile { CompactionProfile { - initial_file_size: 192 * 1024 * 1024, - file_size_multiplier: 1, - write_rate_limit: Some(8 * 1024 * 1024), + initial_file_size: 256 * 1024 * 1024, + block_size: 64 * 1024, + write_rate_limit: Some(16 * 1024 * 1024), } } } @@ -156,8 +156,8 @@ impl CompactionProfile { pub struct DatabaseConfig { /// Max number of open files. pub max_open_files: i32, - /// Cache sizes (in MiB) for specific columns. - pub cache_sizes: HashMap, usize>, + /// Memory budget (in MiB) used for setting block cache size, write buffer size. + pub memory_budget: Option, /// Compaction profile pub compaction: CompactionProfile, /// Set number of columns @@ -175,17 +175,20 @@ impl DatabaseConfig { config } - /// Set the column cache size in MiB. - pub fn set_cache(&mut self, col: Option, size: usize) { - self.cache_sizes.insert(col, size); + pub fn memory_budget(&self) -> usize { + self.memory_budget.unwrap_or(DB_DEFAULT_MEMORY_BUDGET_MB) * 1024 * 1024 + } + + pub fn memory_budget_per_col(&self) -> usize { + self.memory_budget() / self.columns.unwrap_or(1) as usize } } impl Default for DatabaseConfig { fn default() -> DatabaseConfig { DatabaseConfig { - cache_sizes: HashMap::new(), max_open_files: 512, + memory_budget: None, compaction: CompactionProfile::default(), columns: None, wal: true, @@ -216,27 +219,24 @@ struct DBAndColumns { } // get column family configuration from database config. -fn col_config(col: u32, config: &DatabaseConfig) -> Options { - // default cache size for columns not specified. - const DEFAULT_CACHE: usize = 2; - +fn col_config(config: &DatabaseConfig, block_opts: &BlockBasedOptions) -> Result { let mut opts = Options::new(); - opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); + + opts.set_parsed_options("level_compaction_dynamic_level_bytes=true")?; + + opts.set_block_based_table_factory(block_opts); + + opts.set_parsed_options( + &format!("block_based_table_factory={{{};{}}}", + "cache_index_and_filter_blocks=true", + "pin_l0_filter_and_index_blocks_in_cache=true"))?; + + opts.optimize_level_style_compaction(config.memory_budget_per_col() as i32); opts.set_target_file_size_base(config.compaction.initial_file_size); - opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier); - opts.set_db_write_buffer_size(DB_WRITE_BUFFER_SIZE); - let col_opt = config.columns.map(|_| col); + opts.set_parsed_options("compression_per_level=")?; - { - let cache_size = config.cache_sizes.get(&col_opt).cloned().unwrap_or(DEFAULT_CACHE); - let mut block_opts = BlockBasedOptions::new(); - // all goes to read cache. - block_opts.set_cache(Cache::new(cache_size * 1024 * 1024)); - opts.set_block_based_table_factory(&block_opts); - } - - opts + Ok(opts) } /// Key-Value database. @@ -245,6 +245,7 @@ pub struct Database { config: DatabaseConfig, write_opts: WriteOptions, read_opts: ReadOptions, + block_opts: BlockBasedOptions, path: String, // Dirty values added with `write_buffered`. Cleaned on `flush`. overlay: RwLock, KeyState>>>, @@ -264,31 +265,35 @@ impl Database { /// Open database file. Creates if it does not exist. pub fn open(config: &DatabaseConfig, path: &str) -> Result { let mut opts = Options::new(); + if let Some(rate_limit) = config.compaction.write_rate_limit { opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit))?; } - opts.set_parsed_options(&format!("max_total_wal_size={}", 64 * 1024 * 1024))?; - opts.set_parsed_options("verify_checksums_in_compaction=0")?; - opts.set_parsed_options("keep_log_file_num=1")?; - opts.set_max_open_files(config.max_open_files); - opts.create_if_missing(true); opts.set_use_fsync(false); - opts.set_db_write_buffer_size(DB_WRITE_BUFFER_SIZE); + opts.create_if_missing(true); + opts.set_max_open_files(config.max_open_files); + opts.set_parsed_options("keep_log_file_num=1")?; + opts.set_parsed_options("bytes_per_sync=1048576")?; + opts.set_db_write_buffer_size(config.memory_budget_per_col() / 2); + opts.increase_parallelism(cmp::max(1, ::num_cpus::get() as i32 / 2)); - opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES); - opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS); + let mut block_opts = BlockBasedOptions::new(); - // compaction settings - opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); - opts.set_target_file_size_base(config.compaction.initial_file_size); - opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier); + { + block_opts.set_block_size(config.compaction.block_size); + let cache_size = cmp::max(8, config.memory_budget() / 3); + let cache = Cache::new(cache_size); + block_opts.set_cache(cache); + } - let mut cf_options = Vec::with_capacity(config.columns.unwrap_or(0) as usize); - let cfnames: Vec<_> = (0..config.columns.unwrap_or(0)).map(|c| format!("col{}", c)).collect(); + let columns = config.columns.unwrap_or(0) as usize; + + let mut cf_options = Vec::with_capacity(columns); + let cfnames: Vec<_> = (0..columns).map(|c| format!("col{}", c)).collect(); let cfnames: Vec<&str> = cfnames.iter().map(|n| n as &str).collect(); - for col in 0 .. config.columns.unwrap_or(0) { - cf_options.push(col_config(col, &config)); + for _ in 0 .. config.columns.unwrap_or(0) { + cf_options.push(col_config(&config, &block_opts)?); } let mut write_opts = WriteOptions::new(); @@ -347,6 +352,7 @@ impl Database { flushing_lock: Mutex::new((false)), path: path.to_owned(), read_opts: read_opts, + block_opts: block_opts, }) } @@ -621,7 +627,7 @@ impl Database { Some(DBAndColumns { ref mut db, ref mut cfs }) => { let col = cfs.len() as u32; let name = format!("col{}", col); - cfs.push(db.create_cf(&name, &col_config(col, &self.config))?); + cfs.push(db.create_cf(&name, &col_config(&self.config, &self.block_opts)?)?); Ok(()) }, None => Ok(()), diff --git a/util/migration/src/lib.rs b/util/migration/src/lib.rs index c29682124..f832db865 100644 --- a/util/migration/src/lib.rs +++ b/util/migration/src/lib.rs @@ -275,7 +275,7 @@ impl Manager { trace!(target: "migration", "Expecting database to contain {:?} columns", columns); let mut db_config = DatabaseConfig { max_open_files: 64, - cache_sizes: Default::default(), + memory_budget: None, compaction: config.compaction_profile, columns: columns, wal: true,