Merge branch 'master' into miner-improvements

Cargo.lock

@@ -1077,7 +1077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "rocksdb"
 version = "0.4.5"
-source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541"
+source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6"
 dependencies = [
  "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)",
@@ -1086,7 +1086,7 @@ dependencies = [
 [[package]]
 name = "rocksdb-sys"
 version = "0.3.0"
-source = "git+https://github.com/ethcore/rust-rocksdb#6f3c68f5f075433d206be4af6a620651cd9f8541"
+source = "git+https://github.com/ethcore/rust-rocksdb#9be41e05923616dfa28741c58b22776d479751e6"
 dependencies = [
  "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",

ethcore/src/client/client.rs

@@ -38,7 +38,7 @@ use filter::Filter;
 use log_entry::LocalizedLogEntry;
 use block_queue::{BlockQueue, BlockQueueInfo};
 use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute};
-use client::{BlockID, TransactionID, UncleID, TraceId, ClientConfig, BlockChainClient, MiningBlockChainClient, TraceFilter, CallAnalytics};
+use client::{BlockID, TransactionID, UncleID, TraceId, ClientConfig, DatabaseCompactionProfile, BlockChainClient, MiningBlockChainClient, TraceFilter, CallAnalytics};
 use client::Error as ClientError;
 use env_info::EnvInfo;
 use executive::{Executive, Executed, TransactOptions, contract_address};
@@ -146,10 +146,19 @@ impl<V> Client<V> where V: Verifier {
 		let chain = Arc::new(BlockChain::new(config.blockchain, &gb, &path));
 		let tracedb = Arc::new(try!(TraceDB::new(config.tracing, &path, chain.clone())));
 
+		let mut state_db_config = match config.db_cache_size {
+			None => DatabaseConfig::default(),
+			Some(cache_size) => DatabaseConfig::with_cache(cache_size),
+		};
+
+		if config.db_compaction == DatabaseCompactionProfile::HDD {
+			state_db_config = state_db_config.compaction(CompactionProfile::hdd());
+		}
+
 		let mut state_db = journaldb::new(
 			&append_path(&path, "state"),
 			config.pruning,
-			config.db_cache_size
+			state_db_config
 		);
 
 		if state_db.is_empty() && spec.ensure_db_good(state_db.as_hashdb_mut()) {

ethcore/src/client/config.rs

@@ -20,6 +20,19 @@ pub use trace::{Config as TraceConfig, Switch};
 pub use evm::VMType;
 use util::journaldb;
 
+/// Client state db compaction profile
+#[derive(Debug, PartialEq)]
+pub enum DatabaseCompactionProfile {
+	/// Default compaction profile
+	Default,
+	/// HDD or other slow storage io compaction profile
+	HDD,
+}
+
+impl Default for DatabaseCompactionProfile {
+	fn default() -> Self { DatabaseCompactionProfile::Default }
+}
+
 /// Client configuration. Includes configs for all sub-systems.
 #[derive(Debug, Default)]
 pub struct ClientConfig {
@@ -37,4 +50,6 @@ pub struct ClientConfig {
 	pub name: String,
 	/// State db cache-size if not default
 	pub db_cache_size: Option<usize>,
+	/// State db compaction profile
+	pub db_compaction: DatabaseCompactionProfile,
 }

ethcore/src/client/mod.rs

@@ -23,7 +23,7 @@ mod test_client;
 mod trace;
 
 pub use self::client::*;
-pub use self::config::{ClientConfig, BlockQueueConfig, BlockChainConfig, Switch, VMType};
+pub use self::config::{ClientConfig, DatabaseCompactionProfile, BlockQueueConfig, BlockChainConfig, Switch, VMType};
 pub use self::error::Error;
 pub use types::ids::*;
 pub use self::test_client::{TestBlockChainClient, EachBlockWith};

ethcore/src/tests/helpers.rs

@@ -303,7 +303,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult<BlockChain> {
 
 pub fn get_temp_journal_db() -> GuardedTempResult<Box<JournalDB>> {
 	let temp = RandomTempPath::new();
-	let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, None);
+	let journal_db = journaldb::new(temp.as_str(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default());
 	GuardedTempResult {
 		_temp: temp,
 		result: Some(journal_db)
@@ -320,7 +320,7 @@ pub fn get_temp_state() -> GuardedTempResult<State> {
 }
 
 pub fn get_temp_journal_db_in(path: &Path) -> Box<JournalDB> {
-	journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge, None)
+	journaldb::new(path.to_str().unwrap(), journaldb::Algorithm::EarlyMerge, DatabaseConfig::default())
 }
 
 pub fn get_temp_state_in(path: &Path) -> State {

parity/cli.rs

@@ -186,7 +186,12 @@ Footprint Options:
   --cache MEGABYTES        Set total amount of discretionary memory to use for
                            the entire system, overrides other cache and queue
                            options.
-  --db-cache-size MB       Database cache size.
+
+Database Options:
+  --db-cache-size MB       Override RocksDB database cache size.
+  --db-compaction TYPE     Database compaction type. TYPE may be one of:
+                           ssd - suitable for SSDs and fast HDDs;
+                           hdd - suitable for slow HDDs [default: ssd].
 
 Import/Export Options:
   --from BLOCK             Export from block BLOCK, which may be an index or
@@ -342,6 +347,7 @@ pub struct Args {
 	pub flag_ipcpath: Option<String>,
 	pub flag_ipcapi: Option<String>,
 	pub flag_db_cache_size: Option<usize>,
+	pub flag_db_compaction: String,
 }
 
 pub fn print_version() {

parity/configuration.rs

@@ -26,7 +26,7 @@ use die::*;
 use util::*;
 use ethcore::account_provider::AccountProvider;
 use util::network_settings::NetworkSettings;
-use ethcore::client::{append_path, get_db_path, ClientConfig, Switch, VMType};
+use ethcore::client::{append_path, get_db_path, ClientConfig, DatabaseCompactionProfile, Switch, VMType};
 use ethcore::miner::{MinerOptions, PendingSet};
 use ethcore::ethereum;
 use ethcore::spec::Spec;
@@ -259,7 +259,7 @@ impl Configuration {
 		let mut latest_era = None;
 		let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
 		for i in jdb_types.into_iter() {
-			let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, None);
+			let db = journaldb::new(&append_path(&get_db_path(Path::new(&self.path()), *i, spec.genesis_header().hash()), "state"), *i, kvdb::DatabaseConfig::default());
 			trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era());
 			match (latest_era, db.latest_era()) {
 				(Some(best), Some(this)) if best >= this => {}
@@ -310,6 +310,13 @@ impl Configuration {
 		// forced state db cache size if provided
 		client_config.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
 
+		// compaction profile
+		client_config.db_compaction = match self.args.flag_db_compaction.as_str() {
+			"ssd" => DatabaseCompactionProfile::Default,
+			"hdd" => DatabaseCompactionProfile::HDD,
+			_ => { die!("Invalid compaction profile given (--db-compaction argument), expected hdd/default."); }
+		};
+
 		if self.args.flag_jitvm {
 			client_config.vm_type = VMType::jit().unwrap_or_else(|| die!("Parity built without jit vm."))
 		}

parity/migration.rs

@@ -20,7 +20,7 @@ use std::io::{Read, Write, Error as IoError, ErrorKind};
 use std::path::PathBuf;
 use std::fmt::{Display, Formatter, Error as FmtError};
 use util::migration::{Manager as MigrationManager, Config as MigrationConfig, MigrationIterator};
-use util::kvdb::{Database, DatabaseConfig};
+use util::kvdb::{Database, DatabaseConfig, CompactionProfile};
 use ethcore::migrations;
 
 /// Database is assumed to be at default version, when no version file is found.
@@ -163,6 +163,7 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -
 			prefix_size: None,
 			max_open_files: 64,
 			cache_size: None,
+			compaction: CompactionProfile::default(),
 		};
 
 		// open old database

util/src/journaldb/archivedb.rs

@@ -43,13 +43,8 @@ const DB_VERSION : u32 = 0x103;
 
 impl ArchiveDB {
 	/// Create a new instance from file
-	pub fn new(path: &str, cache_size: Option<usize>) -> ArchiveDB {
+	pub fn new(path: &str, config: DatabaseConfig) -> ArchiveDB {
-		let opts = DatabaseConfig {
+		let opts = config.prefix(DB_PREFIX_LEN);
-			// this must match account_db prefix
-			prefix_size: Some(DB_PREFIX_LEN),
-			max_open_files: 256,
-			cache_size: cache_size,
-		};
 		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
 			panic!("Error opening state db: {}", e);
 		});
@@ -75,7 +70,7 @@ impl ArchiveDB {
 	fn new_temp() -> ArchiveDB {
 		let mut dir = env::temp_dir();
 		dir.push(H32::random().hex());
-		Self::new(dir.to_str().unwrap(), None)
+		Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
 	}
 
 	fn payload(&self, key: &H256) -> Option<Bytes> {
@@ -187,6 +182,7 @@ mod tests {
 	use super::*;
 	use hashdb::*;
 	use journaldb::traits::JournalDB;
+	use kvdb::DatabaseConfig;
 
 	#[test]
 	fn insert_same_in_fork() {
@@ -328,7 +324,7 @@ mod tests {
 		let bar = H256::random();
 
 		let foo = {
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.emplace(bar.clone(), b"bar".to_vec());
@@ -337,13 +333,13 @@ mod tests {
 		};
 
 		{
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.remove(&foo);
 			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
 		}
 
 		{
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			assert!(jdb.contains(&foo));
 			assert!(jdb.contains(&bar));
 			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@@ -356,7 +352,7 @@ mod tests {
 		dir.push(H32::random().hex());
 
 		let foo = {
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -370,7 +366,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.remove(&foo);
 			jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap();
 			assert!(jdb.contains(&foo));
@@ -385,7 +381,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 		let (foo, _, _) = {
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			let bar = jdb.insert(b"bar");
@@ -400,7 +396,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = ArchiveDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
 			assert!(jdb.contains(&foo));
 		}
@@ -411,14 +407,14 @@ mod tests {
 		let temp = ::devtools::RandomTempPath::new();
 
 		let key = {
-			let mut jdb = ArchiveDB::new(temp.as_str(), None);
+			let mut jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
 			let key = jdb.insert(b"foo");
 			jdb.commit(0, &b"0".sha3(), None).unwrap();
 			key
 		};
 
 		{
-			let jdb = ArchiveDB::new(temp.as_str(), None);
+			let jdb = ArchiveDB::new(temp.as_str(), DatabaseConfig::default());
 			let state = jdb.state(&key);
 			assert!(state.is_some());
 		}

util/src/journaldb/earlymergedb.rs

@@ -73,13 +73,8 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl EarlyMergeDB {
 	/// Create a new instance from file
-	pub fn new(path: &str, cache_size: Option<usize>) -> EarlyMergeDB {
+	pub fn new(path: &str, config: DatabaseConfig) -> EarlyMergeDB {
-		let opts = DatabaseConfig {
+		let opts = config.prefix(DB_PREFIX_LEN);
-			// this must match account_db prefix
-			prefix_size: Some(DB_PREFIX_LEN),
-			max_open_files: 256,
-			cache_size: cache_size,
-		};
 		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
 			panic!("Error opening state db: {}", e);
 		});
@@ -107,7 +102,7 @@ impl EarlyMergeDB {
 	fn new_temp() -> EarlyMergeDB {
 		let mut dir = env::temp_dir();
 		dir.push(H32::random().hex());
-		Self::new(dir.to_str().unwrap(), None)
+		Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
 	}
 
 	fn morph_key(key: &H256, index: u8) -> Bytes {
@@ -537,6 +532,7 @@ mod tests {
 	use super::super::traits::JournalDB;
 	use hashdb::*;
 	use log::init_log;
+	use kvdb::DatabaseConfig;
 
 	#[test]
 	fn insert_same_in_fork() {
@@ -714,7 +710,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -742,7 +738,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -770,7 +766,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -808,7 +804,7 @@ mod tests {
 		let bar = H256::random();
 
 		let foo = {
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.emplace(bar.clone(), b"bar".to_vec());
@@ -818,14 +814,14 @@ mod tests {
 		};
 
 		{
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.remove(&foo);
 			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 		}
 
 		{
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			assert!(jdb.contains(&foo));
 			assert!(jdb.contains(&bar));
 			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@@ -840,7 +836,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 		// history is 4
 		let foo = jdb.insert(b"foo");
@@ -869,7 +865,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 		// history is 4
 		let foo = jdb.insert(b"foo");
@@ -918,7 +914,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		// history is 1
 		let foo = jdb.insert(b"foo");
 		jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
@@ -949,7 +945,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		// history is 4
 		let foo = jdb.insert(b"foo");
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -989,7 +985,7 @@ mod tests {
 		let foo = b"foo".sha3();
 
 		{
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			jdb.insert(b"foo");
 			jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -1010,7 +1006,7 @@ mod tests {
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.remove(&foo);
 			jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
@@ -1018,14 +1014,14 @@ mod tests {
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
@@ -1038,7 +1034,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 		let (foo, bar, baz) = {
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			let bar = jdb.insert(b"bar");
@@ -1056,7 +1052,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = EarlyMergeDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			assert!(jdb.contains(&foo));

util/src/journaldb/mod.rs

@@ -17,6 +17,7 @@
 //! `JournalDB` interface and implementation.
 
 use common::*;
+use kvdb::DatabaseConfig;
 
 /// Export the journaldb module.
 pub mod traits;
@@ -71,12 +72,12 @@ impl fmt::Display for Algorithm {
 }
 
 /// Create a new `JournalDB` trait object.
-pub fn new(path: &str, algorithm: Algorithm, cache_size: Option<usize>) -> Box<JournalDB> {
+pub fn new(path: &str, algorithm: Algorithm, config: DatabaseConfig) -> Box<JournalDB> {
 	match algorithm {
-		Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, cache_size)),
+		Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(path, config)),
-		Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, cache_size)),
+		Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(path, config)),
-		Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, cache_size)),
+		Algorithm::OverlayRecent => Box::new(overlayrecentdb::OverlayRecentDB::new(path, config)),
-		Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, cache_size)),
+		Algorithm::RefCounted => Box::new(refcounteddb::RefCountedDB::new(path, config)),
 	}
 }
 

util/src/journaldb/overlayrecentdb.rs

@@ -98,18 +98,13 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl OverlayRecentDB {
 	/// Create a new instance from file
-	pub fn new(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
+	pub fn new(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
-		Self::from_prefs(path, cache_size)
+		Self::from_prefs(path, config)
 	}
 
 	/// Create a new instance from file
-	pub fn from_prefs(path: &str, cache_size: Option<usize>) -> OverlayRecentDB {
+	pub fn from_prefs(path: &str, config: DatabaseConfig) -> OverlayRecentDB {
-		let opts = DatabaseConfig {
+		let opts = config.prefix(DB_PREFIX_LEN);
-			// this must match account_db prefix
-			prefix_size: Some(DB_PREFIX_LEN),
-			max_open_files: 256,
-			cache_size: cache_size,
-		};
 		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
 			panic!("Error opening state db: {}", e);
 		});
@@ -135,7 +130,7 @@ impl OverlayRecentDB {
 	pub fn new_temp() -> OverlayRecentDB {
 		let mut dir = env::temp_dir();
 		dir.push(H32::random().hex());
-		Self::new(dir.to_str().unwrap(), None)
+		Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
 	}
 
 	#[cfg(test)]
@@ -369,6 +364,7 @@ mod tests {
 	use hashdb::*;
 	use log::init_log;
 	use journaldb::JournalDB;
+	use kvdb::DatabaseConfig;
 
 	#[test]
 	fn insert_same_in_fork() {
@@ -526,7 +522,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -554,7 +550,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -582,7 +578,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
 		assert!(jdb.can_reconstruct_refs());
 
@@ -620,7 +616,7 @@ mod tests {
 		let bar = H256::random();
 
 		let foo = {
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			jdb.emplace(bar.clone(), b"bar".to_vec());
@@ -630,14 +626,14 @@ mod tests {
 		};
 
 		{
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.remove(&foo);
 			jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 		}
 
 		{
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			assert!(jdb.contains(&foo));
 			assert!(jdb.contains(&bar));
 			jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap();
@@ -652,7 +648,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 		// history is 4
 		let foo = jdb.insert(b"foo");
@@ -681,7 +677,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 		// history is 4
 		let foo = jdb.insert(b"foo");
@@ -730,7 +726,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		// history is 1
 		let foo = jdb.insert(b"foo");
 		jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap();
@@ -761,7 +757,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 
-		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 		// history is 4
 		let foo = jdb.insert(b"foo");
 		jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -801,7 +797,7 @@ mod tests {
 		let foo = b"foo".sha3();
 
 		{
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			jdb.insert(b"foo");
 			jdb.commit(0, &b"0".sha3(), None).unwrap();
@@ -822,7 +818,7 @@ mod tests {
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.remove(&foo);
 			jdb.commit(4, &b"4".sha3(), Some((2, b"2".sha3()))).unwrap();
@@ -830,14 +826,14 @@ mod tests {
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.commit(5, &b"5".sha3(), Some((3, b"3".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			assert!(jdb.contains(&foo));
 
 		// incantation to reopen the db
-		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+		}; { let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 
 			jdb.commit(6, &b"6".sha3(), Some((4, b"4".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
@@ -850,7 +846,7 @@ mod tests {
 		let mut dir = ::std::env::temp_dir();
 		dir.push(H32::random().hex());
 		let (foo, bar, baz) = {
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			// history is 1
 			let foo = jdb.insert(b"foo");
 			let bar = jdb.insert(b"bar");
@@ -868,7 +864,7 @@ mod tests {
 		};
 
 		{
-			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), None);
+			let mut jdb = OverlayRecentDB::new(dir.to_str().unwrap(), DatabaseConfig::default());
 			jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap();
 			assert!(jdb.can_reconstruct_refs());
 			assert!(jdb.contains(&foo));

util/src/journaldb/refcounteddb.rs

@@ -46,13 +46,8 @@ const PADDING : [u8; 10] = [ 0u8; 10 ];
 
 impl RefCountedDB {
 	/// Create a new instance given a `backing` database.
-	pub fn new(path: &str, cache_size: Option<usize>) -> RefCountedDB {
+	pub fn new(path: &str, config: DatabaseConfig) -> RefCountedDB {
-		let opts = DatabaseConfig {
+		let opts = config.prefix(DB_PREFIX_LEN);
-			// this must match account_db prefix
-			prefix_size: Some(DB_PREFIX_LEN),
-			max_open_files: 256,
-			cache_size: cache_size,
-		};
 		let backing = Database::open(&opts, path).unwrap_or_else(|e| {
 			panic!("Error opening state db: {}", e);
 		});
@@ -82,7 +77,7 @@ impl RefCountedDB {
 	fn new_temp() -> RefCountedDB {
 		let mut dir = env::temp_dir();
 		dir.push(H32::random().hex());
-		Self::new(dir.to_str().unwrap(), None)
+		Self::new(dir.to_str().unwrap(), DatabaseConfig::default())
 	}
 }
 

util/src/kvdb.rs

@@ -20,8 +20,6 @@ use std::default::Default;
 use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator,
 	IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction};
 
-const DB_FILE_SIZE_BASE: u64 = 32 * 1024 * 1024;
-const DB_FILE_SIZE_MULTIPLIER: i32 = 2;
 const DB_BACKGROUND_FLUSHES: i32 = 2;
 const DB_BACKGROUND_COMPACTIONS: i32 = 2;
 
@@ -53,6 +51,36 @@ impl DBTransaction {
 	}
 }
 
+/// Compaction profile for the database settings
+pub struct CompactionProfile {
+	/// L0-L1 target file size
+	pub initial_file_size: u64,
+	/// L2-LN target file size multiplier
+	pub file_size_multiplier: i32,
+	/// rate limiter for background flushes and compactions, bytes/sec, if any
+	pub write_rate_limit: Option<u64>,
+}
+
+impl CompactionProfile {
+	/// Default profile suitable for most storage
+	pub fn default() -> CompactionProfile {
+		CompactionProfile {
+			initial_file_size: 32 * 1024 * 1024,
+			file_size_multiplier: 2,
+			write_rate_limit: None,
+		}
+	}
+
+	/// Slow hdd compaction profile
+	pub fn hdd() -> CompactionProfile {
+		CompactionProfile {
+			initial_file_size: 192 * 1024 * 1024,
+			file_size_multiplier: 1,
+			write_rate_limit: Some(8 * 1024 * 1024),
+		}
+	}
+}
+
 /// Database configuration
 pub struct DatabaseConfig {
 	/// Optional prefix size in bytes. Allows lookup by partial key.
@@ -61,6 +89,8 @@ pub struct DatabaseConfig {
 	pub max_open_files: i32,
 	/// Cache-size
 	pub cache_size: Option<usize>,
+	/// Compaction profile
+	pub compaction: CompactionProfile,
 }
 
 impl DatabaseConfig {
@@ -70,8 +100,21 @@ impl DatabaseConfig {
 			cache_size: Some(cache_size),
 			prefix_size: None,
 			max_open_files: -1,
+			compaction: CompactionProfile::default(),
 		}
 	}
+
+	/// Modify the compaction profile
+	pub fn compaction(mut self, profile: CompactionProfile) -> Self {
+		self.compaction = profile;
+		self
+	}
+
+	/// Modify the prefix of the db
+	pub fn prefix(mut self, prefix_size: usize) -> Self {
+		self.prefix_size = Some(prefix_size);
+		self
+	}
 }
 
 impl Default for DatabaseConfig {
@@ -80,6 +123,7 @@ impl Default for DatabaseConfig {
 			cache_size: None,
 			prefix_size: None,
 			max_open_files: -1,
+			compaction: CompactionProfile::default(),
 		}
 	}
 }
@@ -111,13 +155,18 @@ impl Database {
 	/// Open database file. Creates if it does not exist.
 	pub fn open(config: &DatabaseConfig, path: &str) -> Result<Database, String> {
 		let mut opts = Options::new();
-		try!(opts.set_parsed_options("rate_limiter_bytes_per_sec=256000000"));
+		if let Some(rate_limit) = config.compaction.write_rate_limit {
+			try!(opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit)));
+		}
 		opts.set_max_open_files(config.max_open_files);
 		opts.create_if_missing(true);
 		opts.set_use_fsync(false);
+
+		// compaction settings
 		opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction);
-		opts.set_target_file_size_base(DB_FILE_SIZE_BASE);
+		opts.set_target_file_size_base(config.compaction.initial_file_size);
-		opts.set_target_file_size_multiplier(DB_FILE_SIZE_MULTIPLIER);
+		opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier);
+
 		opts.set_max_background_flushes(DB_BACKGROUND_FLUSHES);
 		opts.set_max_background_compactions(DB_BACKGROUND_COMPACTIONS);
 		if let Some(cache_size) = config.cache_size {
@@ -150,7 +199,16 @@ impl Database {
 			opts.set_block_based_table_factory(&block_opts);
 			opts.set_prefix_extractor_fixed_size(size);
 		}
-		let db = try!(DB::open(&opts, path));
+		let db = match DB::open(&opts, path) {
+			Ok(db) => db,
+			Err(ref s) if s.starts_with("Corruption:") => {
+				info!("{}", s);
+				info!("Attempting DB repair for {}", path);
+				try!(DB::repair(&opts, path));
+				try!(DB::open(&opts, path))
+			},
+			Err(s) => { return Err(s); }
+		};
 		Ok(Database { db: db })
 	}
 
@@ -244,10 +302,10 @@ mod tests {
 		let path = RandomTempPath::create_dir();
 		let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap();
 		assert!(smoke.is_empty());
-		test_db(&DatabaseConfig { prefix_size: None, max_open_files: 256, cache_size: None, });
+		test_db(&DatabaseConfig::default());
-		test_db(&DatabaseConfig { prefix_size: Some(1), max_open_files: 256, cache_size: None,  });
+		test_db(&DatabaseConfig::default().prefix(12));
-		test_db(&DatabaseConfig { prefix_size: Some(8), max_open_files: 256, cache_size: None,  });
+		test_db(&DatabaseConfig::default().prefix(22));
-		test_db(&DatabaseConfig { prefix_size: Some(32), max_open_files: 256, cache_size: None,  });
+		test_db(&DatabaseConfig::default().prefix(8));
 	}
 }