Use upstream rocksdb (#11248)

* Use upstream rocksdb

…by way of https://github.com/paritytech/parity-common/pull/257 by @ordian.

* Hint at how `parity db reset` works in the error message

* migration-rocksdb: fix build

* Cargo.toml: use git dependency instead of path

* update to latest kvdb-rocksdb

* fix tests

* saner default for light client

* rename open_db to open_db_light

* update to latest kvdb-rocksdb

* moar update to latest kvdb-rocksdb

* even moar update to latest kvdb-rocksdb

* use kvdb-rocksdb from crates.io

* Update parity/db/rocksdb/helpers.rs

* add docs to memory_budget division

parity/db/mod.rs

@@ -19,7 +19,7 @@
 #[path="rocksdb/mod.rs"]
 mod impls;
 
-pub use self::impls::{open_db, restoration_db_handler, migrate};
+pub use self::impls::{open_db_light, restoration_db_handler, migrate};
 
 #[cfg(feature = "secretstore")]
 pub use self::impls::open_secretstore_db;

parity/db/rocksdb/helpers.rs

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::collections::HashMap;
 use std::path::Path;
-use ethcore_db::NUM_COLUMNS;
 use ethcore::client::{ClientConfig, DatabaseCompactionProfile};
 use super::kvdb_rocksdb::{CompactionProfile, DatabaseConfig};
 
@@ -27,10 +27,42 @@ pub fn compaction_profile(profile: &DatabaseCompactionProfile, db_path: &Path) -
 	}
 }
 
-pub fn client_db_config(client_path: &Path, client_config: &ClientConfig) -> DatabaseConfig {
-	let mut client_db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
+/// Spreads the `total` (in MiB) memory budget across the db columns.
+/// If it's `None`, the default memory budget will be used for each column.
+pub fn memory_per_column(total: Option<usize>) -> HashMap<Option<u32>, usize> {
+	let mut memory_per_column = HashMap::new();
+	if let Some(budget) = total {
+		// spend 90% of the memory budget on the state column, but at least 256 MiB
+		memory_per_column.insert(ethcore_db::COL_STATE, std::cmp::max(budget * 9 / 10, 256));
+		let num_columns = ethcore_db::NUM_COLUMNS.expect("NUM_COLUMNS is Some; qed");
+		// spread the remaining 10% evenly across columns
+		let rest_budget = budget / 10 / (num_columns as usize - 1);
+		for i in 1..num_columns {
+			// but at least 16 MiB for each column
+			memory_per_column.insert(Some(i), std::cmp::max(rest_budget, 16));
+		}
+	}
+	memory_per_column
+}
 
-	client_db_config.memory_budget = client_config.db_cache_size;
+/// Spreads the `total` (in MiB) memory budget across the light db columns.
+pub fn memory_per_column_light(total: usize) -> HashMap<Option<u32>, usize> {
+	let mut memory_per_column = HashMap::new();
+	let num_columns = ethcore_db::NUM_COLUMNS.expect("NUM_COLUMNS is Some; qed");
+	// spread the memory budget evenly across columns
+	// light client doesn't use the state column
+	let per_column = total / (num_columns as usize - 1);
+	for i in 1..num_columns {
+		// but at least 4 MiB for each column
+		memory_per_column.insert(Some(i), std::cmp::max(per_column, 4));
+	}
+	memory_per_column
+}
+
+pub fn client_db_config(client_path: &Path, client_config: &ClientConfig) -> DatabaseConfig {
+	let mut client_db_config = DatabaseConfig::with_columns(ethcore_db::NUM_COLUMNS);
+
+	client_db_config.memory_budget = memory_per_column(client_config.db_cache_size);
 	client_db_config.compaction = compaction_profile(&client_config.db_compaction, &client_path);
 
 	client_db_config

parity/db/rocksdb/migration.rs

@@ -224,9 +224,9 @@ pub fn migrate(path: &Path, compaction_profile: &DatabaseCompactionProfile) -> R
 			println!("Migrating blooms to blooms-db...");
 			let db_config = DatabaseConfig {
 				max_open_files: 64,
-				memory_budget: None,
 				compaction: compaction_profile,
 				columns: ethcore_db::NUM_COLUMNS,
+				..Default::default()
 			};
 
 			migrate_blooms(&db_path, &db_config).map_err(Error::BloomsDB)?;

parity/db/rocksdb/mod.rs

@@ -86,8 +86,8 @@ pub fn restoration_db_handler(client_path: &Path, client_config: &ClientConfig)
 	})
 }
 
-/// Open a new main DB.
-pub fn open_db(
+/// Open a new light client DB.
+pub fn open_db_light(
 	client_path: &str,
 	cache_config: &CacheConfig,
 	compaction: &DatabaseCompactionProfile
@@ -95,7 +95,7 @@ pub fn open_db(
 	let path = Path::new(client_path);
 
 	let db_config = DatabaseConfig {
-		memory_budget: Some(cache_config.blockchain() as usize * 1024 * 1024),
+		memory_budget: helpers::memory_per_column_light(cache_config.blockchain() as usize),
 		compaction: helpers::compaction_profile(&compaction, path),
 		.. DatabaseConfig::with_columns(NUM_COLUMNS)
 	};