Update to latest kvdb-*: no default column, DBValue is Vec (#11312)

* Only use kvdb "column families"

This PR contains the changes necessary to use the `kvdb-*` crates from https://github.com/paritytech/parity-common/pull/278 (so a synchronized merge is required) which drops support for the old-style rocksdb "default" column to get a smaller and less complex API.

As it stands this PR is working correctly except for secret-store; we need to migrate it to use a new column family.

* Fix secretstore build

* Fix secretstore build: include ethkey when building with the "accounts" feature

* typos

* Restore state test commit

* Override all of parity-common from git

* Be precise about version requirement to migrate secretstore code

* Update ethcore/db/src/db.rs

Co-Authored-By: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* Address review grumbles

* Review grumbles

* Cleanup

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

parity/db/rocksdb/blooms.rs

@@ -37,7 +37,7 @@ pub fn migrate_blooms<P: AsRef<Path>>(path: P, config: &DatabaseConfig) -> Resul
 	// 3u8 -> ExtrasIndex::BlocksBlooms
 	// 0u8 -> level 0
 	let blooms_iterator = db.key_value()
-		.iter_from_prefix(Some(3), &[3u8, 0u8])
+		.iter_from_prefix(3, &[3u8, 0u8])
 		.filter(|(key, _)| key.len() == 6)
 		.take_while(|(key, _)| {
 			key[0] == 3u8 && key[1] == 0u8
@@ -63,7 +63,7 @@ pub fn migrate_blooms<P: AsRef<Path>>(path: P, config: &DatabaseConfig) -> Resul
 	// 1u8 -> TraceDBIndex::BloomGroups
 	// 0u8 -> level 0
 	let trace_blooms_iterator = db.key_value()
-		.iter_from_prefix(Some(4), &[1u8, 0u8])
+		.iter_from_prefix(4, &[1u8, 0u8])
 		.filter(|(key, _)| key.len() == 6)
 		.take_while(|(key, _)| {
 			key[0] == 1u8 && key[1] == 0u8

parity/db/rocksdb/helpers.rs

@@ -29,32 +29,36 @@ pub fn compaction_profile(profile: &DatabaseCompactionProfile, db_path: &Path) -
 
 /// Spreads the `total` (in MiB) memory budget across the db columns.
 /// If it's `None`, the default memory budget will be used for each column.
-pub fn memory_per_column(total: Option<usize>) -> HashMap<Option<u32>, usize> {
+/// 90% of the memory budget is assigned to the first column, `col0`, which is where we store the
+/// state.
+pub fn memory_per_column(total: Option<usize>) -> HashMap<u32, usize> {
 	let mut memory_per_column = HashMap::new();
 	if let Some(budget) = total {
 		// spend 90% of the memory budget on the state column, but at least 256 MiB
 		memory_per_column.insert(ethcore_db::COL_STATE, std::cmp::max(budget * 9 / 10, 256));
-		let num_columns = ethcore_db::NUM_COLUMNS.expect("NUM_COLUMNS is Some; qed");
 		// spread the remaining 10% evenly across columns
-		let rest_budget = budget / 10 / (num_columns as usize - 1);
-		for i in 1..num_columns {
+		let rest_budget = budget / 10 / (ethcore_db::NUM_COLUMNS as usize - 1);
+
+		for i in 1..ethcore_db::NUM_COLUMNS {
 			// but at least 16 MiB for each column
-			memory_per_column.insert(Some(i), std::cmp::max(rest_budget, 16));
+			memory_per_column.insert(i, std::cmp::max(rest_budget, 16));
 		}
 	}
 	memory_per_column
 }
 
 /// Spreads the `total` (in MiB) memory budget across the light db columns.
-pub fn memory_per_column_light(total: usize) -> HashMap<Option<u32>, usize> {
+pub fn memory_per_column_light(total: usize) -> HashMap<u32, usize> {
 	let mut memory_per_column = HashMap::new();
-	let num_columns = ethcore_db::NUM_COLUMNS.expect("NUM_COLUMNS is Some; qed");
 	// spread the memory budget evenly across columns
 	// light client doesn't use the state column
-	let per_column = total / (num_columns as usize - 1);
-	for i in 1..num_columns {
+	let per_column = total / (ethcore_db::NUM_COLUMNS as usize - 1);
+
+	// Note: `col0` (State) is not used for the light client so setting it to a low value.
+	memory_per_column.insert(0, 1);
+	for i in 1..ethcore_db::NUM_COLUMNS {
 		// but at least 4 MiB for each column
-		memory_per_column.insert(Some(i), std::cmp::max(per_column, 4));
+		memory_per_column.insert(i, std::cmp::max(per_column, 4));
 	}
 	memory_per_column
 }

parity/db/rocksdb/migration.rs

@@ -29,24 +29,24 @@ use super::blooms::migrate_blooms;
 /// The migration from v10 to v11.
 /// Adds a column for node info.
 pub const TO_V11: ChangeColumns = ChangeColumns {
-	pre_columns: Some(6),
-	post_columns: Some(7),
+	pre_columns: 6,
+	post_columns: 7,
 	version: 11,
 };
 
 /// The migration from v11 to v12.
 /// Adds a column for light chain storage.
 pub const TO_V12: ChangeColumns = ChangeColumns {
-	pre_columns: Some(7),
-	post_columns: Some(8),
+	pre_columns: 7,
+	post_columns: 8,
 	version: 12,
 };
 
 /// The migration from v12 to v14.
 /// Adds a column for private transactions state storage.
 pub const TO_V14: ChangeColumns = ChangeColumns {
-	pre_columns: Some(8),
-	post_columns: Some(9),
+	pre_columns: 8,
+	post_columns: 9,
 	version: 14,
 };
 

parity/main.rs

@@ -114,7 +114,7 @@ fn take_spec_name_override() -> Option<String> {
 
 #[cfg(windows)]
 fn global_cleanup() {
-	// We need to cleanup all sockets before spawning another Parity process. This makes sure everything is cleaned up.
+	// We need to clean up all sockets before spawning another Parity process. This makes sure everything is cleaned up.
 	// The loop is required because of internal reference counter for winsock dll. We don't know how many crates we use do
 	// initialize it. There's at least 2 now.
 	for _ in 0.. 10 {