new blooms database (#8712)

* new blooms database

* fixed conflict in Cargo.lock

* removed bloomchain

* cleanup in progress

* all tests passing in trace db with new blooms-db

* added trace_blooms to BlockChainDB interface, fixed db flushing

* BlockChainDB no longer exposes RwLock in the interface

* automatically flush blooms-db after every insert

* blooms-db uses io::BufReader to read files, wrap blooms-db into Mutex, cause fs::File is just a shared file handle

* fix json_tests

* blooms-db can filter multiple possibilities at the same time

* removed enum trace/db.rs CacheId

* lint fixes

* fixed tests

* kvdb-rocksdb uses fs-swap crate

* update Cargo.lock

* use fs::rename

* fixed failing test on linux

* fix tests

* use fs_swap

* fixed failing test on linux

* cleanup after swap

* fix tests

* fixed osx permissions

* simplify parity database opening functions

* added migration to blooms-db

* address @niklasad1 grumbles

* fix license and authors field of blooms-db Cargo.toml

* restore blooms-db after snapshot

parity/db/mod.rs

@@ -19,7 +19,7 @@
 #[path="rocksdb/mod.rs"]
 mod impls;
 
-pub use self::impls::{open_db, open_client_db, restoration_db_handler, migrate};
+pub use self::impls::{open_db, restoration_db_handler, migrate};
 
 #[cfg(feature = "secretstore")]
 pub use self::impls::open_secretstore_db;

parity/db/rocksdb/blooms.rs

@@ -0,0 +1,84 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Blooms migration from rocksdb to blooms-db
+
+use std::path::Path;
+use ethereum_types::Bloom;
+use ethcore::error::Error;
+use rlp;
+use super::kvdb_rocksdb::DatabaseConfig;
+use super::open_database;
+
+pub fn migrate_blooms<P: AsRef<Path>>(path: P, config: &DatabaseConfig) -> Result<(), Error> {
+	// init
+	let db = open_database(&path.as_ref().to_string_lossy(), config)?;
+
+	// possible optimization:
+	// pre-allocate space on disk for faster migration
+
+	// iterate over header blooms and insert them in blooms-db
+	// Some(3) -> COL_EXTRA
+	// 3u8 -> ExtrasIndex::BlocksBlooms
+	// 0u8 -> level 0
+	let blooms_iterator = db.key_value()
+		.iter_from_prefix(Some(3), &[3u8, 0u8])
+		.filter(|(key, _)| key.len() == 6)
+		.take_while(|(key, _)| {
+			key[0] == 3u8 && key[1] == 0u8
+		})
+		.map(|(key, group)| {
+			let number =
+				(key[2] as u64) << 24 |
+				(key[3] as u64) << 16 |
+				(key[4] as u64) << 8 |
+				(key[5] as u64);
+
+			let blooms = rlp::decode_list::<Bloom>(&group);
+			(number, blooms)
+		});
+
+	for (number, blooms) in blooms_iterator {
+		db.blooms().insert_blooms(number, blooms.iter())?;
+	}
+
+	// iterate over trace blooms and insert them in blooms-db
+	// Some(4) -> COL_TRACE
+	// 1u8 -> TraceDBIndex::BloomGroups
+	// 0u8 -> level 0
+	let trace_blooms_iterator = db.key_value()
+		.iter_from_prefix(Some(4), &[1u8, 0u8])
+		.filter(|(key, _)| key.len() == 6)
+		.take_while(|(key, _)| {
+			key[0] == 1u8 && key[1] == 0u8
+		})
+		.map(|(key, group)| {
+			let number =
+				(key[2] as u64) |
+				(key[3] as u64) << 8 |
+				(key[4] as u64) << 16 |
+				(key[5] as u64) << 24;
+
+			let blooms = rlp::decode_list::<Bloom>(&group);
+			(number, blooms)
+		});
+
+	for (number, blooms) in trace_blooms_iterator {
+		db.trace_blooms().insert_blooms(number, blooms.iter())?;
+	}
+
+	Ok(())
+}

parity/db/rocksdb/migration.rs

@@ -19,10 +19,12 @@ use std::io::{Read, Write, Error as IoError, ErrorKind};
 use std::path::{Path, PathBuf};
 use std::fmt::{Display, Formatter, Error as FmtError};
 use super::migration_rocksdb::{self, Manager as MigrationManager, Config as MigrationConfig, ChangeColumns};
-use super::kvdb_rocksdb::CompactionProfile;
+use super::kvdb_rocksdb::{CompactionProfile, DatabaseConfig};
 use ethcore::client::DatabaseCompactionProfile;
+use ethcore::{self, db};
 
 use super::helpers;
+use super::blooms::migrate_blooms;
 
 /// The migration from v10 to v11.
 /// Adds a column for node info.
@@ -43,9 +45,9 @@ pub const TO_V12: ChangeColumns = ChangeColumns {
 /// Database is assumed to be at default version, when no version file is found.
 const DEFAULT_VERSION: u32 = 5;
 /// Current version of database models.
-const CURRENT_VERSION: u32 = 12;
-/// First version of the consolidated database.
-const CONSOLIDATION_VERSION: u32 = 9;
+const CURRENT_VERSION: u32 = 13;
+/// A version of database at which blooms-db was introduced
+const BLOOMS_DB_VERSION: u32 = 13;
 /// Defines how many items are migrated to the new version of database at once.
 const BATCH_SIZE: usize = 1024;
 /// Version file name.
@@ -62,6 +64,8 @@ pub enum Error {
 	MigrationImpossible,
 	/// Internal migration error.
 	Internal(migration_rocksdb::Error),
+	/// Blooms-db migration error.
+	BloomsDB(ethcore::error::Error),
 	/// Migration was completed succesfully,
 	/// but there was a problem with io.
 	Io(IoError),
@@ -74,6 +78,7 @@ impl Display for Error {
 			Error::FutureDBVersion => "Database was created with newer client version. Upgrade your client or delete DB and resync.".into(),
 			Error::MigrationImpossible => format!("Database migration to version {} is not possible.", CURRENT_VERSION),
 			Error::Internal(ref err) => format!("{}", err),
+			Error::BloomsDB(ref err) => format!("blooms-db migration error: {}", err),
 			Error::Io(ref err) => format!("Unexpected io error on DB migration: {}.", err),
 		};
 
@@ -212,9 +217,23 @@ pub fn migrate(path: &Path, compaction_profile: &DatabaseCompactionProfile) -> R
 	}
 
 	// Further migrations
-	if version >= CONSOLIDATION_VERSION && version < CURRENT_VERSION && exists(&consolidated_database_path(path)) {
-		println!("Migrating database from version {} to {}", ::std::cmp::max(CONSOLIDATION_VERSION, version), CURRENT_VERSION);
+	if version < CURRENT_VERSION && exists(&consolidated_database_path(path)) {
+		println!("Migrating database from version {} to {}", version, CURRENT_VERSION);
 		migrate_database(version, consolidated_database_path(path), consolidated_database_migrations(&compaction_profile)?)?;
+
+		if version < BLOOMS_DB_VERSION {
+			println!("Migrating blooms to blooms-db...");
+			let db_config = DatabaseConfig {
+				max_open_files: 64,
+				memory_budget: None,
+				compaction: compaction_profile,
+				columns: db::NUM_COLUMNS,
+				wal: true,
+			};
+
+			migrate_blooms(path, &db_config).map_err(Error::BloomsDB)?;
+		}
+
 		println!("Migration finished");
 	}
 

parity/db/rocksdb/mod.rs

@@ -17,20 +17,45 @@
 extern crate kvdb_rocksdb;
 extern crate migration_rocksdb;
 
+use std::fs;
 use std::sync::Arc;
 use std::path::Path;
+use blooms_db;
+use ethcore::{BlockChainDBHandler, BlockChainDB};
+use ethcore::error::Error;
 use ethcore::db::NUM_COLUMNS;
 use ethcore::client::{ClientConfig, DatabaseCompactionProfile};
-use kvdb::{KeyValueDB, KeyValueDBHandler};
+use kvdb::KeyValueDB;
 use self::kvdb_rocksdb::{Database, DatabaseConfig};
 
 use cache::CacheConfig;
 
+mod blooms;
 mod migration;
 mod helpers;
 
 pub use self::migration::migrate;
 
+struct AppDB {
+	key_value: Arc<KeyValueDB>,
+	blooms: blooms_db::Database,
+	trace_blooms: blooms_db::Database,
+}
+
+impl BlockChainDB for AppDB {
+	fn key_value(&self) -> &Arc<KeyValueDB> {
+		&self.key_value
+	}
+
+	fn blooms(&self) -> &blooms_db::Database {
+		&self.blooms
+	}
+
+	fn trace_blooms(&self) -> &blooms_db::Database {
+		&self.trace_blooms
+	}
+}
+
 /// Open a secret store DB using the given secret store data path. The DB path is one level beneath the data path.
 #[cfg(feature = "secretstore")]
 pub fn open_secretstore_db(data_path: &str) -> Result<Arc<KeyValueDB>, String> {
@@ -42,31 +67,17 @@ pub fn open_secretstore_db(data_path: &str) -> Result<Arc<KeyValueDB>, String> {
 	Ok(Arc::new(Database::open_default(&db_path).map_err(|e| format!("Error opening database: {:?}", e))?))
 }
 
-/// Open a new client DB.
-pub fn open_client_db(client_path: &Path, client_config: &ClientConfig) -> Result<Arc<KeyValueDB>, String> {
-	let client_db_config = helpers::client_db_config(client_path, client_config);
-
-	let client_db = Arc::new(Database::open(
-		&client_db_config,
-		&client_path.to_str().expect("DB path could not be converted to string.")
-	).map_err(|e| format!("Client service database error: {:?}", e))?);
-
-	Ok(client_db)
-}
-
 /// Create a restoration db handler using the config generated by `client_path` and `client_config`.
-pub fn restoration_db_handler(client_path: &Path, client_config: &ClientConfig) -> Box<KeyValueDBHandler> {
-	use kvdb::Error;
-
+pub fn restoration_db_handler(client_path: &Path, client_config: &ClientConfig) -> Box<BlockChainDBHandler> {
 	let client_db_config = helpers::client_db_config(client_path, client_config);
 
 	struct RestorationDBHandler {
 		config: DatabaseConfig,
 	}
 
-	impl KeyValueDBHandler for RestorationDBHandler {
-		fn open(&self, db_path: &Path) -> Result<Arc<KeyValueDB>, Error> {
-			Ok(Arc::new(Database::open(&self.config, &db_path.to_string_lossy())?))
+	impl BlockChainDBHandler for RestorationDBHandler {
+		fn open(&self, db_path: &Path) -> Result<Arc<BlockChainDB>, Error> {
+			open_database(&db_path.to_string_lossy(), &self.config)
 		}
 	}
 
@@ -76,16 +87,32 @@ pub fn restoration_db_handler(client_path: &Path, client_config: &ClientConfig)
 }
 
 /// Open a new main DB.
-pub fn open_db(client_path: &str, cache_config: &CacheConfig, compaction: &DatabaseCompactionProfile, wal: bool) -> Result<Arc<KeyValueDB>, String> {
+pub fn open_db(client_path: &str, cache_config: &CacheConfig, compaction: &DatabaseCompactionProfile, wal: bool) -> Result<Arc<BlockChainDB>, Error> {
+	let path = Path::new(client_path);
+
 	let db_config = DatabaseConfig {
 		memory_budget: Some(cache_config.blockchain() as usize * 1024 * 1024),
-		compaction: helpers::compaction_profile(&compaction, &Path::new(client_path)),
-		wal: wal,
+		compaction: helpers::compaction_profile(&compaction, path),
+		wal,
 		.. DatabaseConfig::with_columns(NUM_COLUMNS)
 	};
 
-	Ok(Arc::new(Database::open(
-		&db_config,
-		client_path
-	).map_err(|e| format!("Failed to open database: {}", e))?))
+	open_database(client_path, &db_config)
+}
+
+pub fn open_database(client_path: &str, config: &DatabaseConfig) -> Result<Arc<BlockChainDB>, Error> {
+	let path = Path::new(client_path);
+
+	let blooms_path = path.join("blooms");
+	let trace_blooms_path = path.join("trace_blooms");
+	fs::create_dir(&blooms_path)?;
+	fs::create_dir(&trace_blooms_path)?;
+
+	let db = AppDB {
+		key_value: Arc::new(Database::open(&config, client_path)?),
+		blooms: blooms_db::Database::open(blooms_path)?,
+		trace_blooms: blooms_db::Database::open(trace_blooms_path)?,
+	};
+
+	Ok(Arc::new(db))
 }