Single DB (#1741)

* Consolidation migration

* Started db amalgamation

* Using client constants for columns

* Adding with_columns constructor

* Migrating to single db

* Fixing tests.

* test.sh without verbose

* Fixing warnings

* add migration tests that catch the bug

* make multiple migrations more robust

* add moved v9

* Merge branch 'noop-migrations' into single-db

* spurious line

* clean up migrations ordering

* update comment [ci skip]

* Bumping default number of max_open_files & re-ordering columns.

* fix merge

* fix ignored analysis tests

* Caching best block content

* Faster best_block_header

* Adding progress to v8 migration

* clean up warnings

* Separate hashes and bodies in the DB

* Separate hashes and bodies in the DB

* Fixed tests

parity/dir.rs

@@ -52,13 +52,20 @@ impl Directories {
 		Ok(())
 	}
 
-	/// Get the path for the databases given the root path and information on the databases.
-	pub fn client_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf {
+	/// Get the root path for database
+	pub fn db_version_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf {
 		let mut dir = Path::new(&self.db).to_path_buf();
 		dir.push(format!("{:?}{}", H64::from(genesis_hash), fork_name.map(|f| format!("-{}", f)).unwrap_or_default()));
 		dir.push(format!("v{}-sec-{}", LEGACY_CLIENT_DB_VER_STR, pruning.as_internal_name_str()));
 		dir
 	}
+
+	/// Get the path for the databases given the genesis_hash and information on the databases.
+	pub fn client_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf {
+		let mut dir = self.db_version_path(genesis_hash, fork_name, pruning);
+		dir.push("db");
+		dir
+	}
 }
 
 #[cfg(test)]

parity/helpers.rs

@@ -238,7 +238,7 @@ pub fn execute_upgrades(
 		_ => {},
 	}
 
-	let client_path = dirs.client_path(genesis_hash, fork_name, pruning);
+	let client_path = dirs.db_version_path(genesis_hash, fork_name, pruning);
 	migrate(&client_path, pruning, compaction_profile).map_err(|e| format!("{}", e))
 }
 

parity/migration.rs

@@ -20,14 +20,18 @@ use std::io::{Read, Write, Error as IoError, ErrorKind};
 use std::path::{Path, PathBuf};
 use std::fmt::{Display, Formatter, Error as FmtError};
 use util::journaldb::Algorithm;
-use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError};
-use util::kvdb::CompactionProfile;
+use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError, Migration};
+use util::kvdb::{CompactionProfile, Database, DatabaseConfig};
 use ethcore::migrations;
+use ethcore::client;
+use ethcore::migrations::Extract;
 
 /// Database is assumed to be at default version, when no version file is found.
 const DEFAULT_VERSION: u32 = 5;
 /// Current version of database models.
-const CURRENT_VERSION: u32 = 8;
+const CURRENT_VERSION: u32 = 9;
+/// First version of the consolidated database.
+const CONSOLIDATION_VERSION: u32 = 9;
 /// Defines how many items are migrated to the new version of database at once.
 const BATCH_SIZE: usize = 1024;
 /// Version file name.
@@ -111,27 +115,13 @@ fn update_version(path: &Path) -> Result<(), Error> {
 	Ok(())
 }
 
-/// State database path.
-fn state_database_path(path: &Path) -> PathBuf {
+/// Consolidated database path
+fn consolidated_database_path(path: &Path) -> PathBuf {
 	let mut state_path = path.to_owned();
-	state_path.push("state");
+	state_path.push("db");
 	state_path
 }
 
-/// Blocks database path.
-fn blocks_database_path(path: &Path) -> PathBuf {
-	let mut blocks_path = path.to_owned();
-	blocks_path.push("blocks");
-	blocks_path
-}
-
-/// Extras database path.
-fn extras_database_path(path: &Path) -> PathBuf {
-	let mut extras_path = path.to_owned();
-	extras_path.push("extras");
-	extras_path
-}
-
 /// Database backup
 fn backup_database_path(path: &Path) -> PathBuf {
 	let mut backup_path = path.to_owned();
@@ -141,40 +131,55 @@ fn backup_database_path(path: &Path) -> PathBuf {
 }
 
 /// Default migration settings.
-fn default_migration_settings(compaction_profile: CompactionProfile) -> MigrationConfig {
+pub fn default_migration_settings(compaction_profile: &CompactionProfile) -> MigrationConfig {
 	MigrationConfig {
 		batch_size: BATCH_SIZE,
-		compaction_profile: compaction_profile,
+		compaction_profile: *compaction_profile,
 	}
 }
 
-/// Migrations on the blocks database.
-fn blocks_database_migrations(compaction_profile: CompactionProfile) -> Result<MigrationManager, Error> {
-	let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
-	try!(manager.add_migration(migrations::blocks::V8::default()).map_err(|_| Error::MigrationImpossible));
+/// Migrations on the consolidated database.
+fn consolidated_database_migrations(compaction_profile: &CompactionProfile) -> Result<MigrationManager, Error> {
+	let manager = MigrationManager::new(default_migration_settings(compaction_profile));
 	Ok(manager)
 }
 
-/// Migrations on the extras database.
-fn extras_database_migrations(compaction_profile: CompactionProfile) -> Result<MigrationManager, Error> {
-	let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
-	try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible));
-	Ok(manager)
-}
+/// Consolidates legacy databases into single one.
+fn consolidate_database(
+	old_db_path: PathBuf,
+	new_db_path: PathBuf,
+	column: Option<u32>,
+	extract: Extract,
+	compaction_profile: &CompactionProfile) -> Result<(), Error> {
+	fn db_error(e: String) -> Error {
+		warn!("Cannot open Database for consolidation: {:?}", e);
+		Error::MigrationFailed
+	}
 
-/// Migrations on the state database.
-fn state_database_migrations(pruning: Algorithm, compaction_profile: CompactionProfile) -> Result<MigrationManager, Error> {
-	let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
-	let res = match pruning {
-		Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7::default()),
-		Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()),
-		_ => return Err(Error::UnsuportedPruningMethod),
+	let mut migration = migrations::ToV9::new(column, extract);
+	let config = default_migration_settings(compaction_profile);
+	let mut db_config = DatabaseConfig {
+		max_open_files: 64,
+		cache_size: None,
+		compaction: config.compaction_profile.clone(),
+		columns: None,
 	};
-	try!(res.map_err(|_| Error::MigrationImpossible));
 
-	Ok(manager)
+	let old_path_str = try!(old_db_path.to_str().ok_or(Error::MigrationImpossible));
+	let new_path_str = try!(new_db_path.to_str().ok_or(Error::MigrationImpossible));
+
+	let cur_db = try!(Database::open(&db_config, old_path_str).map_err(db_error));
+	// open new DB with proper number of columns
+	db_config.columns = migration.columns();
+	let mut new_db = try!(Database::open(&db_config, new_path_str).map_err(db_error));
+
+	// Migrate to new database (default column only)
+	try!(migration.migrate(&cur_db, &config, &mut new_db, None));
+
+	Ok(())
 }
 
+
 /// Migrates database at given position with given migration rules.
 fn migrate_database(version: u32, db_path: PathBuf, mut migrations: MigrationManager) -> Result<(), Error> {
 	// check if migration is needed
@@ -216,17 +221,108 @@ pub fn migrate(path: &Path, pruning: Algorithm, compaction_profile: CompactionPr
 
 	// migrate the databases.
 	// main db directory may already exists, so let's check if we have blocks dir
-	if version < CURRENT_VERSION && exists(&blocks_database_path(path)) {
-		println!("Migrating database from version {} to {}", version, CURRENT_VERSION);
-		try!(migrate_database(version, blocks_database_path(path), try!(blocks_database_migrations(compaction_profile.clone()))));
-		try!(migrate_database(version, extras_database_path(path), try!(extras_database_migrations(compaction_profile.clone()))));
-		try!(migrate_database(version, state_database_path(path), try!(state_database_migrations(pruning, compaction_profile))));
-		println!("Migration finished");
-	} else if version > CURRENT_VERSION {
+	if version > CURRENT_VERSION {
 		return Err(Error::FutureDBVersion);
 	}
 
+	// We are in the latest version, yay!
+	if version == CURRENT_VERSION {
+		return Ok(())
+	}
+
+	// Perform pre-consolidation migrations
+	if version < CONSOLIDATION_VERSION && exists(&legacy::blocks_database_path(path)) {
+		println!("Migrating database from version {} to {}", version, CONSOLIDATION_VERSION);
+		try!(migrate_database(version, legacy::blocks_database_path(path), try!(legacy::blocks_database_migrations(&compaction_profile))));
+		try!(migrate_database(version, legacy::extras_database_path(path), try!(legacy::extras_database_migrations(&compaction_profile))));
+		try!(migrate_database(version, legacy::state_database_path(path), try!(legacy::state_database_migrations(pruning, &compaction_profile))));
+		let db_path = consolidated_database_path(path);
+		// Remove the database dir (it shouldn't exist anyway, but it might when migration was interrupted)
+		let _ = fs::remove_dir_all(db_path.clone());
+		try!(consolidate_database(legacy::blocks_database_path(path), db_path.clone(), client::DB_COL_HEADERS, Extract::Header, &compaction_profile));
+		try!(consolidate_database(legacy::blocks_database_path(path), db_path.clone(), client::DB_COL_BODIES, Extract::Header, &compaction_profile));
+		try!(consolidate_database(legacy::extras_database_path(path), db_path.clone(), client::DB_COL_EXTRA, Extract::All, &compaction_profile));
+		try!(consolidate_database(legacy::state_database_path(path), db_path.clone(), client::DB_COL_STATE, Extract::All, &compaction_profile));
+		try!(consolidate_database(legacy::trace_database_path(path), db_path.clone(), client::DB_COL_TRACE, Extract::All, &compaction_profile));
+		let _ = fs::remove_dir_all(legacy::blocks_database_path(path));
+		let _ = fs::remove_dir_all(legacy::extras_database_path(path));
+		let _ = fs::remove_dir_all(legacy::state_database_path(path));
+		let _ = fs::remove_dir_all(legacy::trace_database_path(path));
+		println!("Migration finished");
+	}
+
+	// Further migrations
+	if version >= CONSOLIDATION_VERSION && version < CURRENT_VERSION && exists(&consolidated_database_path(path)) {
+		println!("Migrating database from version {} to {}", ::std::cmp::max(CONSOLIDATION_VERSION, version), CURRENT_VERSION);
+		try!(migrate_database(version, consolidated_database_path(path), try!(consolidated_database_migrations(&compaction_profile))));
+		println!("Migration finished");
+	}
+
 	// update version file.
 	update_version(path)
 }
 
+/// Old migrations utilities
+mod legacy {
+	use super::*;
+	use std::path::{Path, PathBuf};
+	use util::journaldb::Algorithm;
+	use util::migration::{Manager as MigrationManager};
+	use util::kvdb::CompactionProfile;
+	use ethcore::migrations;
+
+	/// Blocks database path.
+	pub fn blocks_database_path(path: &Path) -> PathBuf {
+		let mut blocks_path = path.to_owned();
+		blocks_path.push("blocks");
+		blocks_path
+	}
+
+	/// Extras database path.
+	pub fn extras_database_path(path: &Path) -> PathBuf {
+		let mut extras_path = path.to_owned();
+		extras_path.push("extras");
+		extras_path
+	}
+
+	/// State database path.
+	pub fn state_database_path(path: &Path) -> PathBuf {
+		let mut state_path = path.to_owned();
+		state_path.push("state");
+		state_path
+	}
+
+	/// Trace database path.
+	pub fn trace_database_path(path: &Path) -> PathBuf {
+		let mut blocks_path = path.to_owned();
+		blocks_path.push("tracedb");
+		blocks_path
+	}
+
+	/// Migrations on the blocks database.
+	pub fn blocks_database_migrations(compaction_profile: &CompactionProfile) -> Result<MigrationManager, Error> {
+		let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
+		try!(manager.add_migration(migrations::blocks::V8::default()).map_err(|_| Error::MigrationImpossible));
+		Ok(manager)
+	}
+
+	/// Migrations on the extras database.
+	pub fn extras_database_migrations(compaction_profile: &CompactionProfile) -> Result<MigrationManager, Error> {
+		let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
+		try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible));
+		Ok(manager)
+	}
+
+	/// Migrations on the state database.
+	pub fn state_database_migrations(pruning: Algorithm, compaction_profile: &CompactionProfile) -> Result<MigrationManager, Error> {
+		let mut manager = MigrationManager::new(default_migration_settings(compaction_profile));
+		let res = match pruning {
+			Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7::default()),
+			Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()),
+			_ => return Err(Error::UnsuportedPruningMethod),
+		};
+
+		try!(res.map_err(|_| Error::MigrationImpossible));
+		Ok(manager)
+	}
+}

parity/params.rs

@@ -15,9 +15,11 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::str::FromStr;
+use std::sync::Arc;
 use std::time::Duration;
-use util::{contents, DatabaseConfig, journaldb, H256, Address, U256, version_data};
+use util::{contents, Database, DatabaseConfig, journaldb, H256, Address, U256, version_data};
 use util::journaldb::Algorithm;
+use ethcore::client;
 use ethcore::spec::Spec;
 use ethcore::ethereum;
 use ethcore::miner::{GasPricer, GasPriceCalibratorOptions};
@@ -103,11 +105,15 @@ impl Pruning {
 		algo_types.push(Algorithm::default());
 
 		algo_types.into_iter().max_by_key(|i| {
-			let mut client_path = dirs.client_path(genesis_hash, fork_name, *i);
-			client_path.push("state");
-			let db = journaldb::new(client_path.to_str().unwrap(), *i, DatabaseConfig::default());
+			let client_path = dirs.client_path(genesis_hash, fork_name, *i);
+			let config = DatabaseConfig::with_columns(client::DB_NO_OF_COLUMNS);
+			let db = match Database::open(&config, client_path.to_str().unwrap()) {
+				Ok(db) => db,
+				Err(_) => return 0,
+			};
+			let db = journaldb::new(Arc::new(db), *i, client::DB_COL_STATE);
 			trace!(target: "parity", "Looking for best DB: {} at {:?}", i, db.latest_era());
-			db.latest_era()
+			db.latest_era().unwrap_or(0)
 		}).unwrap()
 	}
 }