have AccountDB use address hash for uniqueness (#1533)

* partially done alternate migration scheme

* finish altering migration framework

* migrate tests to new migration framework

* address comments

* remove superfluous newline
[ci skip]

* TempIdx -> TempIndex
[ci skip]

* modify account_db to work on address hash, not address

* add a database migration for new accountdb

* preserve first 96 bits of keys when combining

* handle metadata keys in migration and preserve first 96 bits

* fix comments and hash address instead of hash

* different migrations based on pruning

* migrations mutably borrow self

* batch abstraction for migration

* added missing licence headers

* overlay recent v7 migration

* better error handling, migrate version key as well

* fix migration tests

* commit final batch and migrate journaled insertions

* two passes on journal to migrate all possible deleted keys

parity/configuration.rs

@@ -295,7 +295,7 @@ impl Configuration {
 		ret
 	}
 
-	pub fn find_best_db(&self, spec: &Spec) -> Option<journaldb::Algorithm> {
+	fn find_best_db(&self, spec: &Spec) -> Option<journaldb::Algorithm> {
 		let mut ret = None;
 		let mut latest_era = None;
 		let jdb_types = [journaldb::Algorithm::Archive, journaldb::Algorithm::EarlyMerge, journaldb::Algorithm::OverlayRecent, journaldb::Algorithm::RefCounted];
@@ -314,6 +314,17 @@ impl Configuration {
 		ret
 	}
 
+	pub fn pruning_algorithm(&self, spec: &Spec) -> journaldb::Algorithm {
+		match self.args.flag_pruning.as_str() {
+			"archive" => journaldb::Algorithm::Archive,
+			"light" => journaldb::Algorithm::EarlyMerge,
+			"fast" => journaldb::Algorithm::OverlayRecent,
+			"basic" => journaldb::Algorithm::RefCounted,
+			"auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent),
+			_ => { die!("Invalid pruning method given."); }
+		}
+	}
+
 	pub fn client_config(&self, spec: &Spec) -> ClientConfig {
 		let mut client_config = ClientConfig::default();
 
@@ -341,14 +352,7 @@ impl Configuration {
 		// forced trace db cache size if provided
 		client_config.tracing.db_cache_size = self.args.flag_db_cache_size.and_then(|cs| Some(cs / 4));
 
-		client_config.pruning = match self.args.flag_pruning.as_str() {
-			"archive" => journaldb::Algorithm::Archive,
-			"light" => journaldb::Algorithm::EarlyMerge,
-			"fast" => journaldb::Algorithm::OverlayRecent,
-			"basic" => journaldb::Algorithm::RefCounted,
-			"auto" => self.find_best_db(spec).unwrap_or(journaldb::Algorithm::OverlayRecent),
-			_ => { die!("Invalid pruning method given."); }
-		};
+		client_config.pruning = self.pruning_algorithm(spec);
 
 		if self.args.flag_fat_db {
 			if let journaldb::Algorithm::Archive = client_config.pruning {

parity/die.rs

@@ -22,7 +22,7 @@ use std::process::exit;
 
 #[macro_export]
 macro_rules! die {
-	($($arg:tt)*) => (die_with_message(&format!("{}", format_args!($($arg)*))));
+	($($arg:tt)*) => (::die::die_with_message(&format!("{}", format_args!($($arg)*))));
 }
 
 pub fn die_with_error(module: &'static str, e: ethcore::error::Error) -> ! {

parity/main.rs

@@ -174,7 +174,7 @@ fn execute_upgrades(conf: &Configuration, spec: &Spec, client_config: &ClientCon
 	}
 
 	let db_path = get_db_path(Path::new(&conf.path()), client_config.pruning, spec.genesis_header().hash());
-	let result = migrate(&db_path);
+	let result = migrate(&db_path, client_config.pruning);
 	if let Err(err) = result {
 		die_with_message(&format!("{}", err));
 	}

parity/migration.rs

@@ -17,15 +17,16 @@
 use std::fs;
 use std::fs::File;
 use std::io::{Read, Write, Error as IoError, ErrorKind};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::fmt::{Display, Formatter, Error as FmtError};
+use util::journaldb::Algorithm;
 use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError};
 use ethcore::migrations;
 
 /// Database is assumed to be at default version, when no version file is found.
 const DEFAULT_VERSION: u32 = 5;
 /// Current version of database models.
-const CURRENT_VERSION: u32 = 6;
+const CURRENT_VERSION: u32 = 7;
 /// Defines how many items are migrated to the new version of database at once.
 const BATCH_SIZE: usize = 1024;
 /// Version file name.
@@ -74,15 +75,15 @@ impl From<MigrationError> for Error {
 }
 
 /// Returns the version file path.
-fn version_file_path(path: &PathBuf) -> PathBuf {
-	let mut file_path = path.clone();
+fn version_file_path(path: &Path) -> PathBuf {
+	let mut file_path = path.to_owned();
 	file_path.push(VERSION_FILE_NAME);
 	file_path
 }
 
 /// Reads current database version from the file at given path.
 /// If the file does not exist returns `DEFAULT_VERSION`.
-fn current_version(path: &PathBuf) -> Result<u32, Error> {
+fn current_version(path: &Path) -> Result<u32, Error> {
 	match File::open(version_file_path(path)) {
 		Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(DEFAULT_VERSION),
 		Err(_) => Err(Error::UnknownDatabaseVersion),
@@ -96,7 +97,7 @@ fn current_version(path: &PathBuf) -> Result<u32, Error> {
 
 /// Writes current database version to the file.
 /// Creates a new file if the version file does not exist yet.
-fn update_version(path: &PathBuf) -> Result<(), Error> {
+fn update_version(path: &Path) -> Result<(), Error> {
 	try!(fs::create_dir_all(path));
 	let mut file = try!(File::create(version_file_path(path)));
 	try!(file.write_all(format!("{}", CURRENT_VERSION).as_bytes()));
@@ -104,22 +105,29 @@ fn update_version(path: &PathBuf) -> Result<(), Error> {
 }
 
 /// Blocks database path.
-fn blocks_database_path(path: &PathBuf) -> PathBuf {
-	let mut blocks_path = path.clone();
+fn blocks_database_path(path: &Path) -> PathBuf {
+	let mut blocks_path = path.to_owned();
 	blocks_path.push("blocks");
 	blocks_path
 }
 
 /// Extras database path.
-fn extras_database_path(path: &PathBuf) -> PathBuf {
-	let mut extras_path = path.clone();
+fn extras_database_path(path: &Path) -> PathBuf {
+	let mut extras_path = path.to_owned();
 	extras_path.push("extras");
 	extras_path
 }
 
+/// State database path.
+fn state_database_path(path: &Path) -> PathBuf {
+	let mut state_path = path.to_owned();
+	state_path.push("state");
+	state_path
+}
+
 /// Database backup
-fn backup_database_path(path: &PathBuf) -> PathBuf {
-	let mut backup_path = path.clone();
+fn backup_database_path(path: &Path) -> PathBuf {
+	let mut backup_path = path.to_owned();
 	backup_path.pop();
 	backup_path.push("temp_backup");
 	backup_path
@@ -132,21 +140,34 @@ fn default_migration_settings() -> MigrationConfig {
 	}
 }
 
-/// Migrations on blocks database.
+/// Migrations on the blocks database.
 fn blocks_database_migrations() -> Result<MigrationManager, Error> {
 	let manager = MigrationManager::new(default_migration_settings());
 	Ok(manager)
 }
 
-/// Migrations on extras database.
+/// Migrations on the extras database.
 fn extras_database_migrations() -> Result<MigrationManager, Error> {
 	let mut manager = MigrationManager::new(default_migration_settings());
 	try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible));
 	Ok(manager)
 }
 
+/// Migrations on the state database.
+fn state_database_migrations(pruning: Algorithm) -> Result<MigrationManager, Error> {
+	let mut manager = MigrationManager::new(default_migration_settings());
+	let res = match pruning {
+		Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7),
+		Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()),
+		_ => die!("Unsupported pruning method for migration. Delete DB and resync"),
+	};
+
+	try!(res.map_err(|_| Error::MigrationImpossible));
+	Ok(manager)
+}
+
 /// Migrates database at given position with given migration rules.
-fn migrate_database(version: u32, db_path: PathBuf, migrations: MigrationManager) -> Result<(), Error> {
+fn migrate_database(version: u32, db_path: PathBuf, mut migrations: MigrationManager) -> Result<(), Error> {
 	// check if migration is needed
 	if !migrations.is_needed(version) {
 		return Ok(())
@@ -175,12 +196,12 @@ fn migrate_database(version: u32, db_path: PathBuf, migrations: MigrationManager
 	Ok(())
 }
 
-fn exists(path: &PathBuf) -> bool {
+fn exists(path: &Path) -> bool {
 	fs::metadata(path).is_ok()
 }
 
 /// Migrates the database.
-pub fn migrate(path: &PathBuf) -> Result<(), Error> {
+pub fn migrate(path: &Path, pruning: Algorithm) -> Result<(), Error> {
 	// read version file.
 	let version = try!(current_version(path));
 
@@ -190,6 +211,7 @@ pub fn migrate(path: &PathBuf) -> Result<(), Error> {
 		println!("Migrating database from version {} to {}", version, CURRENT_VERSION);
 		try!(migrate_database(version, blocks_database_path(path), try!(blocks_database_migrations())));
 		try!(migrate_database(version, extras_database_path(path), try!(extras_database_migrations())));
+		try!(migrate_database(version, state_database_path(path), try!(state_database_migrations(pruning))));
 		println!("Migration finished");
 	}
 

parity/rpc_apis.rs

@@ -18,7 +18,6 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::sync::Arc;
 
-use die::*;
 use ethsync::EthSync;
 use ethcore::miner::{Miner, ExternalMiner};
 use ethcore::client::Client;