Bloomchain (#1014)

* use bloomchain crate in blockchain module. remove obsole chainfilter submodule

* update database version to 6.0

* removed redundant line

* simple db migration

* make migration slightly more functional

* bloomchain migration

* migration version is just a single unsigned integer

* updated migration v6

* parity migration

* db migration

* removed hardcoded migration dir

* replace ptr::copy with clone_from_slice, removed potential endianess problem from trace/db.rs

* removed superfluous line

* blockchains log blooms config is not exposed any more

parity/main.rs

@@ -62,20 +62,28 @@ mod informant;
 mod io_handler;
 mod cli;
 mod configuration;
+mod migration;
 
-use ctrlc::CtrlC;
-use util::*;
-use std::time::Duration;
+use std::io::{Write, Read, BufReader, BufRead};
+use std::ops::Deref;
+use std::sync::{Arc, Mutex, Condvar};
+use std::path::Path;
 use std::fs::File;
+use std::str::{FromStr, from_utf8};
 use std::thread::sleep;
-use std::io::{BufReader, BufRead};
+use std::time::Duration;
+use rustc_serialize::hex::FromHex;
+use ctrlc::CtrlC;
+use util::{H256, ToPretty, NetworkConfiguration, PayloadInfo, Bytes};
 use util::panics::{MayPanic, ForwardPanic, PanicHandler};
-use ethcore::client::{BlockID, BlockChainClient};
+use ethcore::client::{BlockID, BlockChainClient, ClientConfig, get_db_path};
 use ethcore::error::{Error, ImportError};
 use ethcore::service::ClientService;
+use ethcore::spec::Spec;
 use ethsync::EthSync;
 use ethminer::{Miner, MinerService, ExternalMiner};
 use daemonize::Daemonize;
+use migration::migrate;
 use informant::Informant;
 
 use die::*;
@@ -96,7 +104,10 @@ fn execute(conf: Configuration) {
 		return;
 	}
 
-	execute_upgrades(&conf);
+	let spec = conf.spec();
+	let client_config = conf.client_config(&spec);
+
+	execute_upgrades(&conf, &spec, &client_config);
 
 	if conf.args.cmd_daemon {
 		Daemonize::new()
@@ -121,10 +132,10 @@ fn execute(conf: Configuration) {
 		return;
 	}
 
-	execute_client(conf);
+	execute_client(conf, spec, client_config);
 }
 
-fn execute_upgrades(conf: &Configuration) {
+fn execute_upgrades(conf: &Configuration, spec: &Spec, client_config: &ClientConfig) {
 	match ::upgrade::upgrade(Some(&conf.path())) {
 		Ok(upgrades_applied) if upgrades_applied > 0 => {
 			println!("Executed {} upgrade scripts - ok", upgrades_applied);
@@ -134,9 +145,15 @@ fn execute_upgrades(conf: &Configuration) {
 		},
 		_ => {},
 	}
+
+	let db_path = get_db_path(Path::new(&conf.path()), client_config.pruning, spec.genesis_header().hash());
+	let result = migrate(&db_path);
+	if let Err(err) = result {
+		die_with_message(&format!("{}", err));
+	}
 }
 
-fn execute_client(conf: Configuration) {
+fn execute_client(conf: Configuration, spec: Spec, client_config: ClientConfig) {
 	// Setup panic handler
 	let panic_handler = PanicHandler::new_in_arc();
 
@@ -145,10 +162,8 @@ fn execute_client(conf: Configuration) {
 	// Raise fdlimit
 	unsafe { ::fdlimit::raise_fd_limit(); }
 
-	let spec = conf.spec();
 	let net_settings = conf.net_settings(&spec);
 	let sync_config = conf.sync_config(&spec);
-	let client_config = conf.client_config(&spec);
 
 	// Secret Store
 	let account_service = Arc::new(conf.account_service());
@@ -399,7 +414,7 @@ fn execute_import(conf: Configuration) {
 		DataFormat::Hex => {
 			for line in BufReader::new(instream).lines() {
 				let s = line.unwrap_or_else(|_| die!("Error reading from the file/stream."));
-				let s = if first_read > 0 {str::from_utf8(&first_bytes).unwrap().to_owned() + &(s[..])} else {s};
+				let s = if first_read > 0 {from_utf8(&first_bytes).unwrap().to_owned() + &(s[..])} else {s};
 				first_read = 0;
 				let bytes = FromHex::from_hex(&(s[..])).unwrap_or_else(|_| die!("Invalid hex in file/stream."));
 				do_import(bytes);

parity/migration.rs

@@ -0,0 +1,186 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fs, env};
+use std::fs::File;
+use std::io::{Read, Write, Error as IoError, ErrorKind};
+use std::path::PathBuf;
+use std::fmt::{Display, Formatter, Error as FmtError};
+use util::migration::{Manager as MigrationManager, Config as MigrationConfig, MigrationIterator};
+use util::kvdb::Database;
+use ethcore::migrations;
+
+/// Database is assumed to be at default version, when no version file is found.
+const DEFAULT_VERSION: u32 = 5;
+/// Current version of database models.
+const CURRENT_VERSION: u32 = 6;
+/// Defines how many items are migrated to the new version of database at once.
+const BATCH_SIZE: usize = 1024;
+/// Version file name.
+const VERSION_FILE_NAME: &'static str = "db_version";
+
+/// Migration related erorrs.
+#[derive(Debug)]
+pub enum Error {
+	/// Returned when current version cannot be read or guessed.
+	UnknownDatabaseVersion,
+	/// Returned when migration is not possible.
+	MigrationImpossible,
+	/// Returned when migration unexpectadly failed.
+	MigrationFailed,
+	/// Returned when migration was completed succesfully,
+	/// but there was a problem with io.
+	Io(IoError),
+}
+
+impl Display for Error {
+	fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
+		let out = match *self {
+			Error::UnknownDatabaseVersion => "Current database version cannot be read".into(),
+			Error::MigrationImpossible => format!("Migration to version {} is not possible", CURRENT_VERSION),
+			Error::MigrationFailed => "Migration unexpectedly failed".into(),
+			Error::Io(ref err) => format!("Unexpected io error: {}", err),
+		};
+
+		write!(f, "{}", out)
+	}
+}
+
+impl From<IoError> for Error {
+	fn from(err: IoError) -> Self {
+		Error::Io(err)
+	}
+}
+
+/// Returns the version file path.
+fn version_file_path(path: &PathBuf) -> PathBuf {
+	let mut file_path = path.clone();
+	file_path.push(VERSION_FILE_NAME);
+	file_path
+}
+
+/// Reads current database version from the file at given path.
+/// If the file does not exist returns DEFAULT_VERSION.
+fn current_version(path: &PathBuf) -> Result<u32, Error> {
+	match File::open(version_file_path(path)) {
+		Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(DEFAULT_VERSION),
+		Err(_) => Err(Error::UnknownDatabaseVersion),
+		Ok(mut file) => {
+			let mut s = String::new();
+			try!(file.read_to_string(&mut s).map_err(|_| Error::UnknownDatabaseVersion));
+			u32::from_str_radix(&s, 10).map_err(|_| Error::UnknownDatabaseVersion)
+		},
+	}
+}
+
+/// Writes current database version to the file.
+/// Creates a new file if the version file does not exist yet.
+fn update_version(path: &PathBuf) -> Result<(), Error> {
+	let mut file = try!(File::create(version_file_path(path)));
+	try!(file.write_all(format!("{}", CURRENT_VERSION).as_bytes()));
+	Ok(())
+}
+
+/// Blocks database path.
+fn blocks_database_path(path: &PathBuf) -> PathBuf {
+	let mut blocks_path = path.clone();
+	blocks_path.push("blocks");
+	blocks_path
+}
+
+/// Extras database path.
+fn extras_database_path(path: &PathBuf) -> PathBuf {
+	let mut extras_path = path.clone();
+	extras_path.push("extras");
+	extras_path
+}
+
+/// Temporary database path used for migration.
+fn temp_database_path() -> PathBuf {
+	let mut dir = env::temp_dir();
+	dir.push("parity_migration");
+	dir
+}
+
+/// Default migration settings.
+fn default_migration_settings() -> MigrationConfig {
+	MigrationConfig {
+		batch_size: BATCH_SIZE,
+	}
+}
+
+/// Migrations on blocks database.
+fn blocks_database_migrations() -> Result<MigrationManager, Error> {
+	let manager = MigrationManager::new(default_migration_settings());
+	Ok(manager)
+}
+
+/// Migrations on extras database.
+fn extras_database_migrations() -> Result<MigrationManager, Error> {
+	let mut manager = MigrationManager::new(default_migration_settings());
+	try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible));
+	Ok(manager)
+}
+
+/// Migrates database at given position with given migration rules.
+fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -> Result<(), Error> {
+	// check if migration is needed
+	if !migrations.is_needed(version) {
+		return Ok(())
+	}
+
+	println!("Migrating database {} from version {} to {}", path.to_string_lossy(), version, CURRENT_VERSION);
+
+	// get temp path
+	let temp_path = temp_database_path();
+	// remote the dir if it exists
+	let _ = fs::remove_dir_all(&temp_path);
+
+	{
+		// open old database
+		let old = try!(Database::open_default(path.to_str().unwrap()).map_err(|_| Error::MigrationFailed));
+
+		// create new database
+		let mut temp = try!(Database::open_default(temp_path.to_str().unwrap()).map_err(|_| Error::MigrationFailed));
+
+		// migrate old database to the new one
+		try!(migrations.execute(MigrationIterator::from(old.iter()), version, &mut temp).map_err(|_| Error::MigrationFailed));
+	}
+
+	// replace the old database with the new one
+	try!(fs::remove_dir_all(&path));
+	try!(fs::rename(&temp_path, &path));
+
+	println!("Migration finished");
+
+	Ok(())
+}
+
+/// Migrates the database.
+pub fn migrate(path: &PathBuf) -> Result<(), Error> {
+	// read version file.
+	let version = try!(current_version(path));
+
+	// migrate the databases.
+	if version != CURRENT_VERSION {
+		try!(migrate_database(version, blocks_database_path(path), try!(blocks_database_migrations())));
+		try!(migrate_database(version, extras_database_path(path), try!(extras_database_migrations())));
+	}
+
+	// update version file.
+	update_version(path)
+}
+