diff --git a/ethcore/src/migrations/extras/v6.rs b/ethcore/src/migrations/extras/v6.rs index 9b97e594b..3b8160e67 100644 --- a/ethcore/src/migrations/extras/v6.rs +++ b/ethcore/src/migrations/extras/v6.rs @@ -1,4 +1,4 @@ -use util::migration::Migration; +use util::migration::SimpleMigration; /// This migration reduces the sizes of keys and moves `ExtrasIndex` byte from back to the front. pub struct ToV6; @@ -17,7 +17,7 @@ impl ToV6 { } } -impl Migration for ToV6 { +impl SimpleMigration for ToV6 { fn version(&self) -> u32 { 6 } diff --git a/parity/migration.rs b/parity/migration.rs index df2c2116f..b7cdecb16 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -19,7 +19,7 @@ use std::fs::File; use std::io::{Read, Write, Error as IoError, ErrorKind}; use std::path::PathBuf; use std::fmt::{Display, Formatter, Error as FmtError}; -use util::migration::{Manager as MigrationManager, Config as MigrationConfig, MigrationIterator}; +use util::migration::{Manager as MigrationManager, Config as MigrationConfig, Error as MigrationError}; use util::kvdb::{Database, DatabaseConfig, CompactionProfile}; use ethcore::migrations; @@ -65,6 +65,16 @@ impl From for Error { } } +impl From for Error { + fn from(err: MigrationError) -> Self { + match err { + MigrationError::Io(e) => Error::Io(e), + _ => Error::MigrationFailed, + + } + } +} + /// Returns the version file path. fn version_file_path(path: &PathBuf) -> PathBuf { let mut file_path = path.clone(); @@ -109,14 +119,6 @@ fn extras_database_path(path: &PathBuf) -> PathBuf { extras_path } -/// Temporary database path used for migration. -fn temp_database_path(path: &PathBuf) -> PathBuf { - let mut temp_path = path.clone(); - temp_path.pop(); - temp_path.push("temp_migration"); - temp_path -} - /// Database backup fn backup_database_path(path: &PathBuf) -> PathBuf { let mut backup_path = path.clone(); @@ -152,29 +154,13 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) - return Ok(()) } - let temp_path = temp_database_path(&path); let backup_path = backup_database_path(&path); - // remote the dir if it exists + // remove the dir if it exists let _ = fs::remove_dir_all(&temp_path); let _ = fs::remove_dir_all(&backup_path); - { - let db_config = DatabaseConfig { - prefix_size: None, - max_open_files: 64, - cache_size: None, - compaction: CompactionProfile::default(), - }; - - // open old database - let old = try!(Database::open(&db_config, path.to_str().unwrap()).map_err(|_| Error::MigrationFailed)); - - // create new database - let mut temp = try!(Database::open(&db_config, temp_path.to_str().unwrap()).map_err(|_| Error::MigrationFailed)); - - // migrate old database to the new one - try!(migrations.execute(MigrationIterator::from(old.iter()), version, &mut temp).map_err(|_| Error::MigrationFailed)); - } + // migrate old database to the new one + let temp_path = try!(migrations.execute(version)); // create backup try!(fs::rename(&path, &backup_path)); @@ -183,7 +169,7 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) - if let Err(err) = fs::rename(&temp_path, &path) { // if something went wrong, bring back backup try!(fs::rename(&backup_path, path)); - return Err(From::from(err)); + return Err(err.into()); } // remove backup diff --git a/util/src/migration/db_impl.rs b/util/src/migration/db_impl.rs deleted file mode 100644 index 9adf22a4a..000000000 --- a/util/src/migration/db_impl.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! `kvdb::Database` as `migration::Destination` - -use std::collections::BTreeMap; -use kvdb::{Database, DatabaseIterator, DBTransaction}; -use migration::{Destination, Error}; - -/// Database iterator with `Item` complient with migration `Manager` interface. -pub struct MigrationIterator { - iter: DatabaseIterator, -} - -impl From for MigrationIterator { - fn from(iter: DatabaseIterator) -> Self { - MigrationIterator { - iter: iter - } - } -} - -impl Iterator for MigrationIterator { - type Item = (Vec, Vec); - - fn next(&mut self) -> Option { - self.iter.next().map(|(k, v)| (k.to_vec(), v.to_vec())) - } -} - -impl Destination for Database { - fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error> { - let transaction = DBTransaction::new(); - - for keypair in &batch { - try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom)) - } - - self.write(transaction).map_err(Error::Custom) - } -} - diff --git a/util/src/migration/manager.rs b/util/src/migration/manager.rs deleted file mode 100644 index bd21d1637..000000000 --- a/util/src/migration/manager.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Migration manager - -use std::collections::BTreeMap; -use migration::{Migration, Destination}; - -/// Migration error. -#[derive(Debug)] -pub enum Error { - /// Error returned when it is impossible to add new migration rules. - CannotAddMigration, - /// Error returned when migration from specific version can not be performed. - MigrationImpossible, - /// Custom error. - Custom(String), -} - -/// Migration config. -pub struct Config { - /// Defines how many elements should be migrated at once. - pub batch_size: usize, -} - -impl Default for Config { - fn default() -> Self { - Config { - batch_size: 1024, - } - } -} - -/// Manages database migration. -pub struct Manager { - config: Config, - migrations: Vec>, -} - -impl Manager { - /// Creates new migration manager with given configuration. - pub fn new(config: Config) -> Self { - Manager { - config: config, - migrations: vec![] - } - } - - /// Adds new migration rules. - pub fn add_migration(&mut self, migration: T) -> Result<(), Error> where T: Migration { - let version_match = match self.migrations.last() { - Some(last) => last.version() + 1 == migration.version(), - None => true, - }; - - match version_match { - true => Ok(self.migrations.push(Box::new(migration))), - false => Err(Error::CannotAddMigration), - } - } - - /// Performs migration to destination. - pub fn execute(&self, db_iter: D, version: u32, destination: &mut Destination) -> Result<(), Error> where - D: Iterator, Vec)> { - - let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible)); - - let mut batch: BTreeMap, Vec> = BTreeMap::new(); - - for keypair in db_iter { - let migrated = migrations.iter().fold(Some(keypair), |migrated, migration| { - migrated.and_then(|(key, value)| migration.simple_migrate(key, value)) - }); - - if let Some((key, value)) = migrated { - batch.insert(key, value); - } - - if batch.len() == self.config.batch_size { - try!(destination.commit(batch)); - batch = BTreeMap::new(); - } - } - - try!(destination.commit(batch)); - - Ok(()) - } - - /// Returns true if migration is needed. - pub fn is_needed(&self, version: u32) -> bool { - match self.migrations.last() { - Some(last) => version < last.version(), - None => false, - } - } - - fn migrations_from(&self, version: u32) -> Option<&[Box]> { - // index of the first required migration - let position = self.migrations.iter().position(|m| m.version() == version + 1); - position.map(|p| &self.migrations[p..]) - } -} - diff --git a/util/src/migration/mod.rs b/util/src/migration/mod.rs index 83d42d098..fdf771050 100644 --- a/util/src/migration/mod.rs +++ b/util/src/migration/mod.rs @@ -15,19 +15,97 @@ // along with Parity. If not, see . //! DB Migration module. - -mod db_impl; -mod manager; - #[cfg(test)] mod tests; -pub use self::manager::{Error, Config, Manager}; -pub use self::db_impl::MigrationIterator; use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; -/// Single migration. +use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction}; + +/// Migration config. +pub struct Config { + /// Defines how many elements should be migrated at once. + pub batch_size: usize, +} + +impl Default for Config { + fn default() -> Self { + Config { + batch_size: 1024, + } + } +} + +/// Migration error. +#[derive(Debug)] +pub enum Error { + /// Error returned when it is impossible to add new migration rules. + CannotAddMigration, + /// Error returned when migration from specific version can not be performed. + MigrationImpossible, + /// Io Error. + Io(::std::io::Error), + /// Custom error. + Custom(String), +} + +impl From<::std::io::Error> for Error { + fn from(e: ::std::io::Error) -> Self { + Error::Io(e) + } +} + +/// Migration source. +pub trait Source { + /// Get an iterator over the key, value pairs. + fn iter(&self) -> Box, Vec)>>; + + /// Query a specific key. + fn get(&self, key: &[u8]) -> Option>; +} + +impl Source for Database { + fn iter(&self) -> Box, Vec)>> { + Box::new(self.iter().map(|(k, v)| (k.to_vec(), v.to_vec()))) + } + + fn get(&self, key: &[u8]) -> Option> { + self.get(key).ok().and_then(|x| x).map(|x| x.to_owned()) + } +} + + + +/// Object being migrated +pub trait Migrateable { + /// Called on destination to commit batch of migrated entries. + fn commit(&mut self, batch: &BTreeMap, Vec>) -> Result<(), Error>; +} + +impl Migrateable for Database { + fn commit(&mut self, batch: &BTreeMap, Vec>) -> Result<(), Error> { + let transaction = DBTransaction::new(); + + for keypair in batch { + try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom)) + } + + self.write(transaction).map_err(Error::Custom) + } +} + +/// A generalized migration from the given db to a destination db. pub trait Migration: 'static { + /// Version of the database after the migration. + fn version(&self) -> u32; + /// Migrate a source to a destination. + fn migrate(&self, source: &Source, config: &Config, destination: &mut Destination) -> Result<(), Error>; +} + +/// A simple migration over key-value pairs. +pub trait SimpleMigration: 'static { /// Version of database after the migration. fn version(&self) -> u32; /// Should migrate existing object to new database. @@ -35,8 +113,140 @@ pub trait Migration: 'static { fn simple_migrate(&self, key: Vec, value: Vec) -> Option<(Vec, Vec)>; } -/// Migration destination. -pub trait Destination { - /// Called on destination to commit batch of migrated entries. - fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error>; +impl Migration for T { + fn version(&self) -> u32 { SimpleMigration::version(self) } + + fn migrate(&self, source: &Source, config: &Config, destination: &mut Destination) -> Result<(), Error> { + let mut batch: BTreeMap, Vec> = BTreeMap::new(); + + for (key, value) in source.iter() { + + if let Some((key, value)) = self.simple_migrate(key.to_vec(), value.to_vec()) { + batch.insert(key, value); + } + + if batch.len() == config.batch_size { + try!(destination.commit(&batch)); + batch.clear(); + } + } + + try!(destination.commit(&batch)); + + Ok(()) + } } + +/// Get the path where all databases reside. +fn database_path(path: &Path) -> PathBuf { + let mut temp_path = path.to_owned(); + temp_path.pop(); + temp_path +} + +// which temp database we are currently using +enum TempDB { + One, + Two, +} + +impl TempDB { + // given the path that all databases reside in, + // return the path of this one. + fn path(&self, db_path: &Path) -> PathBuf { + let mut path = db_path.to_owned(); + match *self { + TempDB::One => path.push("migration_1"), + TempDB::Two => path.push("migration_2"), + } + + path + } + + fn swap(&mut self) { + *self = match *self { + TempDB::One => TempDB::Two, + TempDB::Two => TempDB::One, + }; + } +} + +/// Manages database migration. +pub struct Manager { + config: Config, + migrations: Vec>, +} + +impl Manager { + /// Creates new migration manager with given configuration. + pub fn new(config: Config) -> Self { + Manager { + config: config, + migrations: vec![], + } + } + + /// Adds new migration rules. + pub fn add_migration(&mut self, migration: T) -> Result<(), Error> where T: Migration { + let version_match = match self.migrations.last() { + Some(last) => last.version() + 1 == migration.version(), + None => true, + }; + + match version_match { + true => Ok(self.migrations.push(Box::new(migration))), + false => Err(Error::CannotAddMigration), + } + } + + /// Performs migrations in order. This alternates migrating between two + /// temporary databases, and returns the path to the final one used. + /// The other is deleted by this function. + pub fn execute(&self, db_path: PathBuf, version: u32) -> Result { + let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible)); + let db_config = DatabaseConfig { + prefix_size: None, + max_open_files: 64, + cache_size: None, + compaction: CompactionProfile::default(), + }; + + let mut temp_db = TempDB::One; + let mut temp_path = temp_db.path(&db_path); + + // start with the old db. + let old_path_str = try!(old_path.to_str().ok_or(Error::MigrationImpossible)); + let mut cur_db = try!(Database::open(&db_config, old_path_str).map_err(|s| Error::Custom(s))); + for migration in migrations { + // open the target temporary database. + temp_path = temp_db.path(&db_path); + let temp_path_str = try!(temp_path.to_str().ok_or(Error::MigrationImpossible)); + let mut new_db = try!(Database::open(&db_config, temp_path_str).map_err(|s| Error::Custom(s))); + + // perform the migration from cur_db to new_db. + try!(migration.migrate(&cur_db, &self.config, &mut new_db)); + // next iteration, we will migrate from this db into the other temp. + cur_db = new_db; + temp_db.swap(); + + // remove the other temporary migration database. + let _ = fs::remove_dir_all(temp_db.path(&db_path)); + } + Ok(temp_path) + } + + /// Returns true if migration is needed. + pub fn is_needed(&self, version: u32) -> bool { + match self.migrations.last() { + Some(last) => version < last.version(), + None => false, + } + } + + fn migrations_from(&self, version: u32) -> Option<&[Box]> { + // index of the first required migration + let position = self.migrations.iter().position(|m| m.version() == version + 1); + position.map(|p| &self.migrations[p..]) + } +} + diff --git a/util/src/migration/tests.rs b/util/src/migration/tests.rs index b20e18cc9..162bec054 100644 --- a/util/src/migration/tests.rs +++ b/util/src/migration/tests.rs @@ -15,18 +15,18 @@ // along with Parity. If not, see . use std::collections::BTreeMap; -use migration::{Error, Destination, Migration, Manager, Config}; +use migration::{Error, Destination, Migration, Config, SimpleMigration}; impl Destination for BTreeMap, Vec> { - fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error> { - self.extend(batch); + fn commit(&mut self, batch: &BTreeMap, Vec>) -> Result<(), Error> { + self.extend(batch.cloned()); Ok(()) } } struct Migration0; -impl Migration for Migration0 { +impl SimpleMigration for Migration0 { fn version(&self) -> u32 { 1 } @@ -42,7 +42,7 @@ impl Migration for Migration0 { struct Migration1; -impl Migration for Migration1 { +impl SimpleMigration for Migration1 { fn version(&self) -> u32 { 2 }