From 1741597a2001ddb57151ad135bfaa7ce5f13e192 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Tue, 24 May 2016 22:38:11 +0200 Subject: [PATCH] foundation of simple db migration (#1128) * simple db migration * make migration slightly more functional * migration version is just a single unsigned integer --- util/src/lib.rs | 1 + util/src/migration/db_impl.rs | 34 ++++++++++ util/src/migration/manager.rs | 121 ++++++++++++++++++++++++++++++++++ util/src/migration/mod.rs | 41 ++++++++++++ util/src/migration/tests.rs | 119 +++++++++++++++++++++++++++++++++ 5 files changed, 316 insertions(+) create mode 100644 util/src/migration/db_impl.rs create mode 100644 util/src/migration/manager.rs create mode 100644 util/src/migration/mod.rs create mode 100644 util/src/migration/tests.rs diff --git a/util/src/lib.rs b/util/src/lib.rs index a8f74b5a4..a72406542 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -134,6 +134,7 @@ pub mod vector; pub mod sha3; pub mod hashdb; pub mod memorydb; +pub mod migration; pub mod overlaydb; pub mod journaldb; pub mod kvdb; diff --git a/util/src/migration/db_impl.rs b/util/src/migration/db_impl.rs new file mode 100644 index 000000000..02c66d19b --- /dev/null +++ b/util/src/migration/db_impl.rs @@ -0,0 +1,34 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! kvdb::Database as migration::Destination + +use std::collections::BTreeMap; +use kvdb::{Database, DBTransaction}; +use migration::{Destination, Error}; + +impl Destination for Database { + fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error> { + let transaction = DBTransaction::new(); + + for keypair in &batch { + try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom)) + } + + self.write(transaction).map_err(Error::Custom) + } +} + diff --git a/util/src/migration/manager.rs b/util/src/migration/manager.rs new file mode 100644 index 000000000..8bf264350 --- /dev/null +++ b/util/src/migration/manager.rs @@ -0,0 +1,121 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Migration manager + +use std::collections::BTreeMap; +use migration::{Migration, Destination}; + +/// Migration error. +#[derive(Debug)] +pub enum Error { + /// Error returned when it is impossible to add new migration rules. + CannotAddMigration, + /// Error returned when migration from specific version can not be performed. + MigrationImpossible, + /// Custom error. + Custom(String), +} + +/// Migration config. +pub struct Config { + /// Defines how many elements should be migrated at once. + pub batch_size: usize, +} + +impl Default for Config { + fn default() -> Self { + Config { + batch_size: 1024, + } + } +} + +/// Manages database migration. +pub struct Manager { + config: Config, + migrations: Vec>, +} + +impl Manager { + /// Creates new migration manager with given configuration. + pub fn new(config: Config) -> Self { + Manager { + config: config, + migrations: vec![] + } + } + + /// Adds new migration rules. + pub fn add_migration(&mut self, migration: T) -> Result<(), Error> where T: Migration { + let version_match = match self.migrations.last() { + Some(last) => last.version() + 1 == migration.version(), + None => true, + }; + + match version_match { + true => Ok(self.migrations.push(Box::new(migration))), + false => Err(Error::CannotAddMigration), + } + } + + /// Performs migration to destination. + pub fn execute(&self, db_iter: D, version: u32, destination: &mut Destination) -> Result<(), Error> where + D: Iterator, Vec)> { + + if self.is_latest_version(version) { + return Ok(()); + } + + let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible)); + + let mut batch: BTreeMap, Vec> = BTreeMap::new(); + + for keypair in db_iter { + let migrated = migrations.iter().fold(Some(keypair), |migrated, migration| { + migrated.and_then(|(key, value)| migration.simple_migrate(key, value)) + }); + + if let Some((key, value)) = migrated { + batch.insert(key, value); + } + + if batch.len() == self.config.batch_size { + try!(destination.commit(batch)); + batch = BTreeMap::new(); + } + } + + try!(destination.commit(batch)); + + Ok(()) + } + + /// Returns true if given string is equal to latest known version. + pub fn is_latest_version(&self, version: u32) -> bool { + match self.migrations.last() { + Some(last) => version == last.version(), + None => true + } + } + + fn migrations_from(&self, version: u32) -> Option<&[Box]> { + // index of the first required migration + let position = self.migrations.iter().position(|m| m.version() == version + 1); + position.map(|p| &self.migrations[p..]) + } +} + diff --git a/util/src/migration/mod.rs b/util/src/migration/mod.rs new file mode 100644 index 000000000..e37406e2d --- /dev/null +++ b/util/src/migration/mod.rs @@ -0,0 +1,41 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! DB Migration module. + +mod db_impl; +mod manager; + +#[cfg(test)] +mod tests; + +pub use self::manager::{Error, Config, Manager}; +use std::collections::BTreeMap; + +/// Single migration. +pub trait Migration: 'static { + /// Version of database after the migration. + fn version(&self) -> u32; + /// Should migrate existing object to new database. + /// Returns `None` if the object does not exist in new version of database. + fn simple_migrate(&self, key: Vec, value: Vec) -> Option<(Vec, Vec)>; +} + +/// Migration destination. +pub trait Destination { + /// Called on destination to commit batch of migrated entries. + fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error>; +} diff --git a/util/src/migration/tests.rs b/util/src/migration/tests.rs new file mode 100644 index 000000000..1738f5467 --- /dev/null +++ b/util/src/migration/tests.rs @@ -0,0 +1,119 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::collections::BTreeMap; +use migration::{Error, Destination, Migration, Manager, Config}; + +impl Destination for BTreeMap, Vec> { + fn commit(&mut self, batch: BTreeMap, Vec>) -> Result<(), Error> { + self.extend(batch); + Ok(()) + } +} + +struct Migration0; + +impl Migration for Migration0 { + fn version(&self) -> u32 { + 1 + } + + fn simple_migrate(&self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { + let mut key = key; + key.push(0x11); + let mut value = value; + value.push(0x22); + Some((key, value)) + } +} + +struct Migration1; + +impl Migration for Migration1 { + fn version(&self) -> u32 { + 2 + } + + fn simple_migrate(&self, key: Vec, _value: Vec) -> Option<(Vec, Vec)> { + Some((key, vec![])) + } +} + +#[test] +fn one_simple_migration() { + let mut manager = Manager::new(Config::default()); + let keys = vec![vec![], vec![1u8]]; + let values = vec![vec![], vec![1u8]]; + let db = keys.into_iter().zip(values.into_iter()); + + let expected_keys = vec![vec![0x11u8], vec![1, 0x11]]; + let expected_values = vec![vec![0x22u8], vec![1, 0x22]]; + let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::>(); + + let mut result = BTreeMap::new(); + manager.add_migration(Migration0).unwrap(); + manager.execute(db, 0, &mut result).unwrap(); + assert_eq!(expected_db, result); +} + +#[test] +fn no_migration_needed() { + let mut manager = Manager::new(Config::default()); + let keys = vec![vec![], vec![1u8]]; + let values = vec![vec![], vec![1u8]]; + let db = keys.into_iter().zip(values.into_iter()); + + let mut result = BTreeMap::new(); + manager.add_migration(Migration0).unwrap(); + manager.execute(db, 1, &mut result).unwrap(); + assert!(result.is_empty()); +} + +#[test] +fn multiple_migrations() { + let mut manager = Manager::new(Config::default()); + let keys = vec![vec![], vec![1u8]]; + let values = vec![vec![], vec![1u8]]; + let db = keys.into_iter().zip(values.into_iter()); + + let expected_keys = vec![vec![0x11u8], vec![1, 0x11]]; + let expected_values = vec![vec![], vec![]]; + let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::>(); + + let mut result = BTreeMap::new(); + manager.add_migration(Migration0).unwrap(); + manager.add_migration(Migration1).unwrap(); + manager.execute(db, 0, &mut result).unwrap(); + assert_eq!(expected_db, result); +} + +#[test] +fn second_migration() { + let mut manager = Manager::new(Config::default()); + let keys = vec![vec![], vec![1u8]]; + let values = vec![vec![], vec![1u8]]; + let db = keys.into_iter().zip(values.into_iter()); + + let expected_keys = vec![vec![], vec![1u8]]; + let expected_values = vec![vec![], vec![]]; + let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::>(); + + let mut result = BTreeMap::new(); + manager.add_migration(Migration0).unwrap(); + manager.add_migration(Migration1).unwrap(); + manager.execute(db, 1, &mut result).unwrap(); + assert_eq!(expected_db, result); +}