foundation of simple db migration (#1128)

* simple db migration

* make migration slightly more functional

* migration version is just a single unsigned integer

util/src/lib.rs

@@ -134,6 +134,7 @@ pub mod vector;
 pub mod sha3;
 pub mod hashdb;
 pub mod memorydb;
+pub mod migration;
 pub mod overlaydb;
 pub mod journaldb;
 pub mod kvdb;

util/src/migration/db_impl.rs

@@ -0,0 +1,34 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! kvdb::Database as migration::Destination
+
+use std::collections::BTreeMap;
+use kvdb::{Database, DBTransaction};
+use migration::{Destination, Error};
+
+impl Destination for Database {
+	fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error> {
+		let transaction = DBTransaction::new();
+
+		for keypair in &batch {
+			try!(transaction.put(&keypair.0, &keypair.1).map_err(Error::Custom))
+		}
+
+		self.write(transaction).map_err(Error::Custom)
+	}
+}
+

util/src/migration/manager.rs

@@ -0,0 +1,121 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Migration manager
+
+use std::collections::BTreeMap;
+use migration::{Migration, Destination};
+
+/// Migration error.
+#[derive(Debug)]
+pub enum Error {
+	/// Error returned when it is impossible to add new migration rules.
+	CannotAddMigration,
+	/// Error returned when migration from specific version can not be performed.
+	MigrationImpossible,
+	/// Custom error.
+	Custom(String),
+}
+
+/// Migration config.
+pub struct Config {
+	/// Defines how many elements should be migrated at once.
+	pub batch_size: usize,
+}
+
+impl Default for Config {
+	fn default() -> Self {
+		Config {
+			batch_size: 1024,
+		}
+	}
+}
+
+/// Manages database migration.
+pub struct Manager {
+	config: Config,
+	migrations: Vec<Box<Migration>>,
+}
+
+impl Manager {
+	/// Creates new migration manager with given configuration.
+	pub fn new(config: Config) -> Self {
+		Manager {
+			config: config,
+			migrations: vec![]
+		}
+	}
+
+	/// Adds new migration rules.
+	pub fn add_migration<T>(&mut self, migration: T) -> Result<(), Error> where T: Migration {
+		let version_match = match self.migrations.last() {
+			Some(last) => last.version() + 1 == migration.version(),
+			None => true,
+		};
+
+		match version_match {
+			true => Ok(self.migrations.push(Box::new(migration))),
+			false => Err(Error::CannotAddMigration),
+		}
+	}
+
+	/// Performs migration to destination.
+	pub fn execute<D>(&self, db_iter: D, version: u32, destination: &mut Destination) -> Result<(), Error> where
+		D: Iterator<Item = (Vec<u8>, Vec<u8>)> {
+
+		if self.is_latest_version(version) {
+			return Ok(());
+		}
+
+		let migrations = try!(self.migrations_from(version).ok_or(Error::MigrationImpossible));
+
+		let mut batch: BTreeMap<Vec<u8>, Vec<u8>> = BTreeMap::new();
+
+		for keypair in db_iter {
+			let migrated = migrations.iter().fold(Some(keypair), |migrated, migration| {
+				migrated.and_then(|(key, value)| migration.simple_migrate(key, value))
+			});
+
+			if let Some((key, value)) = migrated {
+				batch.insert(key, value);
+			}
+
+			if batch.len() == self.config.batch_size {
+				try!(destination.commit(batch));
+				batch = BTreeMap::new();
+			}
+		}
+
+		try!(destination.commit(batch));
+
+		Ok(())
+	}
+
+	/// Returns true if given string is equal to latest known version.
+	pub fn is_latest_version(&self, version: u32) -> bool {
+		match self.migrations.last() {
+			Some(last) => version == last.version(),
+			None => true
+		}
+	}
+
+	fn migrations_from(&self, version: u32) -> Option<&[Box<Migration>]> {
+		// index of the first required migration
+		let position = self.migrations.iter().position(|m| m.version() == version + 1);
+		position.map(|p| &self.migrations[p..])
+	}
+}
+

util/src/migration/mod.rs

@@ -0,0 +1,41 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! DB Migration module.
+
+mod db_impl;
+mod manager;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::manager::{Error, Config, Manager};
+use std::collections::BTreeMap;
+
+/// Single migration.
+pub trait Migration: 'static {
+	/// Version of database after the migration.
+	fn version(&self) -> u32;
+	/// Should migrate existing object to new database.
+	/// Returns `None` if the object does not exist in new version of database.
+	fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)>;
+}
+
+/// Migration destination.
+pub trait Destination {
+	/// Called on destination to commit batch of migrated entries.
+	fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error>;
+}

util/src/migration/tests.rs

@@ -0,0 +1,119 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::BTreeMap;
+use migration::{Error, Destination, Migration, Manager, Config};
+
+impl Destination for BTreeMap<Vec<u8>, Vec<u8>> {
+	fn commit(&mut self, batch: BTreeMap<Vec<u8>, Vec<u8>>) -> Result<(), Error> {
+		self.extend(batch);
+		Ok(())
+	}
+}
+
+struct Migration0;
+
+impl Migration for Migration0 {
+	fn version(&self) -> u32 {
+		1
+	}
+
+	fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+		let mut key = key;
+		key.push(0x11);
+		let mut value = value;
+		value.push(0x22);
+		Some((key, value))
+	}
+}
+
+struct Migration1;
+
+impl Migration for Migration1 {
+	fn version(&self) -> u32 {
+		2
+	}
+
+	fn simple_migrate(&self, key: Vec<u8>, _value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+		Some((key, vec![]))
+	}
+}
+
+#[test]
+fn one_simple_migration() {
+	let mut manager = Manager::new(Config::default());
+	let keys = vec![vec![], vec![1u8]];
+	let values = vec![vec![], vec![1u8]];
+	let db = keys.into_iter().zip(values.into_iter());
+
+	let expected_keys = vec![vec![0x11u8], vec![1, 0x11]];
+	let expected_values = vec![vec![0x22u8], vec![1, 0x22]];
+	let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
+
+	let mut result = BTreeMap::new();
+	manager.add_migration(Migration0).unwrap();
+	manager.execute(db, 0, &mut result).unwrap();
+	assert_eq!(expected_db, result);
+}
+
+#[test]
+fn no_migration_needed() {
+	let mut manager = Manager::new(Config::default());
+	let keys = vec![vec![], vec![1u8]];
+	let values = vec![vec![], vec![1u8]];
+	let db = keys.into_iter().zip(values.into_iter());
+
+	let mut result = BTreeMap::new();
+	manager.add_migration(Migration0).unwrap();
+	manager.execute(db, 1, &mut result).unwrap();
+	assert!(result.is_empty());
+}
+
+#[test]
+fn multiple_migrations() {
+	let mut manager = Manager::new(Config::default());
+	let keys = vec![vec![], vec![1u8]];
+	let values = vec![vec![], vec![1u8]];
+	let db = keys.into_iter().zip(values.into_iter());
+
+	let expected_keys = vec![vec![0x11u8], vec![1, 0x11]];
+	let expected_values = vec![vec![], vec![]];
+	let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
+
+	let mut result = BTreeMap::new();
+	manager.add_migration(Migration0).unwrap();
+	manager.add_migration(Migration1).unwrap();
+	manager.execute(db, 0, &mut result).unwrap();
+	assert_eq!(expected_db, result);
+}
+
+#[test]
+fn second_migration() {
+	let mut manager = Manager::new(Config::default());
+	let keys = vec![vec![], vec![1u8]];
+	let values = vec![vec![], vec![1u8]];
+	let db = keys.into_iter().zip(values.into_iter());
+
+	let expected_keys = vec![vec![], vec![1u8]];
+	let expected_values = vec![vec![], vec![]];
+	let expected_db = expected_keys.into_iter().zip(expected_values.into_iter()).collect::<BTreeMap<_, _>>();
+
+	let mut result = BTreeMap::new();
+	manager.add_migration(Migration0).unwrap();
+	manager.add_migration(Migration1).unwrap();
+	manager.execute(db, 1, &mut result).unwrap();
+	assert_eq!(expected_db, result);
+}