have AccountDB use address hash for uniqueness (#1533)

* partially done alternate migration scheme

* finish altering migration framework

* migrate tests to new migration framework

* address comments

* remove superfluous newline
[ci skip]

* TempIdx -> TempIndex
[ci skip]

* modify account_db to work on address hash, not address

* add a database migration for new accountdb

* preserve first 96 bits of keys when combining

* handle metadata keys in migration and preserve first 96 bits

* fix comments and hash address instead of hash

* different migrations based on pruning

* migrations mutably borrow self

* batch abstraction for migration

* added missing licence headers

* overlay recent v7 migration

* better error handling, migrate version key as well

* fix migration tests

* commit final batch and migrate journaled insertions

* two passes on journal to migrate all possible deleted keys

ethcore/src/account_db.rs

@@ -1,26 +1,59 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 //! DB backend wrapper for Account trie
 use util::*;
 
 static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];
 
+// combines a key with an address hash to ensure uniqueness.
+// leaves the first 96 bits untouched in order to support partial key lookup.
+#[inline]
+fn combine_key<'a>(address_hash: &'a H256, key: &'a H256) -> H256 {
+	let mut dst = key.clone();
+	{
+		let last_src: &[u8] = &*address_hash;
+		let last_dst: &mut [u8] = &mut *dst;
+		for (k, a) in last_dst[12..].iter_mut().zip(&last_src[12..]) {
+			*k ^= *a
+		}
+	}
+
+	dst
+}
+
 // TODO: introduce HashDBMut?
 /// DB backend wrapper for Account trie
 /// Transforms trie node keys for the database
 pub struct AccountDB<'db> {
 	db: &'db HashDB,
-	address: H256,
-}
-
-#[inline]
-fn combine_key<'a>(address: &'a H256, key: &'a H256) -> H256 {
-	address ^ key
+	address_hash: H256,
 }
 
 impl<'db> AccountDB<'db> {
-	pub fn new(db: &'db HashDB, address: &Address) -> AccountDB<'db> {
+	/// Create a new AccountDB from an address.
+	pub fn new(db: &'db HashDB, address: &Address) -> Self {
+		Self::from_hash(db, address.sha3())
+	}
+
+	/// Create a new AcountDB from an address' hash.
+	pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self {
 		AccountDB {
 			db: db,
-			address: address.into(),
+			address_hash: address_hash,
 		}
 	}
 }
@@ -34,14 +67,14 @@ impl<'db> HashDB for AccountDB<'db>{
 		if key == &SHA3_NULL_RLP {
 			return Some(&NULL_RLP_STATIC);
 		}
-		self.db.get(&combine_key(&self.address, key))
+		self.db.get(&combine_key(&self.address_hash, key))
 	}
 
 	fn contains(&self, key: &H256) -> bool {
 		if key == &SHA3_NULL_RLP {
 			return true;
 		}
-		self.db.contains(&combine_key(&self.address, key))
+		self.db.contains(&combine_key(&self.address_hash, key))
 	}
 
 	fn insert(&mut self, _value: &[u8]) -> H256 {
@@ -60,20 +93,26 @@ impl<'db> HashDB for AccountDB<'db>{
 /// DB backend wrapper for Account trie
 pub struct AccountDBMut<'db> {
 	db: &'db mut HashDB,
-	address: H256,
+	address_hash: H256,
 }
 
 impl<'db> AccountDBMut<'db> {
-	pub fn new(db: &'db mut HashDB, address: &Address) -> AccountDBMut<'db> {
+	/// Create a new AccountDB from an address.
+	pub fn new(db: &'db mut HashDB, address: &Address) -> Self {
+		Self::from_hash(db, address.sha3())
+	}
+
+	/// Create a new AcountDB from an address' hash.
+	pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self {
 		AccountDBMut {
 			db: db,
-			address: address.into(),
+			address_hash: address_hash,
 		}
 	}
 
 	#[allow(dead_code)]
 	pub fn immutable(&'db self) -> AccountDB<'db> {
-		AccountDB { db: self.db, address: self.address.clone() }
+		AccountDB { db: self.db, address_hash: self.address_hash.clone() }
 	}
 }
 
@@ -86,14 +125,14 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		if key == &SHA3_NULL_RLP {
 			return Some(&NULL_RLP_STATIC);
 		}
-		self.db.get(&combine_key(&self.address, key))
+		self.db.get(&combine_key(&self.address_hash, key))
 	}
 
 	fn contains(&self, key: &H256) -> bool {
 		if key == &SHA3_NULL_RLP {
 			return true;
 		}
-		self.db.contains(&combine_key(&self.address, key))
+		self.db.contains(&combine_key(&self.address_hash, key))
 	}
 
 	fn insert(&mut self, value: &[u8]) -> H256 {
@@ -101,7 +140,7 @@ impl<'db> HashDB for AccountDBMut<'db>{
 			return SHA3_NULL_RLP.clone();
 		}
 		let k = value.sha3();
-		let ak = combine_key(&self.address, &k);
+		let ak = combine_key(&self.address_hash, &k);
 		self.db.emplace(ak, value.to_vec());
 		k
 	}
@@ -110,7 +149,7 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		if key == SHA3_NULL_RLP {
 			return;
 		}
-		let key = combine_key(&self.address, &key);
+		let key = combine_key(&self.address_hash, &key);
 		self.db.emplace(key, value.to_vec())
 	}
 
@@ -118,7 +157,7 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		if key == &SHA3_NULL_RLP {
 			return;
 		}
-		let key = combine_key(&self.address, key);
+		let key = combine_key(&self.address_hash, key);
 		self.db.remove(&key)
 	}
 }

ethcore/src/migrations/extras/mod.rs

@@ -1,5 +1,21 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 //! Extras database migrations.
 
 mod v6;
 
-pub use self::v6::ToV6;
+pub use self::v6::ToV6;

ethcore/src/migrations/extras/v6.rs

@@ -1,3 +1,19 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 use util::migration::SimpleMigration;
 
 /// This migration reduces the sizes of keys and moves `ExtrasIndex` byte from back to the front.
@@ -22,7 +38,7 @@ impl SimpleMigration for ToV6 {
 		6
 	}
 
-	fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
 
 		//// at this version all extras keys are 33 bytes long.
 		if key.len() == 33 {

ethcore/src/migrations/mod.rs

@@ -1,3 +1,4 @@
 //! Database migrations.
 
 pub mod extras;
+pub mod state;

ethcore/src/migrations/state/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State database migrations.
+
+mod v7;
+
+pub use self::v7::{ArchiveV7, OverlayRecentV7};

ethcore/src/migrations/state/v7.rs

@@ -0,0 +1,247 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This migration migrates the state db to use an accountdb which ensures uniqueness
+//! using an address' hash as opposed to the address itself.
+
+use std::collections::HashMap;
+
+use util::Bytes;
+use util::hash::{Address, FixedHash, H256};
+use util::kvdb::Database;
+use util::migration::{Batch, Config, Error, Migration, SimpleMigration};
+use util::rlp::{decode, Rlp, RlpStream, Stream, View};
+use util::sha3::Hashable;
+
+// attempt to migrate a key, value pair. None if migration not possible.
+fn attempt_migrate(mut key_h: H256, val: &[u8]) -> Option<H256> {
+	let val_hash = val.sha3();
+
+	if key_h != val_hash {
+		// this is a key which has been xor'd with an address.
+		// recover the address.
+		let address = key_h ^ val_hash;
+
+		// check that the address is actually a 20-byte value.
+		// the leftmost 12 bytes should be zero.
+		if &address[0..12] != &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] {
+			return None;
+		}
+
+		let address_hash = Address::from(address).sha3();
+
+		// create the xor'd key in place.
+		key_h.copy_from_slice(&*val_hash);
+		assert_eq!(key_h, val_hash);
+
+		{
+			let last_src: &[u8] = &*address_hash;
+			let last_dst: &mut [u8] = &mut *key_h;
+			for (k, a) in last_dst[12..].iter_mut().zip(&last_src[12..]) {
+				*k ^= *a;
+			}
+		}
+
+		Some(key_h)
+	} else {
+		None
+	}
+}
+
+/// Version for ArchiveDB.
+pub struct ArchiveV7;
+
+impl SimpleMigration for ArchiveV7 {
+	fn version(&self) -> u32 {
+		7
+	}
+
+	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+		if key.len() != 32 {
+			// metadata key, ignore.
+			return Some((key, value));
+		}
+
+		let key_h = H256::from_slice(&key[..]);
+		if let Some(new_key) = attempt_migrate(key_h, &value[..]) {
+			Some((new_key[..].to_owned(), value))
+		} else {
+			Some((key, value))
+		}
+	}
+}
+
+// magic numbers and constants for overlay-recent at v6.
+// re-written here because it may change in the journaldb module.
+const V7_LATEST_ERA_KEY: &'static [u8] = &[ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
+const V7_VERSION_KEY: &'static [u8] = &[ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
+const DB_VERSION: u32 = 0x203;
+const PADDING : [u8; 10] = [0u8; 10];
+
+/// Version for OverlayRecent database.
+/// more involved than the archive version because of journaling.
+#[derive(Default)]
+pub struct OverlayRecentV7 {
+	migrated_keys: HashMap<H256, H256>,
+}
+
+impl OverlayRecentV7 {
+	// walk all journal entries in the database backwards.
+	// find migrations for any possible inserted keys.
+	fn walk_journal(&mut self, source: &Database) -> Result<(), Error> {
+		if let Some(val) = try!(source.get(V7_LATEST_ERA_KEY).map_err(Error::Custom)) {
+			let mut era = decode::<u64>(&val);
+			loop {
+				let mut index: usize = 0;
+				loop {
+					let entry_key = {
+						let mut r = RlpStream::new_list(3);
+						r.append(&era).append(&index).append(&&PADDING[..]);
+						r.out()
+					};
+
+					if let Some(journal_raw) = try!(source.get(&entry_key).map_err(Error::Custom)) {
+						let rlp = Rlp::new(&journal_raw);
+
+						// migrate all inserted keys.
+						for r in rlp.at(1).iter() {
+							let key: H256 = r.val_at(0);
+							let v: Bytes = r.val_at(1);
+
+							if self.migrated_keys.get(&key).is_none() {
+								if let Some(new_key) = attempt_migrate(key, &v) {
+									self.migrated_keys.insert(key, new_key);
+								}
+							}
+						}
+						index += 1;
+					} else {
+						break;
+					}
+				}
+
+				if index == 0 || era == 0 {
+					break;
+				}
+				era -= 1;
+			}
+		}
+		Ok(())
+	}
+
+	// walk all journal entries in the database backwards.
+	// replace all possible inserted/deleted keys with their migrated counterparts
+	// and commit the altered entries.
+	fn migrate_journal(&self, source: &Database, mut batch: Batch, dest: &mut Database) -> Result<(), Error> {
+		if let Some(val) = try!(source.get(V7_LATEST_ERA_KEY).map_err(Error::Custom)) {
+			try!(batch.insert(V7_LATEST_ERA_KEY.into(), val.to_owned(), dest));
+
+			let mut era = decode::<u64>(&val);
+			loop {
+				let mut index: usize = 0;
+				loop {
+					let entry_key = {
+						let mut r = RlpStream::new_list(3);
+						r.append(&era).append(&index).append(&&PADDING[..]);
+						r.out()
+					};
+
+					if let Some(journal_raw) = try!(source.get(&entry_key).map_err(Error::Custom)) {
+						let rlp = Rlp::new(&journal_raw);
+						let id: H256 = rlp.val_at(0);
+						let mut inserted_keys: Vec<(H256, Bytes)> = Vec::new();
+
+						// migrate all inserted keys.
+						for r in rlp.at(1).iter() {
+							let mut key: H256 = r.val_at(0);
+							let v: Bytes = r.val_at(1);
+
+							if let Some(new_key) = self.migrated_keys.get(&key) {
+								key = *new_key;
+							}
+
+							inserted_keys.push((key, v));
+						}
+
+						// migrate all deleted keys.
+						let mut deleted_keys: Vec<H256> = rlp.val_at(2);
+						for old_key in &mut deleted_keys {
+							if let Some(new) = self.migrated_keys.get(&*old_key) {
+								*old_key = new.clone();
+							}
+						}
+
+						// rebuild the journal entry rlp.
+						let mut stream = RlpStream::new_list(3);
+						stream.append(&id);
+						stream.begin_list(inserted_keys.len());
+						for (k, v) in inserted_keys {
+							stream.begin_list(2).append(&k).append(&v);
+						}
+
+						stream.append(&deleted_keys);
+
+						// and insert it into the new database.
+						try!(batch.insert(entry_key, stream.out(), dest));
+
+						index += 1;
+					} else {
+						break;
+					}
+				}
+
+				if index == 0 || era == 0 {
+					break;
+				}
+				era -= 1;
+			}
+		}
+		batch.commit(dest)
+	}
+}
+
+impl Migration for OverlayRecentV7 {
+	fn version(&self) -> u32 { 7 }
+
+	// walk all records in the database, attempting to migrate any possible and
+	// keeping records of those that we do. then migrate the journal using
+	// this information.
+	fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database) -> Result<(), Error> {
+		let mut batch = Batch::new(config);
+
+		// check version metadata.
+		match try!(source.get(V7_VERSION_KEY).map_err(Error::Custom)) {
+			Some(ref version) if decode::<u32>(&*version) == DB_VERSION => {}
+			_ => return Err(Error::MigrationImpossible), // missing or wrong version
+		}
+
+		for (key, value) in source.iter() {
+			let mut key = key.into_vec();
+			if key.len() == 32 {
+				let key_h = H256::from_slice(&key[..]);
+				if let Some(new_key) = attempt_migrate(key_h.clone(), &value) {
+					self.migrated_keys.insert(key_h, new_key);
+					key.copy_from_slice(&new_key[..]);
+				}
+			}
+
+			try!(batch.insert(key, value.into_vec(), dest));
+		}
+
+		try!(self.walk_journal(source));
+		self.migrate_journal(source, batch, dest)
+	}
+}