separated ethcore_migrations from ethcore (#7586)

* separated ethcore_migrations from ethcore

* fixed authorship nitpick

* make ethcore_migrations related exports and imports more consistent
This commit is contained in:
Marek Kotewicz
2018-01-17 22:11:13 +01:00
committed by GitHub
parent d429ce1849
commit 0c01db4a49
16 changed files with 72 additions and 8 deletions

View File

@@ -138,13 +138,13 @@ pub mod ethereum;
pub mod executed;
pub mod header;
pub mod machine;
pub mod migrations;
pub mod miner;
pub mod pod_state;
pub mod service;
pub mod snapshot;
pub mod spec;
pub mod state;
pub mod state_db;
pub mod timer;
pub mod trace;
pub mod verification;
@@ -153,7 +153,6 @@ pub mod views;
mod cache_manager;
mod blooms;
mod pod_account;
mod state_db;
mod account_db;
mod builtin;
mod executive;

View File

@@ -1,21 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Blocks database migrations.
mod v8;
pub use self::v8::V8;

View File

@@ -1,37 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! This migration compresses the state db.
use migration::{SimpleMigration, Progress};
use rlp::{Compressible, UntrustedRlp, RlpType};
/// Compressing migration.
#[derive(Default)]
pub struct V8(Progress);
impl SimpleMigration for V8 {
fn version(&self) -> u32 {
8
}
fn columns(&self) -> Option<u32> { None }
fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
self.0.tick();
Some((key,UntrustedRlp::new(&value).compress(RlpType::Blocks).into_vec()))
}
}

View File

@@ -1,21 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Extras database migrations.
mod v6;
pub use self::v6::ToV6;

View File

@@ -1,102 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use migration::SimpleMigration;
/// This migration reduces the sizes of keys and moves `ExtrasIndex` byte from back to the front.
pub struct ToV6;
impl ToV6 {
fn migrate_old_key(&self, old_key: Vec<u8>, index: u8, len: usize) -> Vec<u8> {
let mut result = vec![];
result.reserve(len);
unsafe {
result.set_len(len);
}
result[0] = index;
let old_key_start = 33 - len;
result[1..].clone_from_slice(&old_key[old_key_start..32]);
result
}
}
impl SimpleMigration for ToV6 {
fn columns(&self) -> Option<u32> { None }
fn version(&self) -> u32 { 6 }
fn simple_migrate(&mut self, mut key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
//// at this version all extras keys are 33 bytes long.
if key.len() == 33 {
// block details key changes:
// - index is moved to the front
if key[32] == 0 {
return Some((self.migrate_old_key(key, 0, 33), value));
}
// block hash key changes:
// - key is shorter 33 -> 5 bytes
// - index is moved to the front
if key[32] == 1 {
return Some((self.migrate_old_key(key, 1, 5), value));
}
// transaction addresses changes:
// - index is moved to the front
if key[32] == 2 {
return Some((self.migrate_old_key(key, 2, 33), value));
}
// block log blooms are removed
if key[32] == 3 {
return None;
}
// blocks blooms key changes:
// - key is shorter 33 -> 6 bytes
// - index is moved to the front
// - index is changed 4 -> 3
if key[32] == 4 {
key.reverse();
// i have no idea why it was reversed
let reverse = key;
let result = vec![
// new extras index is 3
3,
// 9th (+ prefix) byte was the level. Now it's second.
reverse[9],
reverse[4],
reverse[3],
reverse[2],
reverse[1],
];
return Some((result, value));
}
// blocks receipts key changes:
// - index is moved to the front
// - index is changed 5 -> 4
if key[32] == 5 {
return Some((self.migrate_old_key(key, 4, 33), value));
}
}
Some((key, value))
}
}

View File

@@ -1,46 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Database migrations.
use migration::ChangeColumns;
pub mod state;
pub mod blocks;
pub mod extras;
mod v9;
pub use self::v9::ToV9;
pub use self::v9::Extract;
mod v10;
pub use self::v10::ToV10;
/// The migration from v10 to v11.
/// Adds a column for node info.
pub const TO_V11: ChangeColumns = ChangeColumns {
pre_columns: Some(6),
post_columns: Some(7),
version: 11,
};
/// The migration from v11 to v12.
/// Adds a column for light chain storage.
pub const TO_V12: ChangeColumns = ChangeColumns {
pre_columns: Some(7),
post_columns: Some(8),
version: 12,
};

View File

@@ -1,21 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! State database migrations.
mod v7;
pub use self::v7::{ArchiveV7, OverlayRecentV7};

View File

@@ -1,263 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! This migration migrates the state db to use an accountdb which ensures uniqueness
//! using an address' hash as opposed to the address itself.
use std::collections::HashMap;
use ethereum_types::{H256, Address};
use bytes::Bytes;
use kvdb_rocksdb::Database;
use migration::{Batch, Config, Error, ErrorKind, Migration, SimpleMigration, Progress};
use hash::keccak;
use std::sync::Arc;
use rlp::{decode, Rlp, RlpStream};
// attempt to migrate a key, value pair. None if migration not possible.
fn attempt_migrate(mut key_h: H256, val: &[u8]) -> Option<H256> {
let val_hash = keccak(val);
if key_h != val_hash {
// this is a key which has been xor'd with an address.
// recover the address.
let address = key_h ^ val_hash;
// check that the address is actually a 20-byte value.
// the leftmost 12 bytes should be zero.
if &address[0..12] != &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] {
return None;
}
let address_hash = keccak(Address::from(address));
// create the xor'd key in place.
key_h.copy_from_slice(&*val_hash);
assert_eq!(key_h, val_hash);
{
let last_src: &[u8] = &*address_hash;
let last_dst: &mut [u8] = &mut *key_h;
for (k, a) in last_dst[12..].iter_mut().zip(&last_src[12..]) {
*k ^= *a;
}
}
Some(key_h)
} else {
None
}
}
/// Version for `ArchiveDB`.
#[derive(Default)]
pub struct ArchiveV7(Progress);
impl SimpleMigration for ArchiveV7 {
fn columns(&self) -> Option<u32> { None }
fn version(&self) -> u32 { 7 }
fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
self.0.tick();
if key.len() != 32 {
// metadata key, ignore.
return Some((key, value));
}
let key_h = H256::from_slice(&key[..]);
if let Some(new_key) = attempt_migrate(key_h, &value[..]) {
Some((new_key[..].to_owned(), value))
} else {
Some((key, value))
}
}
}
// magic numbers and constants for overlay-recent at v6.
// re-written here because it may change in the journaldb module.
const V7_LATEST_ERA_KEY: &'static [u8] = &[ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ];
const V7_VERSION_KEY: &'static [u8] = &[ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ];
const DB_VERSION: u32 = 0x203;
const PADDING : [u8; 10] = [0u8; 10];
/// Version for `OverlayRecent` database.
/// more involved than the archive version because of journaling.
#[derive(Default)]
pub struct OverlayRecentV7 {
migrated_keys: HashMap<H256, H256>,
}
impl OverlayRecentV7 {
// walk all journal entries in the database backwards.
// find migrations for any possible inserted keys.
fn walk_journal(&mut self, source: Arc<Database>) -> Result<(), Error> {
if let Some(val) = source.get(None, V7_LATEST_ERA_KEY)? {
let mut era = decode::<u64>(&val);
loop {
let mut index: usize = 0;
loop {
let entry_key = {
let mut r = RlpStream::new_list(3);
r.append(&era).append(&index).append(&&PADDING[..]);
r.out()
};
if let Some(journal_raw) = source.get(None, &entry_key)? {
let rlp = Rlp::new(&journal_raw);
// migrate all inserted keys.
for r in rlp.at(1).iter() {
let key: H256 = r.val_at(0);
let v: Bytes = r.val_at(1);
if self.migrated_keys.get(&key).is_none() {
if let Some(new_key) = attempt_migrate(key, &v) {
self.migrated_keys.insert(key, new_key);
}
}
}
index += 1;
} else {
break;
}
}
if index == 0 || era == 0 {
break;
}
era -= 1;
}
}
Ok(())
}
// walk all journal entries in the database backwards.
// replace all possible inserted/deleted keys with their migrated counterparts
// and commit the altered entries.
fn migrate_journal(&self, source: Arc<Database>, mut batch: Batch, dest: &mut Database) -> Result<(), Error> {
if let Some(val) = source.get(None, V7_LATEST_ERA_KEY)? {
batch.insert(V7_LATEST_ERA_KEY.into(), val.clone().into_vec(), dest)?;
let mut era = decode::<u64>(&val);
loop {
let mut index: usize = 0;
loop {
let entry_key = {
let mut r = RlpStream::new_list(3);
r.append(&era).append(&index).append(&&PADDING[..]);
r.out()
};
if let Some(journal_raw) = source.get(None, &entry_key)? {
let rlp = Rlp::new(&journal_raw);
let id: H256 = rlp.val_at(0);
let mut inserted_keys: Vec<(H256, Bytes)> = Vec::new();
// migrate all inserted keys.
for r in rlp.at(1).iter() {
let mut key: H256 = r.val_at(0);
let v: Bytes = r.val_at(1);
if let Some(new_key) = self.migrated_keys.get(&key) {
key = *new_key;
}
inserted_keys.push((key, v));
}
// migrate all deleted keys.
let mut deleted_keys: Vec<H256> = rlp.list_at(2);
for old_key in &mut deleted_keys {
if let Some(new) = self.migrated_keys.get(&*old_key) {
*old_key = new.clone();
}
}
// rebuild the journal entry rlp.
let mut stream = RlpStream::new_list(3);
stream.append(&id);
stream.begin_list(inserted_keys.len());
for (k, v) in inserted_keys {
stream.begin_list(2).append(&k).append(&v);
}
stream.append_list(&deleted_keys);
// and insert it into the new database.
batch.insert(entry_key, stream.out(), dest)?;
index += 1;
} else {
break;
}
}
if index == 0 || era == 0 {
break;
}
era -= 1;
}
}
batch.commit(dest)
}
}
impl Migration for OverlayRecentV7 {
fn columns(&self) -> Option<u32> { None }
fn version(&self) -> u32 { 7 }
// walk all records in the database, attempting to migrate any possible and
// keeping records of those that we do. then migrate the journal using
// this information.
fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
let mut batch = Batch::new(config, col);
// check version metadata.
match source.get(None, V7_VERSION_KEY)? {
Some(ref version) if decode::<u32>(&*version) == DB_VERSION => {}
_ => return Err(ErrorKind::MigrationImpossible.into()), // missing or wrong version
}
let mut count = 0;
for (key, value) in source.iter(None).into_iter().flat_map(|inner| inner) {
count += 1;
if count == 100_000 {
count = 0;
flush!(".");
}
let mut key = key.into_vec();
if key.len() == 32 {
let key_h = H256::from_slice(&key[..]);
if let Some(new_key) = attempt_migrate(key_h.clone(), &value) {
self.migrated_keys.insert(key_h, new_key);
key.copy_from_slice(&new_key[..]);
}
}
batch.insert(key, value.into_vec(), dest)?;
}
self.walk_journal(source.clone())?;
self.migrate_journal(source, batch, dest)
}
}

View File

@@ -1,119 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Bloom upgrade
use std::sync::Arc;
use db::{COL_EXTRA, COL_HEADERS, COL_STATE};
use state_db::{ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET, StateDB};
use trie::TrieDB;
use views::HeaderView;
use bloom_journal::Bloom;
use migration::{Error, Migration, Progress, Batch, Config, ErrorKind};
use journaldb;
use ethereum_types::H256;
use trie::Trie;
use kvdb::{DBTransaction, ResultExt};
use kvdb_rocksdb::Database;
/// Account bloom upgrade routine. If bloom already present, does nothing.
/// If database empty (no best block), does nothing.
/// Can be called on upgraded database with no issues (will do nothing).
pub fn generate_bloom(source: Arc<Database>, dest: &mut Database) -> Result<(), Error> {
trace!(target: "migration", "Account bloom upgrade started");
let best_block_hash = match source.get(COL_EXTRA, b"best")? {
// no migration needed
None => {
trace!(target: "migration", "No best block hash, skipping");
return Ok(());
},
Some(hash) => hash,
};
let best_block_header = match source.get(COL_HEADERS, &best_block_hash)? {
// no best block, nothing to do
None => {
trace!(target: "migration", "No best block header, skipping");
return Ok(())
},
Some(x) => x,
};
let state_root = HeaderView::new(&best_block_header).state_root();
trace!("Adding accounts bloom (one-time upgrade)");
let bloom_journal = {
let mut bloom = Bloom::new(ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET);
// no difference what algorithm is passed, since there will be no writes
let state_db = journaldb::new(
source.clone(),
journaldb::Algorithm::OverlayRecent,
COL_STATE);
let account_trie = TrieDB::new(state_db.as_hashdb(), &state_root).chain_err(|| "Cannot open trie")?;
for item in account_trie.iter().map_err(|_| ErrorKind::MigrationImpossible)? {
let (ref account_key, _) = item.map_err(|_| ErrorKind::MigrationImpossible)?;
let account_key_hash = H256::from_slice(account_key);
bloom.set(&*account_key_hash);
}
bloom.drain_journal()
};
trace!(target: "migration", "Generated {} bloom updates", bloom_journal.entries.len());
let mut batch = DBTransaction::new();
StateDB::commit_bloom(&mut batch, bloom_journal).chain_err(|| "Failed to commit bloom")?;
dest.write(batch)?;
trace!(target: "migration", "Finished bloom update");
Ok(())
}
/// Account bloom migration.
#[derive(Default)]
pub struct ToV10 {
progress: Progress,
}
impl ToV10 {
/// New v10 migration
pub fn new() -> ToV10 { ToV10 { progress: Progress::default() } }
}
impl Migration for ToV10 {
fn version(&self) -> u32 {
10
}
fn pre_columns(&self) -> Option<u32> { Some(5) }
fn columns(&self) -> Option<u32> { Some(6) }
fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
let mut batch = Batch::new(config, col);
for (key, value) in source.iter(col).into_iter().flat_map(|inner| inner) {
self.progress.tick();
batch.insert(key.into_vec(), value.into_vec(), dest)?;
}
batch.commit(dest)?;
if col == COL_STATE {
generate_bloom(source, dest)?;
}
Ok(())
}
}

View File

@@ -1,82 +0,0 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! This migration consolidates all databases into single one using Column Families.
use rlp::{Rlp, RlpStream};
use kvdb_rocksdb::Database;
use migration::{Batch, Config, Error, Migration, Progress};
use std::sync::Arc;
/// Which part of block to preserve
pub enum Extract {
/// Extract block header RLP.
Header,
/// Extract block body RLP.
Body,
/// Don't change the value.
All,
}
/// Consolidation of extras/block/state databases into single one.
pub struct ToV9 {
progress: Progress,
column: Option<u32>,
extract: Extract,
}
impl ToV9 {
/// Creates new V9 migration and assigns all `(key,value)` pairs from `source` DB to given Column Family
pub fn new(column: Option<u32>, extract: Extract) -> Self {
ToV9 {
progress: Progress::default(),
column: column,
extract: extract,
}
}
}
impl Migration for ToV9 {
fn columns(&self) -> Option<u32> { Some(5) }
fn version(&self) -> u32 { 9 }
fn migrate(&mut self, source: Arc<Database>, config: &Config, dest: &mut Database, col: Option<u32>) -> Result<(), Error> {
let mut batch = Batch::new(config, self.column);
for (key, value) in source.iter(col).into_iter().flat_map(|inner| inner) {
self.progress.tick();
match self.extract {
Extract::Header => {
batch.insert(key.into_vec(), Rlp::new(&value).at(0).as_raw().to_vec(), dest)?
},
Extract::Body => {
let mut body = RlpStream::new_list(2);
let block_rlp = Rlp::new(&value);
body.append_raw(block_rlp.at(1).as_raw(), 1);
body.append_raw(block_rlp.at(2).as_raw(), 1);
batch.insert(key.into_vec(), body.out(), dest)?
},
Extract::All => {
batch.insert(key.into_vec(), value.into_vec(), dest)?
}
}
}
batch.commit(dest)
}
}

View File

@@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! State database abstraction.
use std::collections::{VecDeque, HashSet};
use std::sync::Arc;
use lru_cache::LruCache;
@@ -31,9 +33,13 @@ use bloom_journal::{Bloom, BloomJournal};
use db::COL_ACCOUNT_BLOOM;
use byteorder::{LittleEndian, ByteOrder};
/// Number of bytes allocated in the memory for accounts bloom.
pub const ACCOUNT_BLOOM_SPACE: usize = 1048576;
/// Estimated maximum number of accounts in memory bloom.
pub const DEFAULT_ACCOUNT_PRESET: usize = 1000000;
/// Database key represening number of account hashes.
pub const ACCOUNT_BLOOM_HASHCOUNT_KEY: &'static [u8] = b"account_hash_count";
const STATE_CACHE_BLOCKS: usize = 12;
@@ -169,6 +175,7 @@ impl StateDB {
bloom
}
/// Commit bloom to a database transaction
pub fn commit_bloom(batch: &mut DBTransaction, journal: BloomJournal) -> Result<(), UtilError> {
assert!(journal.hash_functions <= 255);
batch.put(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY, &[journal.hash_functions as u8]);
@@ -298,10 +305,12 @@ impl StateDB {
}
}
/// Returns immutable reference to underlying hashdb.
pub fn as_hashdb(&self) -> &HashDB {
self.db.as_hashdb()
}
/// Returns mutable reference to underlying hashdb.
pub fn as_hashdb_mut(&mut self) -> &mut HashDB {
self.db.as_hashdb_mut()
}