grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Snapshot creation and restoration (#1679)

Browse Source 
* to_rlp takes self by-reference

* clean up some derefs

* out-of-order insertion for blockchain

* implement block rebuilder without verification

* group block chunk header into struct

* block rebuilder does verification

* integrate snapshot service with client service; flesh out implementation more

* initial implementation of snapshot service

* remove snapshottaker trait

* snapshot writer trait with packed and loose implementations

* write chunks using "snapshotwriter" in service

* have snapshot taking use snapshotwriter

* implement snapshot readers

* back up client dbs when replacing

* use snapshot reader in snapshot service

* describe offset format

* use new get_db_path in parity, allow some errors in service

* blockchain formatting

* implement parity snapshot

* implement snapshot restore

* force blocks to be submitted in order

* fix bug loading block hashes in packed reader

* fix seal field loading

* fix uncle hash computation

* fix a few bugs

* store genesis state in db. reverse block chunk order in packed writer

* allow out-of-order import for blocks

* bring restoration types together

* only snapshot the last 30000 blocks

* restore into overlaydb instead of journaldb

* commit version to database

* use memorydbs and commit directly

* fix trie test compilation

* fix failing tests

* sha3_null_rlp, not H256::zero

* move overlaydb to ref_overlaydb, add new overlaydb without on-disk rc

* port archivedb to new overlaydb

* add deletion mode tests for overlaydb

* use new overlaydb, check state root at end

* share chain info between state and block snapshotting

* create blocks snapshot using blockchain directly

* allow snapshot from arbitrary block, remove panickers from snapshot creation

* begin test framework

* blockchain chunking test

* implement stateproducer::tick

* state snapshot test

* create block and state chunks concurrently, better restoration informant

* fix tests

* add deletion mode tests for overlaydb

* address comments

* more tests

* Fix up tests.

* remove a few printlns

* add a little more documentation to `commit`

* fix tests

* fix ref_overlaydb test names

* snapshot command skeleton

* revert ref_overlaydb renaming

* reimplement snapshot commands

* fix many errors

* everything but inject

* get ethcore compiling

* get snapshot tests passing again

* instrument snapshot commands again

* fix fallout from other changes, mark snapshots as experimental

* optimize injection patterns

* do two injections

* fix up tests

* take snapshots from 1000 blocks efore

* address minor comments

* fix a few io crate related errors

* clarify names about total difficulty

[ci skip]
This commit is contained in:
Robert Habermeier
2016-08-05 17:00:46 +02:00
committed by Gav Wood
parent 725d32083a
commit 76a7246369
42 changed files with 1913 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
parity/cli.rs
View File

@@ -32,6 +32,8 @@ Usage:
parity import [ <file> ] [options]
parity export [ <file> ] [options]
parity signer new-token [options]
parity snapshot <file> [options]
parity restore <file> [options]
Operating Options:
--mode MODE Set the operating mode. MODE can be one of:
@@ -286,6 +288,8 @@ pub struct Args {
pub cmd_import: bool,
pub cmd_signer: bool,
pub cmd_new_token: bool,
pub cmd_snapshot: bool,
pub cmd_restore: bool,
pub cmd_ui: bool,
pub arg_pid_file: String,
pub arg_file: Option<String>,

32
parity/configuration.rs
View File

@@ -41,6 +41,7 @@ use run::RunCmd;
use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat};
use presale::ImportWallet;
use account::{AccountCmd, NewAccount, ImportAccounts};
use snapshot::{self, SnapshotCommand};
#[derive(Debug, PartialEq)]
pub enum Cmd {
@@ -50,6 +51,7 @@ pub enum Cmd {
ImportPresaleWallet(ImportWallet),
Blockchain(BlockchainCmd),
SignerToken(String),
Snapshot(SnapshotCommand),
}
#[derive(Debug, PartialEq)]
@@ -156,6 +158,36 @@ impl Configuration {
to_block: try!(to_block_id(&self.args.flag_to)),
};
Cmd::Blockchain(BlockchainCmd::Export(export_cmd))
} else if self.args.cmd_snapshot {
let snapshot_cmd = SnapshotCommand {
cache_config: cache_config,
dirs: dirs,
spec: spec,
pruning: pruning,
logger_config: logger_config,
mode: mode,
tracing: tracing,
compaction: compaction,
file_path: self.args.arg_file.clone(),
wal: wal,
kind: snapshot::Kind::Take,
};
Cmd::Snapshot(snapshot_cmd)
} else if self.args.cmd_restore {
let restore_cmd = SnapshotCommand {
cache_config: cache_config,
dirs: dirs,
spec: spec,
pruning: pruning,
logger_config: logger_config,
mode: mode,
tracing: tracing,
compaction: compaction,
file_path: self.args.arg_file.clone(),
wal: wal,
kind: snapshot::Kind::Restore,
};
Cmd::Snapshot(restore_cmd)
} else {
let daemon = if self.args.cmd_daemon {
Some(self.args.arg_pid_file.clone())

3
parity/main.rs
View File

@@ -82,6 +82,7 @@ mod blockchain;
mod presale;
mod run;
mod sync;
mod snapshot;
use std::{process, env};
use cli::print_version;
@@ -99,6 +100,7 @@ fn execute(command: Cmd) -> Result<String, String> {
Cmd::ImportPresaleWallet(presale_cmd) => presale::execute(presale_cmd),
Cmd::Blockchain(blockchain_cmd) => blockchain::execute(blockchain_cmd),
Cmd::SignerToken(path) => signer::new_token(path),
Cmd::Snapshot(snapshot_cmd) => snapshot::execute(snapshot_cmd),
}
}
@@ -131,4 +133,3 @@ fn main() {
}
}
}

2
parity/run.rs
View File

@@ -171,7 +171,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
net_conf.boot_nodes = spec.nodes.clone();
}
// create client
// create client service.
let service = try!(ClientService::start(
client_config,
spec,

195
parity/snapshot.rs Normal file
View File

@@ -0,0 +1,195 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Snapshot and restoration commands.
use std::time::Duration;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use ethcore_logger::{setup_log, Config as LogConfig};
use ethcore::snapshot::{RestorationStatus, SnapshotService};
use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter};
use ethcore::service::ClientService;
use ethcore::client::{Mode, DatabaseCompactionProfile, Switch, VMType};
use ethcore::miner::Miner;
use cache::CacheConfig;
use params::{SpecType, Pruning};
use helpers::{to_client_config, execute_upgrades};
use dir::Directories;
use fdlimit;
use io::PanicHandler;
/// Kinds of snapshot commands.
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum Kind {
/// Take a snapshot.
Take,
/// Restore a snapshot.
Restore
}
/// Command for snapshot creation or restoration.
#[derive(Debug, PartialEq)]
pub struct SnapshotCommand {
pub cache_config: CacheConfig,
pub dirs: Directories,
pub spec: SpecType,
pub pruning: Pruning,
pub logger_config: LogConfig,
pub mode: Mode,
pub tracing: Switch,
pub compaction: DatabaseCompactionProfile,
pub file_path: Option<String>,
pub wal: bool,
pub kind: Kind,
}
impl SnapshotCommand {
// shared portion of snapshot commands: start the client service
fn start_service(self) -> Result<(ClientService, Arc<PanicHandler>), String> {
// Setup panic handler
let panic_handler = PanicHandler::new_in_arc();
// load spec file
let spec = try!(self.spec.spec());
// load genesis hash
let genesis_hash = spec.genesis_header().hash();
// Setup logging
let _logger = setup_log(&self.logger_config);
fdlimit::raise_fd_limit();
// select pruning algorithm
let algorithm = self.pruning.to_algorithm(&self.dirs, genesis_hash, spec.fork_name.as_ref());
// prepare client_path
let client_path = self.dirs.client_path(genesis_hash, spec.fork_name.as_ref(), algorithm);
// execute upgrades
try!(execute_upgrades(&self.dirs, genesis_hash, spec.fork_name.as_ref(), algorithm, self.compaction.compaction_profile()));
// prepare client config
let client_config = to_client_config(&self.cache_config, &self.dirs, genesis_hash, self.mode, self.tracing, self.pruning, self.compaction, self.wal, VMType::default(), "".into(), spec.fork_name.as_ref());
let service = try!(ClientService::start(
client_config,
spec,
Path::new(&client_path),
Arc::new(Miner::with_spec(try!(self.spec.spec())))
).map_err(|e| format!("Client service error: {:?}", e)));
Ok((service, panic_handler))
}
/// restore from a snapshot
pub fn restore(self) -> Result<(), String> {
let file = try!(self.file_path.clone().ok_or("No file path provided.".to_owned()));
let (service, _panic_handler) = try!(self.start_service());
warn!("Snapshot restoration is experimental and the format may be subject to change.");
let snapshot = service.snapshot_service();
let reader = PackedReader::new(&Path::new(&file))
.map_err(|e| format!("Couldn't open snapshot file: {}", e))
.and_then(|x| x.ok_or("Snapshot file has invalid format.".into()));
let reader = try!(reader);
let manifest = reader.manifest();
// drop the client so we don't restore while it has open DB handles.
drop(service);
if !snapshot.begin_restore(manifest.clone()) {
return Err("Failed to begin restoration.".into());
}
let (num_state, num_blocks) = (manifest.state_hashes.len(), manifest.block_hashes.len());
let informant_handle = snapshot.clone();
::std::thread::spawn(move || {
while let RestorationStatus::Ongoing = informant_handle.status() {
let (state_chunks, block_chunks) = informant_handle.chunks_done();
info!("Processed {}/{} state chunks and {}/{} block chunks.",
state_chunks, num_state, block_chunks, num_blocks);
::std::thread::sleep(Duration::from_secs(5));
}
});
info!("Restoring state");
for &state_hash in &manifest.state_hashes {
if snapshot.status() == RestorationStatus::Failed {
return Err("Restoration failed".into());
}
let chunk = try!(reader.chunk(state_hash)
.map_err(|e| format!("Encountered error while reading chunk {:?}: {}", state_hash, e)));
snapshot.feed_state_chunk(state_hash, &chunk);
}
info!("Restoring blocks");
for &block_hash in &manifest.block_hashes {
if snapshot.status() == RestorationStatus::Failed {
return Err("Restoration failed".into());
}
let chunk = try!(reader.chunk(block_hash)
.map_err(|e| format!("Encountered error while reading chunk {:?}: {}", block_hash, e)));
snapshot.feed_block_chunk(block_hash, &chunk);
}
match snapshot.status() {
RestorationStatus::Ongoing => Err("Snapshot file is incomplete and missing chunks.".into()),
RestorationStatus::Failed => Err("Snapshot restoration failed.".into()),
RestorationStatus::Inactive => {
info!("Restoration complete.");
Ok(())
}
}
}
/// Take a snapshot from the head of the chain.
pub fn take_snapshot(self) -> Result<(), String> {
let file_path = try!(self.file_path.clone().ok_or("No file path provided.".to_owned()));
let file_path: PathBuf = file_path.into();
let (service, _panic_handler) = try!(self.start_service());
warn!("Snapshots are currently experimental. File formats may be subject to change.");
let writer = try!(PackedWriter::new(&file_path)
.map_err(|e| format!("Failed to open snapshot writer: {}", e)));
if let Err(e) = service.client().take_snapshot(writer) {
let _ = ::std::fs::remove_file(&file_path);
return Err(format!("Encountered fatal error while creating snapshot: {}", e));
}
Ok(())
}
}
/// Execute this snapshot command.
pub fn execute(cmd: SnapshotCommand) -> Result<String, String> {
match cmd.kind {
Kind::Take => try!(cmd.take_snapshot()),
Kind::Restore => try!(cmd.restore()),
}
Ok(String::new())
}