Extract snapshot to own crate (#11010)

* Move snapshot to own crate
Sort out imports

* WIP cargo toml

* Make snapshotting generic over the client
Sort out tests

* Sort out types from blockchain and client

* Sort out sync

* Sort out imports and generics

* Sort out main binary

* Fix sync test-helpers

* Sort out import for secret-store

* Sort out more imports

* Fix easy todos

* cleanup

* Cleanup

* remove unneded workspace member

* cleanup

* Sort out test-helpers dependency on account-db

* Update ethcore/client-traits/src/lib.rs

Co-Authored-By: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* Update ethcore/snapshot/Cargo.toml

parity/configuration.rs

@@ -31,7 +31,7 @@ use sync::{NetworkConfiguration, validate_node_url, self};
 use ethkey::{Secret, Public};
 use ethcore::client::{VMType};
 use ethcore::miner::{stratum, MinerOptions};
-use ethcore::snapshot::SnapshotConfiguration;
+use snapshot::SnapshotConfiguration;
 use miner::pool;
 use verification::queue::VerifierSettings;
 
@@ -52,7 +52,7 @@ use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, KillBlockcha
 use export_hardcoded_sync::ExportHsyncCmd;
 use presale::ImportWallet;
 use account::{AccountCmd, NewAccount, ListAccounts, ImportAccounts, ImportFromGethAccounts};
-use snapshot::{self, SnapshotCommand};
+use snapshot_cmd::{self, SnapshotCommand};
 use network::{IpFilter};
 
 const DEFAULT_MAX_PEERS: u16 = 50;
@@ -321,7 +321,7 @@ impl Configuration {
 				fat_db: fat_db,
 				compaction: compaction,
 				file_path: self.args.arg_snapshot_file.clone(),
-				kind: snapshot::Kind::Take,
+				kind: snapshot_cmd::Kind::Take,
 				block_at: to_block_id(&self.args.arg_snapshot_at)?,
 				max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import,
 				snapshot_conf: snapshot_conf,
@@ -339,7 +339,7 @@ impl Configuration {
 				fat_db: fat_db,
 				compaction: compaction,
 				file_path: self.args.arg_restore_file.clone(),
-				kind: snapshot::Kind::Restore,
+				kind: snapshot_cmd::Kind::Restore,
 				block_at: to_block_id("latest")?, // unimportant.
 				max_round_blocks_to_import: self.args.arg_max_round_blocks_to_import,
 				snapshot_conf: snapshot_conf,

parity/informant.rs

@@ -23,10 +23,11 @@ use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
 use std::time::{Instant, Duration};
 
 use atty;
-use ethcore::client::{ChainNotify, NewBlocks, Client};
-use client_traits::{BlockInfo, ChainInfo, BlockChainClient};
+use ethcore::client::Client;
+use client_traits::{BlockInfo, ChainInfo, BlockChainClient, ChainNotify};
 use types::{
 	BlockNumber,
+	chain_notify::NewBlocks,
 	client_types::ClientReport,
 	ids::BlockId,
 	io_message::ClientIoMessage,
@@ -34,8 +35,8 @@ use types::{
 	verification::VerificationQueueInfo as BlockQueueInfo,
 	snapshot::RestorationStatus,
 };
-use ethcore::snapshot::SnapshotService as SS;
-use ethcore::snapshot::service::Service as SnapshotService;
+use snapshot::SnapshotService as SS;
+use snapshot::service::Service as SnapshotService;
 use sync::{LightSyncProvider, LightSync, SyncProvider, ManageNetwork};
 use io::{TimerToken, IoContext, IoHandler};
 use light::Cache as LightDataCache;
@@ -224,7 +225,7 @@ pub struct Informant<T> {
 	last_tick: RwLock<Instant>,
 	with_color: bool,
 	target: T,
-	snapshot: Option<Arc<SnapshotService>>,
+	snapshot: Option<Arc<SnapshotService<Client>>>,
 	rpc_stats: Option<Arc<RpcStats>>,
 	last_import: Mutex<Instant>,
 	skipped: AtomicUsize,
@@ -237,16 +238,16 @@ impl<T: InformantData> Informant<T> {
 	/// Make a new instance potentially `with_color` output.
 	pub fn new(
 		target: T,
-		snapshot: Option<Arc<SnapshotService>>,
+		snapshot: Option<Arc<SnapshotService<Client>>>,
 		rpc_stats: Option<Arc<RpcStats>>,
 		with_color: bool,
 	) -> Self {
 		Informant {
 			last_tick: RwLock::new(Instant::now()),
-			with_color: with_color,
-			target: target,
-			snapshot: snapshot,
-			rpc_stats: rpc_stats,
+			with_color,
+			target,
+			snapshot,
+			rpc_stats,
 			last_import: Mutex::new(Instant::now()),
 			skipped: AtomicUsize::new(0),
 			skipped_txs: AtomicUsize::new(0),

parity/lib.rs

@@ -73,6 +73,7 @@ extern crate parity_runtime;
 extern crate parity_updater as updater;
 extern crate parity_version;
 extern crate registrar;
+extern crate snapshot;
 extern crate spec;
 extern crate verification;
 
@@ -112,7 +113,7 @@ mod rpc_apis;
 mod run;
 mod secretstore;
 mod signer;
-mod snapshot;
+mod snapshot_cmd;
 mod upgrade;
 mod user_defaults;
 mod db;
@@ -213,7 +214,7 @@ fn execute<Cr, Rr>(
 		Cmd::SignerSign { id, pwfile, port, authfile } => cli_signer::signer_sign(id, pwfile, port, authfile).map(|s| ExecutionAction::Instant(Some(s))),
 		Cmd::SignerList { port, authfile } => cli_signer::signer_list(port, authfile).map(|s| ExecutionAction::Instant(Some(s))),
 		Cmd::SignerReject { id, port, authfile } => cli_signer::signer_reject(id, port, authfile).map(|s| ExecutionAction::Instant(Some(s))),
-		Cmd::Snapshot(snapshot_cmd) => snapshot::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))),
+		Cmd::Snapshot(snapshot_cmd) => snapshot_cmd::execute(snapshot_cmd).map(|s| ExecutionAction::Instant(Some(s))),
 		Cmd::ExportHardcodedSync(export_hs_cmd) => export_hardcoded_sync::execute(export_hs_cmd).map(|s| ExecutionAction::Instant(Some(s))),
 	}
 }

parity/modules.rs

@@ -16,15 +16,14 @@
 
 use std::sync::{Arc, mpsc};
 
-use client_traits::BlockChainClient;
+use client_traits::{BlockChainClient, ChainNotify};
 use sync::{self, SyncConfig, NetworkConfiguration, Params, ConnectionFilter};
-use ethcore::snapshot::SnapshotService;
+use snapshot::SnapshotService;
 use ethcore_private_tx::PrivateStateDB;
 use light::Provider;
 use parity_runtime::Executor;
 
 pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler};
-pub use ethcore::client::ChainNotify;
 use ethcore_logger::Config as LogConfig;
 
 pub type SyncModules = (

parity/rpc_apis.rs

@@ -24,7 +24,7 @@ pub use parity_rpc::signer::SignerService;
 use account_utils::{self, AccountProvider};
 use ethcore::client::Client;
 use ethcore::miner::Miner;
-use ethcore::snapshot::SnapshotService;
+use snapshot::SnapshotService;
 use client_traits::BlockChainClient;
 use sync::SyncState;
 use ethcore_logger::RotatingLogger;

parity/run.rs

@@ -25,7 +25,7 @@ use call_contract::CallContract;
 use client_traits::{BlockInfo, BlockChainClient};
 use ethcore::client::{Client, DatabaseCompactionProfile, VMType};
 use ethcore::miner::{self, stratum, Miner, MinerService, MinerOptions};
-use ethcore::snapshot::{self, SnapshotConfiguration};
+use snapshot::{self, SnapshotConfiguration};
 use spec::SpecParams;
 use verification::queue::VerifierSettings;
 use ethcore_logger::{Config as LogConfig, RotatingLogger};

parity/snapshot_cmd.rs

@@ -22,10 +22,10 @@ use std::sync::Arc;
 
 use client_traits::SnapshotClient;
 use hash::keccak;
-use ethcore::snapshot::{SnapshotConfiguration, SnapshotService as SS};
-use ethcore::snapshot::io::{SnapshotReader, PackedReader, PackedWriter};
-use ethcore::snapshot::service::Service as SnapshotService;
-use ethcore::client::{DatabaseCompactionProfile, VMType};
+use snapshot::{SnapshotConfiguration, SnapshotService as SS};
+use snapshot::io::{SnapshotReader, PackedReader, PackedWriter};
+use snapshot::service::Service as SnapshotService;
+use ethcore::client::{Client, DatabaseCompactionProfile, VMType};
 use ethcore::miner::Miner;
 use ethcore_service::ClientService;
 use types::{
@@ -73,7 +73,7 @@ pub struct SnapshotCommand {
 
 // helper for reading chunks from arbitrary reader and feeding them into the
 // service.
-fn restore_using<R: SnapshotReader>(snapshot: Arc<SnapshotService>, reader: &R, recover: bool) -> Result<(), String> {
+fn restore_using<R: SnapshotReader>(snapshot: Arc<SnapshotService<Client>>, reader: &R, recover: bool) -> Result<(), String> {
 	let manifest = reader.manifest();
 
 	info!("Restoring to block #{} (0x{:?})", manifest.block_number, manifest.block_hash);