Merge pull request #3636 from ethcore/fatdb

Return of the Fat DB

parity/blockchain.rs

@@ -22,7 +22,7 @@ use std::thread::sleep;
 use std::sync::Arc;
 use rustc_serialize::hex::FromHex;
 use io::{PanicHandler, ForwardPanic};
-use util::{ToPretty, Uint};
+use util::{ToPretty, Uint, U256, H256, Address, Hashable};
 use rlp::PayloadInfo;
 use ethcore::service::ClientService;
 use ethcore::client::{Mode, DatabaseCompactionProfile, VMType, BlockImportError, BlockChainClient, BlockID};
@@ -65,6 +65,7 @@ impl FromStr for DataFormat {
 pub enum BlockchainCmd {
 	Import(ImportBlockchain),
 	Export(ExportBlockchain),
+	ExportState(ExportState),
 }
 
 #[derive(Debug, PartialEq)]
@@ -103,10 +104,31 @@ pub struct ExportBlockchain {
 	pub check_seal: bool,
 }
 
+#[derive(Debug, PartialEq)]
+pub struct ExportState {
+	pub spec: SpecType,
+	pub cache_config: CacheConfig,
+	pub dirs: Directories,
+	pub file_path: Option<String>,
+	pub format: Option<DataFormat>,
+	pub pruning: Pruning,
+	pub pruning_history: u64,
+	pub compaction: DatabaseCompactionProfile,
+	pub wal: bool,
+	pub fat_db: Switch,
+	pub tracing: Switch,
+	pub at: BlockID,
+	pub storage: bool,
+	pub code: bool,
+	pub min_balance: Option<U256>,
+	pub max_balance: Option<U256>,
+}
+
 pub fn execute(cmd: BlockchainCmd) -> Result<String, String> {
 	match cmd {
 		BlockchainCmd::Import(import_cmd) => execute_import(import_cmd),
 		BlockchainCmd::Export(export_cmd) => execute_export(export_cmd),
+		BlockchainCmd::ExportState(export_cmd) => execute_export_state(export_cmd),
 	}
 }
 
@@ -245,6 +267,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
 	// save user defaults
 	user_defaults.pruning = algorithm;
 	user_defaults.tracing = tracing;
+	user_defaults.fat_db = fat_db;
 	try!(user_defaults.save(&user_defaults_path));
 
 	let report = client.report();
@@ -261,23 +284,28 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
 	).into())
 }
 
-fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
-	// Setup panic handler
-	let panic_handler = PanicHandler::new_in_arc();
+fn start_client(
+	dirs: Directories,
+	spec: SpecType,
+	pruning: Pruning,
+	pruning_history: u64,
+	tracing: Switch,
+	fat_db: Switch,
+	compaction: DatabaseCompactionProfile,
+	wal: bool,
+	cache_config: CacheConfig) -> Result<ClientService, String> {
 
 	// create dirs used by parity
-	try!(cmd.dirs.create_dirs(false, false));
-
-	let format = cmd.format.unwrap_or_default();
+	try!(dirs.create_dirs(false, false));
 
 	// load spec file
-	let spec = try!(cmd.spec.spec());
+	let spec = try!(spec.spec());
 
 	// load genesis hash
 	let genesis_hash = spec.genesis_header().hash();
 
 	// database paths
-	let db_dirs = cmd.dirs.database(genesis_hash, spec.fork_name.clone());
+	let db_dirs = dirs.database(genesis_hash, spec.fork_name.clone());
 
 	// user defaults path
 	let user_defaults_path = db_dirs.user_defaults_path();
@@ -288,34 +316,42 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
 	fdlimit::raise_fd_limit();
 
 	// select pruning algorithm
-	let algorithm = cmd.pruning.to_algorithm(&user_defaults);
+	let algorithm = pruning.to_algorithm(&user_defaults);
 
 	// check if tracing is on
-	let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
+	let tracing = try!(tracing_switch_to_bool(tracing, &user_defaults));
 
 	// check if fatdb is on
-	let fat_db = try!(fatdb_switch_to_bool(cmd.fat_db, &user_defaults, algorithm));
+	let fat_db = try!(fatdb_switch_to_bool(fat_db, &user_defaults, algorithm));
 
 	// prepare client and snapshot paths.
 	let client_path = db_dirs.client_path(algorithm);
 	let snapshot_path = db_dirs.snapshot_path();
 
 	// execute upgrades
-	try!(execute_upgrades(&db_dirs, algorithm, cmd.compaction.compaction_profile(db_dirs.fork_path().as_path())));
+	try!(execute_upgrades(&db_dirs, algorithm, compaction.compaction_profile(db_dirs.fork_path().as_path())));
 
 	// prepare client config
-	let client_config = to_client_config(&cmd.cache_config, Mode::Active, tracing, fat_db, cmd.compaction, cmd.wal, VMType::default(), "".into(), algorithm, cmd.pruning_history, cmd.check_seal);
+	let client_config = to_client_config(&cache_config, Mode::Active, tracing, fat_db, compaction, wal, VMType::default(), "".into(), algorithm, pruning_history, true);
 
 	let service = try!(ClientService::start(
 		client_config,
 		&spec,
 		&client_path,
 		&snapshot_path,
-		&cmd.dirs.ipc_path(),
+		&dirs.ipc_path(),
 		Arc::new(Miner::with_spec(&spec)),
 	).map_err(|e| format!("Client service error: {:?}", e)));
 
 	drop(spec);
+	Ok(service)
+}
+
+fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
+	// Setup panic handler
+	let service = try!(start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config));
+	let panic_handler = PanicHandler::new_in_arc();
+	let format = cmd.format.unwrap_or_default();
 
 	panic_handler.forward_from(&service);
 	let client = service.client();
@@ -329,6 +365,9 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
 	let to = try!(client.block_number(cmd.to_block).ok_or("To block could not be found"));
 
 	for i in from..(to + 1) {
+		if i % 10000 == 0 {
+			info!("#{}", i);
+		}
 		let b = try!(client.block(BlockID::Number(i)).ok_or("Error exporting incomplete chain"));
 		match format {
 			DataFormat::Binary => { out.write(&b).expect("Couldn't write to stream."); }
@@ -339,6 +378,85 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
 	Ok("Export completed.".into())
 }
 
+fn execute_export_state(cmd: ExportState) -> Result<String, String> {
+	// Setup panic handler
+	let service = try!(start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config));
+	let panic_handler = PanicHandler::new_in_arc();
+
+	panic_handler.forward_from(&service);
+	let client = service.client();
+
+	let mut out: Box<io::Write> = match cmd.file_path {
+		Some(f) => Box::new(try!(fs::File::create(&f).map_err(|_| format!("Cannot write to file given: {}", f)))),
+		None => Box::new(io::stdout()),
+	};
+
+	let mut last: Option<Address> = None;
+	let at = cmd.at;
+	let mut i = 0usize;
+
+	out.write_fmt(format_args!("{{ \"state\": [", )).expect("Couldn't write to stream.");
+	loop {
+		let accounts = try!(client.list_accounts(at, last.as_ref(), 1000).ok_or("Specified block not found"));
+		if accounts.is_empty() {
+			break;
+		}
+
+		for account in accounts.into_iter() {
+			let balance = client.balance(&account, at).unwrap_or_else(U256::zero);
+			if cmd.min_balance.map_or(false, |m| balance < m) || cmd.max_balance.map_or(false, |m| balance > m) {
+				last = Some(account);
+				continue; //filtered out
+			}
+
+			if i != 0 {
+				out.write(b",").expect("Write error");
+			}
+			out.write_fmt(format_args!("\n\"0x{}\": {{\"balance\": \"{:x}\", \"nonce\": \"{:x}\"", account.hex(), balance, client.nonce(&account, at).unwrap_or_else(U256::zero))).expect("Write error");
+			let code = client.code(&account, at).unwrap_or(None).unwrap_or_else(Vec::new);
+			if !code.is_empty() {
+				out.write_fmt(format_args!(", \"code_hash\": \"0x{}\"", code.sha3().hex())).expect("Write error");
+				if cmd.code {
+					out.write_fmt(format_args!(", \"code\": \"{}\"", code.to_hex())).expect("Write error");
+				}
+			}
+			let storage_root = client.storage_root(&account, at).unwrap_or(::util::SHA3_NULL_RLP);
+			if storage_root != ::util::SHA3_NULL_RLP {
+				out.write_fmt(format_args!(", \"storage_root\": \"0x{}\"", storage_root.hex())).expect("Write error");
+				if cmd.storage {
+					out.write_fmt(format_args!(", \"storage\": {{")).expect("Write error");
+					let mut last_storage: Option<H256> = None;
+					loop {
+						let keys = try!(client.list_storage(at, &account, last_storage.as_ref(), 1000).ok_or("Specified block not found"));
+						if keys.is_empty() {
+							break;
+						}
+
+						let mut si = 0;
+						for key in keys.into_iter() {
+							if si != 0 {
+								out.write(b",").expect("Write error");
+							}
+							out.write_fmt(format_args!("\n\t\"0x{}\": \"0x{}\"", key.hex(), client.storage_at(&account, &key, at).unwrap_or_else(Default::default).hex())).expect("Write error");
+							si += 1;
+							last_storage = Some(key);
+						}
+					}
+					out.write(b"\n}").expect("Write error");
+				}
+			}
+			out.write(b"}").expect("Write error");
+			i += 1;
+			if i % 10000 == 0 {
+				info!("Account #{}", i);
+			}
+			last = Some(account);
+		}
+	}
+	out.write_fmt(format_args!("\n]}}")).expect("Write error");
+	Ok("Export completed.".into())
+}
+
 #[cfg(test)]
 mod test {
 	use super::DataFormat;

parity/cli/mod.rs

@@ -26,6 +26,8 @@ usage! {
 		cmd_new: bool,
 		cmd_list: bool,
 		cmd_export: bool,
+		cmd_blocks: bool,
+		cmd_state: bool,
 		cmd_import: bool,
 		cmd_signer: bool,
 		cmd_new_token: bool,
@@ -246,6 +248,10 @@ usage! {
 		flag_to: String = "latest", or |_| None,
 		flag_format: Option<String> = None, or |_| None,
 		flag_no_seal_check: bool = false, or |_| None,
+		flag_no_storage: bool = false, or |_| None,
+		flag_no_code: bool = false, or |_| None,
+		flag_min_balance: Option<String> = None, or |_| None,
+		flag_max_balance: Option<String> = None, or |_| None,
 
 		// -- Snapshot Optons
 		flag_at: String = "latest", or |_| None,
@@ -484,6 +490,8 @@ mod tests {
 			cmd_new: false,
 			cmd_list: false,
 			cmd_export: false,
+			cmd_state: false,
+			cmd_blocks: false,
 			cmd_import: false,
 			cmd_signer: false,
 			cmd_new_token: false,
@@ -600,6 +608,10 @@ mod tests {
 			flag_to: "latest".into(),
 			flag_format: None,
 			flag_no_seal_check: false,
+			flag_no_code: false,
+			flag_no_storage: false,
+			flag_min_balance: None,
+			flag_max_balance: None,
 
 			// -- Snapshot Optons
 			flag_at: "latest".into(),

parity/cli/usage.txt

@@ -10,7 +10,7 @@ Usage:
   parity account import <path>... [options]
   parity wallet import <path> --password FILE [options]
   parity import [ <file> ] [options]
-  parity export [ <file> ] [options]
+  parity export (blocks | state) [ <file> ] [options]
   parity signer new-token [options]
   parity snapshot <file> [options]
   parity restore [ <file> ] [options]
@@ -271,6 +271,16 @@ Import/Export Options:
                            one of 'hex' and 'binary'.
                            (default: {flag_format:?} = Import: auto, Export: binary)
   --no-seal-check          Skip block seal check. (default: {flag_no_seal_check})
+  --at BLOCK               Export state at the given block, which may be an
+                           index, hash, or 'latest'. Note that taking snapshots at
+                           non-recent blocks will only work with --pruning archive
+                           (default: {flag_at})
+  --no-storage             Don't export account storge. (default: {flag_no_storage})
+  --no-code                Don't export account code. (default: {flag_no_code})
+  --min-balance WEI        Don't export accounts with balance less than specified.
+                           (default: {flag_min_balance:?})
+  --max-balance WEI        Don't export accounts with balance greater than specified.
+                           (default: {flag_max_balance:?})
 
 Snapshot Options:
   --at BLOCK               Take a snapshot at the given block, which may be an

parity/configuration.rs

@@ -37,7 +37,7 @@ use dir::Directories;
 use dapps::Configuration as DappsConfiguration;
 use signer::{Configuration as SignerConfiguration};
 use run::RunCmd;
-use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat};
+use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, ExportState, DataFormat};
 use presale::ImportWallet;
 use account::{AccountCmd, NewAccount, ImportAccounts, ImportFromGethAccounts};
 use snapshot::{self, SnapshotCommand};
@@ -161,23 +161,47 @@ impl Configuration {
 			};
 			Cmd::Blockchain(BlockchainCmd::Import(import_cmd))
 		} else if self.args.cmd_export {
-			let export_cmd = ExportBlockchain {
-				spec: spec,
-				cache_config: cache_config,
-				dirs: dirs,
-				file_path: self.args.arg_file.clone(),
-				format: format,
-				pruning: pruning,
-				pruning_history: pruning_history,
-				compaction: compaction,
-				wal: wal,
-				tracing: tracing,
-				fat_db: fat_db,
-				from_block: try!(to_block_id(&self.args.flag_from)),
-				to_block: try!(to_block_id(&self.args.flag_to)),
-				check_seal: !self.args.flag_no_seal_check,
-			};
-			Cmd::Blockchain(BlockchainCmd::Export(export_cmd))
+			if self.args.cmd_blocks {
+				let export_cmd = ExportBlockchain {
+					spec: spec,
+					cache_config: cache_config,
+					dirs: dirs,
+					file_path: self.args.arg_file.clone(),
+					format: format,
+					pruning: pruning,
+					pruning_history: pruning_history,
+					compaction: compaction,
+					wal: wal,
+					tracing: tracing,
+					fat_db: fat_db,
+					from_block: try!(to_block_id(&self.args.flag_from)),
+					to_block: try!(to_block_id(&self.args.flag_to)),
+					check_seal: !self.args.flag_no_seal_check,
+				};
+				Cmd::Blockchain(BlockchainCmd::Export(export_cmd))
+			} else if self.args.cmd_state {
+				let export_cmd = ExportState {
+					spec: spec,
+					cache_config: cache_config,
+					dirs: dirs,
+					file_path: self.args.arg_file.clone(),
+					format: format,
+					pruning: pruning,
+					pruning_history: pruning_history,
+					compaction: compaction,
+					wal: wal,
+					tracing: tracing,
+					fat_db: fat_db,
+					at: try!(to_block_id(&self.args.flag_at)),
+					storage: !self.args.flag_no_storage,
+					code: !self.args.flag_no_code,
+					min_balance: self.args.flag_min_balance.and_then(|s| to_u256(&s).ok()),
+					max_balance: self.args.flag_max_balance.and_then(|s| to_u256(&s).ok()),
+				};
+				Cmd::Blockchain(BlockchainCmd::ExportState(export_cmd))
+			} else {
+				unreachable!();
+			}
 		} else if self.args.cmd_snapshot {
 			let snapshot_cmd = SnapshotCommand {
 				cache_config: cache_config,
@@ -690,7 +714,7 @@ mod tests {
 	use helpers::{replace_home, default_network_config};
 	use run::RunCmd;
 	use signer::{Configuration as SignerConfiguration};
-	use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat};
+	use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat, ExportState};
 	use presale::ImportWallet;
 	use account::{AccountCmd, NewAccount, ImportAccounts};
 	use devtools::{RandomTempPath};
@@ -779,7 +803,7 @@ mod tests {
 
 	#[test]
 	fn test_command_blockchain_export() {
-		let args = vec!["parity", "export", "blockchain.json"];
+		let args = vec!["parity", "export", "blocks", "blockchain.json"];
 		let conf = parse(&args);
 		assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::Export(ExportBlockchain {
 			spec: Default::default(),
@@ -799,9 +823,33 @@ mod tests {
 		})));
 	}
 
+	#[test]
+	fn test_command_state_export() {
+		let args = vec!["parity", "export", "state", "state.json"];
+		let conf = parse(&args);
+		assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::ExportState(ExportState {
+			spec: Default::default(),
+			cache_config: Default::default(),
+			dirs: Default::default(),
+			file_path: Some("state.json".into()),
+			pruning: Default::default(),
+			pruning_history: 64,
+			format: Default::default(),
+			compaction: Default::default(),
+			wal: true,
+			tracing: Default::default(),
+			fat_db: Default::default(),
+			at: BlockID::Latest,
+			storage: true,
+			code: true,
+			min_balance: None,
+			max_balance: None,
+		})));
+	}
+
 	#[test]
 	fn test_command_blockchain_export_with_custom_format() {
-		let args = vec!["parity", "export", "--format", "hex", "blockchain.json"];
+		let args = vec!["parity", "export", "blocks", "--format", "hex", "blockchain.json"];
 		let conf = parse(&args);
 		assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::Export(ExportBlockchain {
 			spec: Default::default(),

parity/params.rs

@@ -257,17 +257,13 @@ pub fn tracing_switch_to_bool(switch: Switch, user_defaults: &UserDefaults) -> R
 	}
 }
 
-pub fn fatdb_switch_to_bool(switch: Switch, user_defaults: &UserDefaults, algorithm: Algorithm) -> Result<bool, String> {
+pub fn fatdb_switch_to_bool(switch: Switch, user_defaults: &UserDefaults, _algorithm: Algorithm) -> Result<bool, String> {
 	let result = match (user_defaults.is_first_launch, switch, user_defaults.fat_db) {
 		(false, Switch::On, false) => Err("FatDB resync required".into()),
 		(_, Switch::On, _) => Ok(true),
 		(_, Switch::Off, _) => Ok(false),
 		(_, Switch::Auto, def) => Ok(def),
 	};
-
-	if result.clone().unwrap_or(false) && algorithm != Algorithm::Archive {
-		return Err("Fat DB is not supported with the chosen pruning option. Please rerun with `--pruning=archive`".into());
-	}
 	result
 }
 

parity/run.rs

@@ -219,7 +219,7 @@ pub fn execute(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<(), String> {
 	// create client config
 	let client_config = to_client_config(
 		&cmd.cache_config,
-		mode,
+		mode.clone(),
 		tracing,
 		fat_db,
 		cmd.compaction,
@@ -354,6 +354,8 @@ pub fn execute(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<(), String> {
 	// save user defaults
 	user_defaults.pruning = algorithm;
 	user_defaults.tracing = tracing;
+	user_defaults.fat_db = fat_db;
+	user_defaults.mode = mode;
 	try!(user_defaults.save(&user_defaults_path));
 
 	let on_mode_change = move |mode: &Mode| {