Importing (#1132)

* Basic hex block exporting.

* Export formats and to file.

* First bits.

* Block importing.

* Fix error text.

* Wait for queue to empty before shutting down after import.

parity/cli.rs

@@ -24,6 +24,7 @@ Parity. Ethereum Client.
 Usage:
   parity daemon <pid-file> [options]
   parity account (new | list) [options]
+  parity import [ <file> ] [options]
   parity export [ <file> ] [options]
   parity [options]
 
@@ -143,13 +144,13 @@ Footprint Options:
                            the entire system, overrides other cache and queue
                            options.
 
-Export Options:
+Import/Export Options:
   --from BLOCK             Export from block BLOCK, which may be an index or
-                           hash [default: 0].
+                           hash [default: 1].
-  --to BLOCK               Export to (including) block NUMBER, which may be an
+  --to BLOCK               Export to (including) block BLOCK, which may be an
                            index, hash or 'latest' [default: latest].
-  --format FORMAT          Export in given format. FORMAT must be one of 'hex'
+  --format FORMAT          For import/export in given format. FORMAT must be
-                           and 'binary' [default: hex].
+                           one of 'hex' and 'binary'.
 
 Virtual Machine Options:
   --jitvm                  Enable the JIT VM.
@@ -196,6 +197,7 @@ pub struct Args {
 	pub cmd_new: bool,
 	pub cmd_list: bool,
 	pub cmd_export: bool,
+	pub cmd_import: bool,
 	pub arg_pid_file: String,
 	pub arg_file: Option<String>,
 	pub flag_chain: String,
@@ -243,7 +245,7 @@ pub struct Args {
 	pub flag_version: bool,
 	pub flag_from: String,
 	pub flag_to: String,
-	pub flag_format: String,
+	pub flag_format: Option<String>,
 	pub flag_jitvm: bool,
 	// legacy...
 	pub flag_geth: bool,

parity/main.rs

@@ -66,8 +66,11 @@ mod configuration;
 use ctrlc::CtrlC;
 use util::*;
 use std::fs::File;
+use std::thread::yield_now;
+use std::io::{BufReader, BufRead};
 use util::panics::{MayPanic, ForwardPanic, PanicHandler};
 use ethcore::client::{BlockID, BlockChainClient};
+use ethcore::error::{Error, ImportError};
 use ethcore::service::ClientService;
 use ethsync::EthSync;
 use ethminer::{Miner, MinerService, ExternalMiner};
@@ -111,6 +114,11 @@ fn execute(conf: Configuration) {
 		return;
 	}
 
+	if conf.args.cmd_import {
+		execute_import(conf);
+		return;
+	}
+
 	execute_client(conf);
 }
 
@@ -229,8 +237,6 @@ enum DataFormat {
 }
 
 fn execute_export(conf: Configuration) {
-	println!("Exporting to {:?} from {}, to {}", conf.args.arg_file, conf.args.flag_from, conf.args.flag_to);
-
 	// Setup panic handler
 	let panic_handler = PanicHandler::new_in_arc();
 
@@ -276,10 +282,14 @@ fn execute_export(conf: Configuration) {
 	};
 	let from = parse_block_id(&conf.args.flag_from, "--from");
 	let to = parse_block_id(&conf.args.flag_to, "--to");
-	let format = match conf.args.flag_format.deref() {
+	let format = match conf.args.flag_format {
-		"binary" | "bin" => DataFormat::Binary,
+		Some(x) => match x.deref() {
-		"hex" => DataFormat::Hex,
+			"binary" | "bin" => DataFormat::Binary,
-		x => die!("Invalid --format parameter given: {:?}", x),
+			"hex" => DataFormat::Hex,
+			x => die!("Invalid --format parameter given: {:?}", x),
+		},
+		None if conf.args.arg_file.is_none() => DataFormat::Hex,
+		None => DataFormat::Binary,
 	};
 
 	let mut out: Box<Write> = if let Some(f) = conf.args.arg_file {
@@ -297,6 +307,103 @@ fn execute_export(conf: Configuration) {
 	}
 }
 
+fn execute_import(conf: Configuration) {
+	// Setup panic handler
+	let panic_handler = PanicHandler::new_in_arc();
+
+	// Raise fdlimit
+	unsafe { ::fdlimit::raise_fd_limit(); }
+
+	let spec = conf.spec();
+	let net_settings = NetworkConfiguration {
+		config_path: None,
+		listen_address: None,
+		public_address: None,
+		udp_port: None,
+		nat_enabled: false,
+		discovery_enabled: false,
+		pin: true,
+		boot_nodes: Vec::new(),
+		use_secret: None,
+		ideal_peers: 0,
+	};
+	let client_config = conf.client_config(&spec);
+
+	// Build client
+	let service = ClientService::start(
+		client_config, spec, net_settings, Path::new(&conf.path())
+	).unwrap_or_else(|e| die_with_error("Client", e));
+
+	panic_handler.forward_from(&service);
+	let client = service.client();
+
+	let mut instream: Box<Read> = if let Some(f) = conf.args.arg_file {
+		let f = File::open(&f).unwrap_or_else(|_| die!("Cannot open the file given: {}", f));
+		Box::new(f)
+	} else {
+		Box::new(::std::io::stdin())
+	};
+
+	let mut first_bytes: Bytes = vec![0; 3];
+	let mut first_read = 0;
+
+	let format = match conf.args.flag_format {
+		Some(x) => match x.deref() {
+			"binary" | "bin" => DataFormat::Binary,
+			"hex" => DataFormat::Hex,
+			x => die!("Invalid --format parameter given: {:?}", x),
+		},
+		None => {
+			// autodetect...
+			first_read = instream.read(&mut(first_bytes[..])).unwrap_or_else(|_| die!("Error reading from the file/stream."));
+			match first_bytes[0] {
+				0xf9 => {
+					println!("Autodetected binary data format.");
+					DataFormat::Binary
+				}
+				_ => {
+					println!("Autodetected hex data format.");
+					DataFormat::Hex
+				}
+			}
+		}
+	};
+
+	let do_import = |bytes| {
+		while client.queue_info().is_full() { yield_now(); }
+		match client.import_block(bytes) {
+			Ok(_) => { println!("Block imported ok"); }
+			Err(Error::Import(ImportError::AlreadyInChain)) => { trace!("Skipping block already in chain."); }
+			Err(e) => die!("Cannot import block: {:?}", e)
+		}
+	};
+
+	match format {
+		DataFormat::Binary => {
+			loop {
+				let mut bytes: Bytes = if first_read > 0 {first_bytes.clone()} else {vec![0; 3]};
+				let n = if first_read > 0 {first_read} else {instream.read(&mut(bytes[..])).unwrap_or_else(|_| die!("Error reading from the file/stream."))};
+				if n == 0 { break; }
+				first_read = 0;
+				let s = PayloadInfo::from(&(bytes[..])).unwrap_or_else(|e| die!("Invalid RLP in the file/stream: {:?}", e)).total();
+				bytes.resize(s, 0);
+				instream.read_exact(&mut(bytes[3..])).unwrap_or_else(|_| die!("Error reading from the file/stream."));
+				do_import(bytes);
+			}
+		}
+		DataFormat::Hex => { 
+			for line in BufReader::new(instream).lines() {
+				let s = line.unwrap_or_else(|_| die!("Error reading from the file/stream."));
+				let s = if first_read > 0 {str::from_utf8(&first_bytes).unwrap().to_owned() + &(s[..])} else {s};
+				first_read = 0;
+				let bytes = FromHex::from_hex(&(s[..])).unwrap_or_else(|_| die!("Invalid hex in file/stream."));
+				do_import(bytes);
+			}
+		}
+	}
+	client.flush_queue();
+}
+
 fn execute_account_cli(conf: Configuration) {
 	use util::keys::store::SecretStore;
 	use rpassword::read_password;

util/src/rlp/untrusted_rlp.rs

@@ -65,6 +65,32 @@ impl PayloadInfo {
 
 	/// Total size of the RLP.
 	pub fn total(&self) -> usize { self.header_len + self.value_len }
+
+	/// Create a new object from the given bytes RLP. The bytes
+	pub fn from(header_bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
+		Ok(match header_bytes.first().cloned() {
+			None => return Err(DecoderError::RlpIsTooShort),
+			Some(0...0x7f) => PayloadInfo::new(0, 1),
+			Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80),
+			Some(l @ 0xb8...0xbf) => {
+				let len_of_len = l as usize - 0xb7;
+				let header_len = 1 + len_of_len;
+				if header_bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); }
+				let value_len = try!(usize::from_bytes(&header_bytes[1..header_len]));
+				PayloadInfo::new(header_len, value_len)
+			}
+			Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0),
+			Some(l @ 0xf8...0xff) => {
+				let len_of_len = l as usize - 0xf7;
+				let header_len = 1 + len_of_len;
+				let value_len = try!(usize::from_bytes(&header_bytes[1..header_len]));
+				if header_bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); }
+				PayloadInfo::new(header_len, value_len)
+			},
+			// we cant reach this place, but rust requires _ to be implemented
+			_ => { unreachable!(); }
+		})
+	}
 }
 
 /// Data-oriented view onto rlp-slice.
@@ -305,31 +331,9 @@ impl<'a> BasicDecoder<'a> {
 		}
 	}
 
-	/// Return first item info
+	/// Return first item info.
 	fn payload_info(bytes: &[u8]) -> Result<PayloadInfo, DecoderError> {
-		let item = match bytes.first().cloned() {
+		let item = try!(PayloadInfo::from(bytes));
-			None => return Err(DecoderError::RlpIsTooShort),
-			Some(0...0x7f) => PayloadInfo::new(0, 1),
-			Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80),
-			Some(l @ 0xb8...0xbf) => {
-				let len_of_len = l as usize - 0xb7;
-				let header_len = 1 + len_of_len;
-				if bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); }
-				let value_len = try!(usize::from_bytes(&bytes[1..header_len]));
-				PayloadInfo::new(header_len, value_len)
-			}
-			Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0),
-			Some(l @ 0xf8...0xff) => {
-				let len_of_len = l as usize - 0xf7;
-				let header_len = 1 + len_of_len;
-				let value_len = try!(usize::from_bytes(&bytes[1..header_len]));
-				if bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); }
-				PayloadInfo::new(header_len, value_len)
-			},
-			// we cant reach this place, but rust requires _ to be implemented
-			_ => { unreachable!(); }
-		};
-
 		match item.header_len + item.value_len <= bytes.len() {
 			true => Ok(item),
 			false => Err(DecoderError::RlpIsTooShort),