From 2969d015ed3a4b042e2a26b35410dc2fff87f7b0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 May 2016 20:29:19 +0200 Subject: [PATCH] Importing (#1132) * Basic hex block exporting. * Export formats and to file. * First bits. * Block importing. * Fix error text. * Wait for queue to empty before shutting down after import. --- parity/cli.rs | 14 ++-- parity/main.rs | 119 ++++++++++++++++++++++++++++++++-- util/src/rlp/untrusted_rlp.rs | 52 ++++++++------- 3 files changed, 149 insertions(+), 36 deletions(-) diff --git a/parity/cli.rs b/parity/cli.rs index e0edec29f..9b0e8c0be 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -24,6 +24,7 @@ Parity. Ethereum Client. Usage: parity daemon [options] parity account (new | list) [options] + parity import [ ] [options] parity export [ ] [options] parity [options] @@ -143,13 +144,13 @@ Footprint Options: the entire system, overrides other cache and queue options. -Export Options: +Import/Export Options: --from BLOCK Export from block BLOCK, which may be an index or - hash [default: 0]. - --to BLOCK Export to (including) block NUMBER, which may be an + hash [default: 1]. + --to BLOCK Export to (including) block BLOCK, which may be an index, hash or 'latest' [default: latest]. - --format FORMAT Export in given format. FORMAT must be one of 'hex' - and 'binary' [default: hex]. + --format FORMAT For import/export in given format. FORMAT must be + one of 'hex' and 'binary'. Virtual Machine Options: --jitvm Enable the JIT VM. @@ -196,6 +197,7 @@ pub struct Args { pub cmd_new: bool, pub cmd_list: bool, pub cmd_export: bool, + pub cmd_import: bool, pub arg_pid_file: String, pub arg_file: Option, pub flag_chain: String, @@ -243,7 +245,7 @@ pub struct Args { pub flag_version: bool, pub flag_from: String, pub flag_to: String, - pub flag_format: String, + pub flag_format: Option, pub flag_jitvm: bool, // legacy... pub flag_geth: bool, diff --git a/parity/main.rs b/parity/main.rs index b36bfbe58..da94a13ea 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -66,8 +66,11 @@ mod configuration; use ctrlc::CtrlC; use util::*; use std::fs::File; +use std::thread::yield_now; +use std::io::{BufReader, BufRead}; use util::panics::{MayPanic, ForwardPanic, PanicHandler}; use ethcore::client::{BlockID, BlockChainClient}; +use ethcore::error::{Error, ImportError}; use ethcore::service::ClientService; use ethsync::EthSync; use ethminer::{Miner, MinerService, ExternalMiner}; @@ -111,6 +114,11 @@ fn execute(conf: Configuration) { return; } + if conf.args.cmd_import { + execute_import(conf); + return; + } + execute_client(conf); } @@ -229,8 +237,6 @@ enum DataFormat { } fn execute_export(conf: Configuration) { - println!("Exporting to {:?} from {}, to {}", conf.args.arg_file, conf.args.flag_from, conf.args.flag_to); - // Setup panic handler let panic_handler = PanicHandler::new_in_arc(); @@ -276,10 +282,14 @@ fn execute_export(conf: Configuration) { }; let from = parse_block_id(&conf.args.flag_from, "--from"); let to = parse_block_id(&conf.args.flag_to, "--to"); - let format = match conf.args.flag_format.deref() { - "binary" | "bin" => DataFormat::Binary, - "hex" => DataFormat::Hex, - x => die!("Invalid --format parameter given: {:?}", x), + let format = match conf.args.flag_format { + Some(x) => match x.deref() { + "binary" | "bin" => DataFormat::Binary, + "hex" => DataFormat::Hex, + x => die!("Invalid --format parameter given: {:?}", x), + }, + None if conf.args.arg_file.is_none() => DataFormat::Hex, + None => DataFormat::Binary, }; let mut out: Box = if let Some(f) = conf.args.arg_file { @@ -297,6 +307,103 @@ fn execute_export(conf: Configuration) { } } +fn execute_import(conf: Configuration) { + // Setup panic handler + let panic_handler = PanicHandler::new_in_arc(); + + // Raise fdlimit + unsafe { ::fdlimit::raise_fd_limit(); } + + let spec = conf.spec(); + let net_settings = NetworkConfiguration { + config_path: None, + listen_address: None, + public_address: None, + udp_port: None, + nat_enabled: false, + discovery_enabled: false, + pin: true, + boot_nodes: Vec::new(), + use_secret: None, + ideal_peers: 0, + }; + let client_config = conf.client_config(&spec); + + // Build client + let service = ClientService::start( + client_config, spec, net_settings, Path::new(&conf.path()) + ).unwrap_or_else(|e| die_with_error("Client", e)); + + panic_handler.forward_from(&service); + let client = service.client(); + + let mut instream: Box = if let Some(f) = conf.args.arg_file { + let f = File::open(&f).unwrap_or_else(|_| die!("Cannot open the file given: {}", f)); + Box::new(f) + } else { + Box::new(::std::io::stdin()) + }; + + let mut first_bytes: Bytes = vec![0; 3]; + let mut first_read = 0; + + let format = match conf.args.flag_format { + Some(x) => match x.deref() { + "binary" | "bin" => DataFormat::Binary, + "hex" => DataFormat::Hex, + x => die!("Invalid --format parameter given: {:?}", x), + }, + None => { + // autodetect... + first_read = instream.read(&mut(first_bytes[..])).unwrap_or_else(|_| die!("Error reading from the file/stream.")); + match first_bytes[0] { + 0xf9 => { + println!("Autodetected binary data format."); + DataFormat::Binary + } + _ => { + println!("Autodetected hex data format."); + DataFormat::Hex + } + } + } + }; + + let do_import = |bytes| { + while client.queue_info().is_full() { yield_now(); } + match client.import_block(bytes) { + Ok(_) => { println!("Block imported ok"); } + Err(Error::Import(ImportError::AlreadyInChain)) => { trace!("Skipping block already in chain."); } + Err(e) => die!("Cannot import block: {:?}", e) + } + }; + + match format { + DataFormat::Binary => { + loop { + let mut bytes: Bytes = if first_read > 0 {first_bytes.clone()} else {vec![0; 3]}; + let n = if first_read > 0 {first_read} else {instream.read(&mut(bytes[..])).unwrap_or_else(|_| die!("Error reading from the file/stream."))}; + if n == 0 { break; } + first_read = 0; + let s = PayloadInfo::from(&(bytes[..])).unwrap_or_else(|e| die!("Invalid RLP in the file/stream: {:?}", e)).total(); + bytes.resize(s, 0); + instream.read_exact(&mut(bytes[3..])).unwrap_or_else(|_| die!("Error reading from the file/stream.")); + do_import(bytes); + } + } + DataFormat::Hex => { + for line in BufReader::new(instream).lines() { + let s = line.unwrap_or_else(|_| die!("Error reading from the file/stream.")); + let s = if first_read > 0 {str::from_utf8(&first_bytes).unwrap().to_owned() + &(s[..])} else {s}; + first_read = 0; + let bytes = FromHex::from_hex(&(s[..])).unwrap_or_else(|_| die!("Invalid hex in file/stream.")); + do_import(bytes); + } + } + } + client.flush_queue(); +} + fn execute_account_cli(conf: Configuration) { use util::keys::store::SecretStore; use rpassword::read_password; diff --git a/util/src/rlp/untrusted_rlp.rs b/util/src/rlp/untrusted_rlp.rs index 957a09b61..1aa688aba 100644 --- a/util/src/rlp/untrusted_rlp.rs +++ b/util/src/rlp/untrusted_rlp.rs @@ -65,6 +65,32 @@ impl PayloadInfo { /// Total size of the RLP. pub fn total(&self) -> usize { self.header_len + self.value_len } + + /// Create a new object from the given bytes RLP. The bytes + pub fn from(header_bytes: &[u8]) -> Result { + Ok(match header_bytes.first().cloned() { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let header_len = 1 + len_of_len; + if header_bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); } + let value_len = try!(usize::from_bytes(&header_bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&header_bytes[1..header_len])); + if header_bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); } + PayloadInfo::new(header_len, value_len) + }, + // we cant reach this place, but rust requires _ to be implemented + _ => { unreachable!(); } + }) + } } /// Data-oriented view onto rlp-slice. @@ -305,31 +331,9 @@ impl<'a> BasicDecoder<'a> { } } - /// Return first item info + /// Return first item info. fn payload_info(bytes: &[u8]) -> Result { - let item = match bytes.first().cloned() { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => PayloadInfo::new(0, 1), - Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let header_len = 1 + len_of_len; - if bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); } - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - } - Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - if bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); } - PayloadInfo::new(header_len, value_len) - }, - // we cant reach this place, but rust requires _ to be implemented - _ => { unreachable!(); } - }; - + let item = try!(PayloadInfo::from(bytes)); match item.header_len + item.value_len <= bytes.len() { true => Ok(item), false => Err(DecoderError::RlpIsTooShort),