Merge branch 'master' into check-updates

This commit is contained in:
Gav Wood
2016-12-04 09:01:50 -08:00
280 changed files with 5818 additions and 4269 deletions

View File

@@ -22,7 +22,7 @@ use std::thread::sleep;
use std::sync::Arc;
use rustc_serialize::hex::FromHex;
use io::{PanicHandler, ForwardPanic};
use util::{ToPretty, Uint};
use util::{ToPretty, Uint, U256, H256, Address, Hashable};
use rlp::PayloadInfo;
use ethcore::service::ClientService;
use ethcore::client::{Mode, DatabaseCompactionProfile, VMType, BlockImportError, BlockChainClient, BlockId};
@@ -65,6 +65,7 @@ impl FromStr for DataFormat {
pub enum BlockchainCmd {
Import(ImportBlockchain),
Export(ExportBlockchain),
ExportState(ExportState),
}
#[derive(Debug, PartialEq)]
@@ -103,10 +104,31 @@ pub struct ExportBlockchain {
pub check_seal: bool,
}
#[derive(Debug, PartialEq)]
pub struct ExportState {
pub spec: SpecType,
pub cache_config: CacheConfig,
pub dirs: Directories,
pub file_path: Option<String>,
pub format: Option<DataFormat>,
pub pruning: Pruning,
pub pruning_history: u64,
pub compaction: DatabaseCompactionProfile,
pub wal: bool,
pub fat_db: Switch,
pub tracing: Switch,
pub at: BlockID,
pub storage: bool,
pub code: bool,
pub min_balance: Option<U256>,
pub max_balance: Option<U256>,
}
pub fn execute(cmd: BlockchainCmd) -> Result<String, String> {
match cmd {
BlockchainCmd::Import(import_cmd) => execute_import(import_cmd),
BlockchainCmd::Export(export_cmd) => execute_export(export_cmd),
BlockchainCmd::ExportState(export_cmd) => execute_export_state(export_cmd),
}
}
@@ -245,6 +267,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
// save user defaults
user_defaults.pruning = algorithm;
user_defaults.tracing = tracing;
user_defaults.fat_db = fat_db;
try!(user_defaults.save(&user_defaults_path));
let report = client.report();
@@ -261,23 +284,29 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
).into())
}
fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
// Setup panic handler
let panic_handler = PanicHandler::new_in_arc();
fn start_client(
dirs: Directories,
spec: SpecType,
pruning: Pruning,
pruning_history: u64,
tracing: Switch,
fat_db: Switch,
compaction: DatabaseCompactionProfile,
wal: bool,
cache_config: CacheConfig
) -> Result<ClientService, String> {
// create dirs used by parity
try!(cmd.dirs.create_dirs(false, false));
let format = cmd.format.unwrap_or_default();
try!(dirs.create_dirs(false, false));
// load spec file
let spec = try!(cmd.spec.spec());
let spec = try!(spec.spec());
// load genesis hash
let genesis_hash = spec.genesis_header().hash();
// database paths
let db_dirs = cmd.dirs.database(genesis_hash, spec.fork_name.clone());
let db_dirs = dirs.database(genesis_hash, spec.fork_name.clone());
// user defaults path
let user_defaults_path = db_dirs.user_defaults_path();
@@ -288,34 +317,42 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
fdlimit::raise_fd_limit();
// select pruning algorithm
let algorithm = cmd.pruning.to_algorithm(&user_defaults);
let algorithm = pruning.to_algorithm(&user_defaults);
// check if tracing is on
let tracing = try!(tracing_switch_to_bool(cmd.tracing, &user_defaults));
let tracing = try!(tracing_switch_to_bool(tracing, &user_defaults));
// check if fatdb is on
let fat_db = try!(fatdb_switch_to_bool(cmd.fat_db, &user_defaults, algorithm));
let fat_db = try!(fatdb_switch_to_bool(fat_db, &user_defaults, algorithm));
// prepare client and snapshot paths.
let client_path = db_dirs.client_path(algorithm);
let snapshot_path = db_dirs.snapshot_path();
// execute upgrades
try!(execute_upgrades(&db_dirs, algorithm, cmd.compaction.compaction_profile(db_dirs.fork_path().as_path())));
try!(execute_upgrades(&db_dirs, algorithm, compaction.compaction_profile(db_dirs.fork_path().as_path())));
// prepare client config
let client_config = to_client_config(&cmd.cache_config, Default::default(), Mode::Active, tracing, fat_db, cmd.compaction, cmd.wal, VMType::default(), "".into(), algorithm, cmd.pruning_history, cmd.check_seal);
let client_config = to_client_config(&cache_config, Default::default(), Mode::Active, tracing, fat_db, compaction, wal, VMType::default(), "".into(), algorithm, pruning_history, true);
let service = try!(ClientService::start(
client_config,
&spec,
&client_path,
&snapshot_path,
&cmd.dirs.ipc_path(),
&dirs.ipc_path(),
Arc::new(Miner::with_spec(&spec)),
).map_err(|e| format!("Client service error: {:?}", e)));
drop(spec);
Ok(service)
}
fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
// Setup panic handler
let service = try!(start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config));
let panic_handler = PanicHandler::new_in_arc();
let format = cmd.format.unwrap_or_default();
panic_handler.forward_from(&service);
let client = service.client();
@@ -329,6 +366,9 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
let to = try!(client.block_number(cmd.to_block).ok_or("To block could not be found"));
for i in from..(to + 1) {
if i % 10000 == 0 {
info!("#{}", i);
}
let b = try!(client.block(BlockId::Number(i)).ok_or("Error exporting incomplete chain"));
match format {
DataFormat::Binary => { out.write(&b).expect("Couldn't write to stream."); }
@@ -339,6 +379,85 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
Ok("Export completed.".into())
}
fn execute_export_state(cmd: ExportState) -> Result<String, String> {
// Setup panic handler
let service = try!(start_client(cmd.dirs, cmd.spec, cmd.pruning, cmd.pruning_history, cmd.tracing, cmd.fat_db, cmd.compaction, cmd.wal, cmd.cache_config));
let panic_handler = PanicHandler::new_in_arc();
panic_handler.forward_from(&service);
let client = service.client();
let mut out: Box<io::Write> = match cmd.file_path {
Some(f) => Box::new(try!(fs::File::create(&f).map_err(|_| format!("Cannot write to file given: {}", f)))),
None => Box::new(io::stdout()),
};
let mut last: Option<Address> = None;
let at = cmd.at;
let mut i = 0usize;
out.write_fmt(format_args!("{{ \"state\": [", )).expect("Couldn't write to stream.");
loop {
let accounts = try!(client.list_accounts(at, last.as_ref(), 1000).ok_or("Specified block not found"));
if accounts.is_empty() {
break;
}
for account in accounts.into_iter() {
let balance = client.balance(&account, at).unwrap_or_else(U256::zero);
if cmd.min_balance.map_or(false, |m| balance < m) || cmd.max_balance.map_or(false, |m| balance > m) {
last = Some(account);
continue; //filtered out
}
if i != 0 {
out.write(b",").expect("Write error");
}
out.write_fmt(format_args!("\n\"0x{}\": {{\"balance\": \"{:x}\", \"nonce\": \"{:x}\"", account.hex(), balance, client.nonce(&account, at).unwrap_or_else(U256::zero))).expect("Write error");
let code = client.code(&account, at).unwrap_or(None).unwrap_or_else(Vec::new);
if !code.is_empty() {
out.write_fmt(format_args!(", \"code_hash\": \"0x{}\"", code.sha3().hex())).expect("Write error");
if cmd.code {
out.write_fmt(format_args!(", \"code\": \"{}\"", code.to_hex())).expect("Write error");
}
}
let storage_root = client.storage_root(&account, at).unwrap_or(::util::SHA3_NULL_RLP);
if storage_root != ::util::SHA3_NULL_RLP {
out.write_fmt(format_args!(", \"storage_root\": \"0x{}\"", storage_root.hex())).expect("Write error");
if cmd.storage {
out.write_fmt(format_args!(", \"storage\": {{")).expect("Write error");
let mut last_storage: Option<H256> = None;
loop {
let keys = try!(client.list_storage(at, &account, last_storage.as_ref(), 1000).ok_or("Specified block not found"));
if keys.is_empty() {
break;
}
let mut si = 0;
for key in keys.into_iter() {
if si != 0 {
out.write(b",").expect("Write error");
}
out.write_fmt(format_args!("\n\t\"0x{}\": \"0x{}\"", key.hex(), client.storage_at(&account, &key, at).unwrap_or_else(Default::default).hex())).expect("Write error");
si += 1;
last_storage = Some(key);
}
}
out.write(b"\n}").expect("Write error");
}
}
out.write(b"}").expect("Write error");
i += 1;
if i % 10000 == 0 {
info!("Account #{}", i);
}
last = Some(account);
}
}
out.write_fmt(format_args!("\n]}}")).expect("Write error");
Ok("Export completed.".into())
}
#[cfg(test)]
mod test {
use super::DataFormat;

View File

@@ -0,0 +1,3 @@
[signer]
passwd = []

View File

@@ -26,6 +26,8 @@ usage! {
cmd_new: bool,
cmd_list: bool,
cmd_export: bool,
cmd_blocks: bool,
cmd_state: bool,
cmd_import: bool,
cmd_signer: bool,
cmd_new_token: bool,
@@ -249,6 +251,10 @@ usage! {
flag_to: String = "latest", or |_| None,
flag_format: Option<String> = None, or |_| None,
flag_no_seal_check: bool = false, or |_| None,
flag_no_storage: bool = false, or |_| None,
flag_no_code: bool = false, or |_| None,
flag_min_balance: Option<String> = None, or |_| None,
flag_max_balance: Option<String> = None, or |_| None,
// -- Snapshot Optons
flag_at: String = "latest", or |_| None,
@@ -490,6 +496,8 @@ mod tests {
cmd_new: false,
cmd_list: false,
cmd_export: false,
cmd_state: false,
cmd_blocks: false,
cmd_import: false,
cmd_signer: false,
cmd_new_token: false,
@@ -609,6 +617,10 @@ mod tests {
flag_to: "latest".into(),
flag_format: None,
flag_no_seal_check: false,
flag_no_code: false,
flag_no_storage: false,
flag_min_balance: None,
flag_max_balance: None,
// -- Snapshot Optons
flag_at: "latest".into(),
@@ -658,11 +670,12 @@ mod tests {
fn should_parse_config_and_return_errors() {
let config1 = Args::parse_config(include_str!("./config.invalid1.toml"));
let config2 = Args::parse_config(include_str!("./config.invalid2.toml"));
let config3 = Args::parse_config(include_str!("./config.invalid3.toml"));
match (config1, config2) {
(Err(ArgsError::Parsing(_)), Err(ArgsError::Decode(_))) => {},
(a, b) => {
assert!(false, "Got invalid error types: {:?}, {:?}", a, b);
match (config1, config2, config3) {
(Err(ArgsError::Parsing(_)), Err(ArgsError::Decode(_)), Err(ArgsError::UnknownFields(_))) => {},
(a, b, c) => {
assert!(false, "Got invalid error types: {:?}, {:?}, {:?}", a, b, c);
}
}
}

View File

@@ -58,6 +58,7 @@ macro_rules! usage {
Parsing(Vec<toml::ParserError>),
Decode(toml::DecodeError),
Config(String, io::Error),
UnknownFields(String),
}
impl ArgsError {
@@ -80,6 +81,11 @@ macro_rules! usage {
println_stderr!("There was an error reading your config file at: {}", path);
println_stderr!("{}", e);
process::exit(2)
},
ArgsError::UnknownFields(fields) => {
println_stderr!("You have some extra fields in your config file:");
println_stderr!("{}", fields);
process::exit(2)
}
}
}
@@ -173,10 +179,13 @@ macro_rules! usage {
let mut value_parser = toml::Parser::new(&config);
match value_parser.parse() {
Some(value) => {
let result = rustc_serialize::Decodable::decode(&mut toml::Decoder::new(toml::Value::Table(value)));
match result {
Ok(config) => Ok(config),
Err(e) => Err(e.into()),
let mut decoder = toml::Decoder::new(toml::Value::Table(value));
let result = rustc_serialize::Decodable::decode(&mut decoder);
match (result, decoder.toml) {
(Err(e), _) => Err(e.into()),
(_, Some(toml)) => Err(ArgsError::UnknownFields(toml::encode_str(&toml))),
(Ok(config), None) => Ok(config),
}
},
None => Err(ArgsError::Parsing(value_parser.errors)),

View File

@@ -10,7 +10,7 @@ Usage:
parity account import <path>... [options]
parity wallet import <path> --password FILE [options]
parity import [ <file> ] [options]
parity export [ <file> ] [options]
parity export (blocks | state) [ <file> ] [options]
parity signer new-token [options]
parity snapshot <file> [options]
parity restore [ <file> ] [options]
@@ -284,6 +284,16 @@ Import/Export Options:
one of 'hex' and 'binary'.
(default: {flag_format:?} = Import: auto, Export: binary)
--no-seal-check Skip block seal check. (default: {flag_no_seal_check})
--at BLOCK Export state at the given block, which may be an
index, hash, or 'latest'. Note that taking snapshots at
non-recent blocks will only work with --pruning archive
(default: {flag_at})
--no-storage Don't export account storge. (default: {flag_no_storage})
--no-code Don't export account code. (default: {flag_no_code})
--min-balance WEI Don't export accounts with balance less than specified.
(default: {flag_min_balance:?})
--max-balance WEI Don't export accounts with balance greater than specified.
(default: {flag_max_balance:?})
Snapshot Options:
--at BLOCK Take a snapshot at the given block, which may be an

View File

@@ -37,7 +37,7 @@ use dir::Directories;
use dapps::Configuration as DappsConfiguration;
use signer::{Configuration as SignerConfiguration};
use run::RunCmd;
use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat};
use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, ExportState, DataFormat};
use presale::ImportWallet;
use account::{AccountCmd, NewAccount, ImportAccounts, ImportFromGethAccounts};
use snapshot::{self, SnapshotCommand};
@@ -162,23 +162,47 @@ impl Configuration {
};
Cmd::Blockchain(BlockchainCmd::Import(import_cmd))
} else if self.args.cmd_export {
let export_cmd = ExportBlockchain {
spec: spec,
cache_config: cache_config,
dirs: dirs,
file_path: self.args.arg_file.clone(),
format: format,
pruning: pruning,
pruning_history: pruning_history,
compaction: compaction,
wal: wal,
tracing: tracing,
fat_db: fat_db,
from_block: try!(to_block_id(&self.args.flag_from)),
to_block: try!(to_block_id(&self.args.flag_to)),
check_seal: !self.args.flag_no_seal_check,
};
Cmd::Blockchain(BlockchainCmd::Export(export_cmd))
if self.args.cmd_blocks {
let export_cmd = ExportBlockchain {
spec: spec,
cache_config: cache_config,
dirs: dirs,
file_path: self.args.arg_file.clone(),
format: format,
pruning: pruning,
pruning_history: pruning_history,
compaction: compaction,
wal: wal,
tracing: tracing,
fat_db: fat_db,
from_block: try!(to_block_id(&self.args.flag_from)),
to_block: try!(to_block_id(&self.args.flag_to)),
check_seal: !self.args.flag_no_seal_check,
};
Cmd::Blockchain(BlockchainCmd::Export(export_cmd))
} else if self.args.cmd_state {
let export_cmd = ExportState {
spec: spec,
cache_config: cache_config,
dirs: dirs,
file_path: self.args.arg_file.clone(),
format: format,
pruning: pruning,
pruning_history: pruning_history,
compaction: compaction,
wal: wal,
tracing: tracing,
fat_db: fat_db,
at: try!(to_block_id(&self.args.flag_at)),
storage: !self.args.flag_no_storage,
code: !self.args.flag_no_code,
min_balance: self.args.flag_min_balance.and_then(|s| to_u256(&s).ok()),
max_balance: self.args.flag_max_balance.and_then(|s| to_u256(&s).ok()),
};
Cmd::Blockchain(BlockchainCmd::ExportState(export_cmd))
} else {
unreachable!();
}
} else if self.args.cmd_snapshot {
let snapshot_cmd = SnapshotCommand {
cache_config: cache_config,
@@ -706,7 +730,7 @@ mod tests {
use helpers::{replace_home, default_network_config};
use run::RunCmd;
use signer::{Configuration as SignerConfiguration};
use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat};
use blockchain::{BlockchainCmd, ImportBlockchain, ExportBlockchain, DataFormat, ExportState};
use presale::ImportWallet;
use account::{AccountCmd, NewAccount, ImportAccounts};
use devtools::{RandomTempPath};
@@ -795,7 +819,7 @@ mod tests {
#[test]
fn test_command_blockchain_export() {
let args = vec!["parity", "export", "blockchain.json"];
let args = vec!["parity", "export", "blocks", "blockchain.json"];
let conf = parse(&args);
assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::Export(ExportBlockchain {
spec: Default::default(),
@@ -815,9 +839,33 @@ mod tests {
})));
}
#[test]
fn test_command_state_export() {
let args = vec!["parity", "export", "state", "state.json"];
let conf = parse(&args);
assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::ExportState(ExportState {
spec: Default::default(),
cache_config: Default::default(),
dirs: Default::default(),
file_path: Some("state.json".into()),
pruning: Default::default(),
pruning_history: 64,
format: Default::default(),
compaction: Default::default(),
wal: true,
tracing: Default::default(),
fat_db: Default::default(),
at: BlockID::Latest,
storage: true,
code: true,
min_balance: None,
max_balance: None,
})));
}
#[test]
fn test_command_blockchain_export_with_custom_format() {
let args = vec!["parity", "export", "--format", "hex", "blockchain.json"];
let args = vec!["parity", "export", "blocks", "--format", "hex", "blockchain.json"];
let conf = parse(&args);
assert_eq!(conf.into_command().unwrap().cmd, Cmd::Blockchain(BlockchainCmd::Export(ExportBlockchain {
spec: Default::default(),

View File

@@ -44,8 +44,6 @@ extern crate serde_json;
extern crate rlp;
extern crate ethcore_hash_fetch as hash_fetch;
extern crate json_ipc_server as jsonipc;
extern crate ethcore_ipc_hypervisor as hypervisor;
extern crate ethcore_rpc;

View File

@@ -257,17 +257,13 @@ pub fn tracing_switch_to_bool(switch: Switch, user_defaults: &UserDefaults) -> R
}
}
pub fn fatdb_switch_to_bool(switch: Switch, user_defaults: &UserDefaults, algorithm: Algorithm) -> Result<bool, String> {
pub fn fatdb_switch_to_bool(switch: Switch, user_defaults: &UserDefaults, _algorithm: Algorithm) -> Result<bool, String> {
let result = match (user_defaults.is_first_launch, switch, user_defaults.fat_db) {
(false, Switch::On, false) => Err("FatDB resync required".into()),
(_, Switch::On, _) => Ok(true),
(_, Switch::Off, _) => Ok(false),
(_, Switch::Auto, def) => Ok(def),
};
if result.clone().unwrap_or(false) && algorithm != Algorithm::Archive {
return Err("Fat DB is not supported with the chosen pruning option. Please rerun with `--pruning=archive`".into());
}
result
}

View File

@@ -19,14 +19,12 @@ use std::sync::Arc;
use std::net::SocketAddr;
use std::io;
use io::PanicHandler;
use ethcore_rpc::{RpcServerError, RpcServer as Server};
use jsonipc;
use ethcore_rpc::{RpcServerError, RpcServer as Server, IpcServerError};
use rpc_apis;
use rpc_apis::ApiSet;
use helpers::parity_ipc_path;
pub use jsonipc::Server as IpcServer;
pub use ethcore_rpc::Server as HttpServer;
pub use ethcore_rpc::{IpcServer, Server as HttpServer};
#[derive(Debug, PartialEq)]
pub struct HttpConfiguration {
@@ -126,7 +124,7 @@ pub fn new_ipc(conf: IpcConfiguration, deps: &Dependencies) -> Result<Option<Ipc
pub fn setup_ipc_rpc_server(dependencies: &Dependencies, addr: &str, apis: ApiSet) -> Result<IpcServer, String> {
let server = try!(setup_rpc_server(apis, dependencies));
match server.start_ipc(addr) {
Err(jsonipc::Error::Io(io_error)) => Err(format!("RPC io error: {}", io_error)),
Err(IpcServerError::Io(io_error)) => Err(format!("RPC io error: {}", io_error)),
Err(any_error) => Err(format!("Rpc error: {:?}", any_error)),
Ok(server) => Ok(server)
}

View File

@@ -158,7 +158,7 @@ pub fn execute(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<(), String> {
// get the mode
let mode = try!(mode_switch_to_bool(cmd.mode, &user_defaults));
trace!(target: "mode", "mode is {:?}", mode);
let network_enabled = match &mode { &Mode::Dark(_) | &Mode::Off => false, _ => true, };
let network_enabled = match mode { Mode::Dark(_) | Mode::Off => false, _ => true, };
// get the update policy
let update_policy = cmd.update_policy;
@@ -225,7 +225,7 @@ pub fn execute(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<(), String> {
let client_config = to_client_config(
&cmd.cache_config,
update_policy,
mode,
mode.clone(),
tracing,
fat_db,
cmd.compaction,
@@ -360,6 +360,8 @@ pub fn execute(cmd: RunCmd, logger: Arc<RotatingLogger>) -> Result<(), String> {
// save user defaults
user_defaults.pruning = algorithm;
user_defaults.tracing = tracing;
user_defaults.fat_db = fat_db;
user_defaults.mode = mode;
try!(user_defaults.save(&user_defaults_path));
let on_mode_change = move |mode: &Mode| {