grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Multithreaded snapshot creation (#9239)

Browse Source 
* Add Progress to Snapshot Secondary chunks creation

* Use half of CPUs to multithread snapshot creation

* Use env var to define number of threads

* info to debug logs

* Add Snapshot threads as CLI option

* Randomize chunks per thread

* Remove randomness, add debugging

* Add warning

* Add tracing

* Use parity-common fix seek branch

* Fix log

* Fix tests

* Fix tests

* PR Grumbles

* PR Grumble II

* Update Cargo.lock

* PR Grumbles

* Default snapshot threads to half number of CPUs

* Fix default snapshot threads // min 1
This commit is contained in:
Nicolas Gotchac
2018-09-13 12:58:49 +02:00
committed by Afri Schoedon
parent ef4a61c769
commit 4ddd69cc55
12 changed files with 454 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
parity/configuration.rs
View File

@@ -30,8 +30,10 @@ use sync::{NetworkConfiguration, validate_node_url, self};
use ethcore::ethstore::ethkey::{Secret, Public};
use ethcore::client::{VMType};
use ethcore::miner::{stratum, MinerOptions};
use ethcore::snapshot::SnapshotConfiguration;
use ethcore::verification::queue::VerifierSettings;
use miner::pool;
use num_cpus;
use rpc::{IpcConfiguration, HttpConfiguration, WsConfiguration};
use parity_rpc::NetworkSettings;
@@ -125,6 +127,7 @@ impl Configuration {
let update_policy = self.update_policy()?;
let logger_config = self.logger_config();
let ws_conf = self.ws_config()?;
let snapshot_conf = self.snapshot_config()?;
let http_conf = self.http_config()?;
let ipc_conf = self.ipc_config()?;
let net_conf = self.net_config()?;
@@ -298,6 +301,7 @@ impl Configuration {
file_path: self.args.arg_snapshot_file.clone(),
kind: snapshot::Kind::Take,
block_at: to_block_id(&self.args.arg_snapshot_at)?,
snapshot_conf: snapshot_conf,
};
Cmd::Snapshot(snapshot_cmd)
} else if self.args.cmd_restore {
@@ -314,6 +318,7 @@ impl Configuration {
file_path: self.args.arg_restore_file.clone(),
kind: snapshot::Kind::Restore,
block_at: to_block_id("latest")?, // unimportant.
snapshot_conf: snapshot_conf,
};
Cmd::Snapshot(restore_cmd)
} else if self.args.cmd_export_hardcoded_sync {
@@ -349,6 +354,7 @@ impl Configuration {
gas_price_percentile: self.args.arg_gas_price_percentile,
poll_lifetime: self.args.arg_poll_lifetime,
ws_conf: ws_conf,
snapshot_conf: snapshot_conf,
http_conf: http_conf,
ipc_conf: ipc_conf,
net_conf: net_conf,
@@ -374,7 +380,6 @@ impl Configuration {
private_tx_enabled,
name: self.args.arg_identity,
custom_bootnodes: self.args.arg_bootnodes.is_some(),
no_periodic_snapshot: self.args.flag_no_periodic_snapshot,
check_seal: !self.args.flag_no_seal_check,
download_old_blocks: !self.args.flag_no_ancient_blocks,
verifier_settings: verifier_settings,
@@ -892,6 +897,18 @@ impl Configuration {
Ok((provider_conf, encryptor_conf, self.args.flag_private_enabled))
}
fn snapshot_config(&self) -> Result<SnapshotConfiguration, String> {
let conf = SnapshotConfiguration {
no_periodic: self.args.flag_no_periodic_snapshot,
processing_threads: match self.args.arg_snapshot_threads {
Some(threads) if threads > 0 => threads,
_ => ::std::cmp::max(1, num_cpus::get() / 2),
},
};
Ok(conf)
}
fn network_settings(&self) -> Result<NetworkSettings, String> {
let http_conf = self.http_config()?;
let net_addresses = self.net_addresses()?;
@@ -1400,7 +1417,7 @@ mod tests {
name: "".into(),
custom_bootnodes: false,
fat_db: Default::default(),
no_periodic_snapshot: false,
snapshot_conf: Default::default(),
stratum: None,
check_seal: true,
download_old_blocks: true,