grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Snapshot sync part 2 (#2098)

Browse Source 
* Split block downloader into a module

* Snapshot sync progress

* Warp sync CLI option

* Increased snapshot chunk and ping timeouts

* Fixed an issue with delayed writes

* Updated bootnodes

* Don't run pending IO tasks on shutdown

* Optional first_block; removed insert_snapshot_block

* Fixing expect calls

* Fixed stalled sync

* style and docs

* Update block_sync.rs

[ci:skip]
This commit is contained in:
Arkadiy Paronyan
2016-10-18 18:16:00 +02:00
committed by GitHub
parent dba2d79b56
commit 487dfb0208
36 changed files with 1347 additions and 528 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parity/blockchain.rs
View File

@@ -194,7 +194,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
}
};
let informant = Informant::new(client.clone(), None, None, cmd.logger_config.color);
let informant = Informant::new(client.clone(), None, None, None, cmd.logger_config.color);
try!(service.register_io_handler(Arc::new(ImportIoHandler {
info: Arc::new(informant),

1
parity/cli/config.full.toml
View File

@@ -28,6 +28,7 @@ nat = "any"
id = "0x1"
bootnodes = []
discovery = true
warp = true
reserved_only = false
reserved_peers = "./path_to_file"

1
parity/cli/config.toml
View File

@@ -13,6 +13,7 @@ disable = true
[network]
disable = false
warp = false
discovery = true
nat = "any"
min_peers = 10

5
parity/cli/mod.rs
View File

@@ -106,6 +106,8 @@ usage! {
// -- Networking Options
flag_no_network: bool = false,
or |c: &Config| otry!(c.network).disable.clone(),
flag_warp: bool = false,
or |c: &Config| otry!(c.network).warp.clone(),
flag_port: u16 = 30303u16,
or |c: &Config| otry!(c.network).port.clone(),
flag_min_peers: u16 = 25u16,
@@ -300,6 +302,7 @@ struct Signer {
#[derive(Default, Debug, PartialEq, RustcDecodable)]
struct Network {
disable: Option<bool>,
warp: Option<bool>,
port: Option<u16>,
min_peers: Option<u16>,
max_peers: Option<u16>,
@@ -486,6 +489,7 @@ mod tests {
// -- Networking Options
flag_no_network: false,
flag_warp: true,
flag_port: 30303u16,
flag_min_peers: 25u16,
flag_max_peers: 50u16,
@@ -643,6 +647,7 @@ mod tests {
}),
network: Some(Network {
disable: Some(false),
warp: Some(false),
port: None,
min_peers: Some(10),
max_peers: Some(20),

1
parity/cli/usage.txt
View File

@@ -67,6 +67,7 @@ Account Options:
Networking Options:
--no-network Disable p2p networking. (default: {flag_no_network})
--warp Enable syncing from the snapshot over the network. (default: {flag_warp})
--port PORT Override the port on which the node should listen
(default: {flag_port}).
--min-peers NUM Try to maintain at least NUM peers (default: {flag_min_peers}).

3
parity/configuration.rs
View File

@@ -89,6 +89,7 @@ impl Configuration {
let compaction = try!(self.args.flag_db_compaction.parse());
let wal = !self.args.flag_fast_and_loose;
let enable_network = self.enable_network(&mode);
let warp_sync = self.args.flag_warp;
let geth_compatibility = self.args.flag_geth;
let signer_port = self.signer_port();
let dapps_conf = self.dapps_config();
@@ -240,6 +241,7 @@ impl Configuration {
wal: wal,
vm_type: vm_type,
enable_network: enable_network,
warp_sync: warp_sync,
geth_compatibility: geth_compatibility,
signer_port: signer_port,
net_settings: self.network_settings(),
@@ -810,6 +812,7 @@ mod tests {
ipc_conf: Default::default(),
net_conf: default_network_config(),
network_id: None,
warp_sync: false,
acc_conf: Default::default(),
gas_pricer: Default::default(),
miner_extras: Default::default(),

54
parity/informant.rs
View File

@@ -26,6 +26,8 @@ use ethsync::{SyncProvider, ManageNetwork};
use util::{Uint, RwLock, Mutex, H256, Colour};
use ethcore::client::*;
use ethcore::views::BlockView;
use ethcore::snapshot::service::Service as SnapshotService;
use ethcore::snapshot::{RestorationStatus, SnapshotService as SS};
use number_prefix::{binary_prefix, Standalone, Prefixed};
pub struct Informant {
@@ -35,6 +37,7 @@ pub struct Informant {
last_tick: RwLock<Instant>,
with_color: bool,
client: Arc<Client>,
snapshot: Option<Arc<SnapshotService>>,
sync: Option<Arc<SyncProvider>>,
net: Option<Arc<ManageNetwork>>,
last_import: Mutex<Instant>,
@@ -55,7 +58,7 @@ impl MillisecondDuration for Duration {
impl Informant {
/// Make a new instance potentially `with_color` output.
pub fn new(client: Arc<Client>, sync: Option<Arc<SyncProvider>>, net: Option<Arc<ManageNetwork>>, with_color: bool) -> Self {
pub fn new(client: Arc<Client>, sync: Option<Arc<SyncProvider>>, net: Option<Arc<ManageNetwork>>, snapshot: Option<Arc<SnapshotService>>, with_color: bool) -> Self {
Informant {
chain_info: RwLock::new(None),
cache_info: RwLock::new(None),
@@ -63,6 +66,7 @@ impl Informant {
last_tick: RwLock::new(Instant::now()),
with_color: with_color,
client: client,
snapshot: snapshot,
sync: sync,
net: net,
last_import: Mutex::new(Instant::now()),
@@ -92,8 +96,16 @@ impl Informant {
let sync_status = self.sync.as_ref().map(|s| s.status());
let importing = queue_info.unverified_queue_size + queue_info.verified_queue_size > 3
|| self.sync.as_ref().map_or(false, |s| s.status().is_major_syncing());
if !importing && elapsed < Duration::from_secs(30) {
|| sync_status.map_or(false, |s| s.is_major_syncing());
let (snapshot_sync, snapshot_current, snapshot_total) = self.snapshot.as_ref().map_or((false, 0, 0), |s|
match s.status() {
RestorationStatus::Ongoing { state_chunks, block_chunks, state_chunks_done, block_chunks_done } =>
(true, state_chunks_done + block_chunks_done, state_chunks + block_chunks),
_ => (false, 0, 0),
}
);
if !importing && !snapshot_sync && elapsed < Duration::from_secs(30) {
return;
}
@@ -109,27 +121,33 @@ impl Informant {
info!(target: "import", "{} {} {}",
match importing {
true => format!("Syncing {} {} {} {}+{} Qed",
paint(White.bold(), format!("{:>8}", format!("#{}", chain_info.best_block_number))),
paint(White.bold(), format!("{}", chain_info.best_block_hash)),
{
let last_report = match *write_report { Some(ref last_report) => last_report.clone(), _ => ClientReport::default() };
format!("{} blk/s {} tx/s {} Mgas/s",
paint(Yellow.bold(), format!("{:4}", ((report.blocks_imported - last_report.blocks_imported) * 1000) as u64 / elapsed.as_milliseconds())),
paint(Yellow.bold(), format!("{:4}", ((report.transactions_applied - last_report.transactions_applied) * 1000) as u64 / elapsed.as_milliseconds())),
paint(Yellow.bold(), format!("{:3}", ((report.gas_processed - last_report.gas_processed) / From::from(elapsed.as_milliseconds() * 1000)).low_u64()))
)
},
paint(Green.bold(), format!("{:5}", queue_info.unverified_queue_size)),
paint(Green.bold(), format!("{:5}", queue_info.verified_queue_size))
),
true => match snapshot_sync {
false => format!("Syncing {} {} {} {}+{} Qed",
paint(White.bold(), format!("{:>8}", format!("#{}", chain_info.best_block_number))),
paint(White.bold(), format!("{}", chain_info.best_block_hash)),
{
let last_report = match *write_report { Some(ref last_report) => last_report.clone(), _ => ClientReport::default() };
format!("{} blk/s {} tx/s {} Mgas/s",
paint(Yellow.bold(), format!("{:4}", ((report.blocks_imported - last_report.blocks_imported) * 1000) as u64 / elapsed.as_milliseconds())),
paint(Yellow.bold(), format!("{:4}", ((report.transactions_applied - last_report.transactions_applied) * 1000) as u64 / elapsed.as_milliseconds())),
paint(Yellow.bold(), format!("{:3}", ((report.gas_processed - last_report.gas_processed) / From::from(elapsed.as_milliseconds() * 1000)).low_u64()))
)
},
paint(Green.bold(), format!("{:5}", queue_info.unverified_queue_size)),
paint(Green.bold(), format!("{:5}", queue_info.verified_queue_size))
),
true => format!("Syncing snapshot {}/{}", snapshot_current, snapshot_total),
},
false => String::new(),
},
match (&sync_status, &network_config) {
(&Some(ref sync_info), &Some(ref net_config)) => format!("{}{}/{}/{} peers",
match importing {
true => format!("{} ", paint(Green.bold(), format!("{:>8}", format!("#{}", sync_info.last_imported_block_number.unwrap_or(chain_info.best_block_number))))),
false => String::new(),
false => match sync_info.last_imported_old_block_number {
Some(number) => format!("{} ", paint(Yellow.bold(), format!("{:>8}", format!("#{}", number)))),
None => String::new(),
}
},
paint(Cyan.bold(), format!("{:2}", sync_info.num_active_peers)),
paint(Cyan.bold(), format!("{:2}", sync_info.num_peers)),

2
parity/io_handler.rs
View File

@@ -61,4 +61,4 @@ impl IoHandler<ClientIoMessage> for ImportIoHandler {
self.info.tick()
}
}
}
}

12
parity/run.rs
View File

@@ -70,6 +70,7 @@ pub struct RunCmd {
pub ipc_conf: IpcConfiguration,
pub net_conf: NetworkConfiguration,
pub network_id: Option<U256>,
pub warp_sync: bool,
pub acc_conf: AccountsConfig,
pub gas_pricer: GasPricerConfig,
pub miner_extras: MinerExtras,
@@ -171,6 +172,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
sync_config.subprotocol_name.clone_from_slice(spec.subprotocol_name().as_bytes());
}
sync_config.fork_block = spec.fork_block();
sync_config.warp_sync = cmd.warp_sync;
// prepare account provider
let account_provider = Arc::new(try!(prepare_account_provider(&cmd.dirs, cmd.acc_conf)));
@@ -231,7 +233,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
// create sync object
let (sync_provider, manage_network, chain_notify) = try!(modules::sync(
&mut hypervisor, sync_config, net_conf.into(), client.clone(), snapshot_service, &cmd.logger_config,
&mut hypervisor, sync_config, net_conf.into(), client.clone(), snapshot_service.clone(), &cmd.logger_config,
).map_err(|e| format!("Sync error: {}", e)));
service.add_notify(chain_notify.clone());
@@ -287,7 +289,13 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
// start signer server
let signer_server = try!(signer::start(cmd.signer_conf, signer_deps));
let informant = Arc::new(Informant::new(service.client(), Some(sync_provider.clone()), Some(manage_network.clone()), cmd.logger_config.color));
let informant = Arc::new(Informant::new(
service.client(),
Some(sync_provider.clone()),
Some(manage_network.clone()),
Some(snapshot_service.clone()),
cmd.logger_config.color
));
let info_notify: Arc<ChainNotify> = informant.clone();
service.add_notify(info_notify);
let io_handler = Arc::new(ClientIoHandler {

2
parity/snapshot.rs
View File

@@ -81,7 +81,7 @@ fn restore_using<R: SnapshotReader>(snapshot: Arc<SnapshotService>, reader: &R,
let informant_handle = snapshot.clone();
::std::thread::spawn(move || {
while let RestorationStatus::Ongoing { state_chunks_done, block_chunks_done } = informant_handle.status() {
while let RestorationStatus::Ongoing { state_chunks_done, block_chunks_done, .. } = informant_handle.status() {
info!("Processed {}/{} state chunks and {}/{} block chunks.",
state_chunks_done, num_state, block_chunks_done, num_blocks);
::std::thread::sleep(Duration::from_secs(5));