2020-01-17 14:27:28 +01:00
|
|
|
// Copyright 2015-2020 Parity Technologies (UK) Ltd.
|
2019-01-07 11:33:07 +01:00
|
|
|
// This file is part of Parity Ethereum.
|
2016-02-05 13:40:41 +01:00
|
|
|
|
2019-01-07 11:33:07 +01:00
|
|
|
// Parity Ethereum is free software: you can redistribute it and/or modify
|
2016-02-05 13:40:41 +01:00
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
|
2019-01-07 11:33:07 +01:00
|
|
|
// Parity Ethereum is distributed in the hope that it will be useful,
|
2016-02-05 13:40:41 +01:00
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU General Public License for more details.
|
|
|
|
|
|
|
|
// You should have received a copy of the GNU General Public License
|
2019-01-07 11:33:07 +01:00
|
|
|
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
|
2016-02-05 13:40:41 +01:00
|
|
|
|
2019-08-16 14:45:52 +02:00
|
|
|
use std::sync::Arc;
|
2016-10-18 18:16:00 +02:00
|
|
|
use std::collections::HashMap;
|
2019-09-19 13:12:07 +02:00
|
|
|
|
|
|
|
use crate::chain::sync_packet::{PacketInfo, SyncPacket};
|
|
|
|
|
2017-09-06 20:47:45 +02:00
|
|
|
use bytes::Bytes;
|
2019-08-15 17:59:22 +02:00
|
|
|
use client_traits::BlockChainClient;
|
2019-08-16 14:45:52 +02:00
|
|
|
use ethcore_private_tx::PrivateStateDB;
|
2019-09-19 13:12:07 +02:00
|
|
|
use network::client_version::ClientVersion;
|
|
|
|
use network::{NetworkContext, PeerId, PacketId, Error, SessionInfo, ProtocolId};
|
2016-10-18 18:16:00 +02:00
|
|
|
use parking_lot::RwLock;
|
2019-09-19 13:12:07 +02:00
|
|
|
use snapshot::SnapshotService;
|
|
|
|
use common_types::BlockNumber;
|
2016-01-09 18:50:45 +01:00
|
|
|
|
2016-11-10 18:30:17 +01:00
|
|
|
/// IO interface for the syncing handler.
|
2016-01-10 23:37:09 +01:00
|
|
|
/// Provides peer connection management and an interface to the blockchain client.
|
|
|
|
// TODO: ratings
|
2016-01-09 18:50:45 +01:00
|
|
|
pub trait SyncIo {
|
2016-01-10 23:37:09 +01:00
|
|
|
/// Disable a peer
|
2016-01-14 19:03:48 +01:00
|
|
|
fn disable_peer(&mut self, peer_id: PeerId);
|
2016-02-02 14:54:46 +01:00
|
|
|
/// Disconnect peer
|
|
|
|
fn disconnect_peer(&mut self, peer_id: PeerId);
|
2016-01-10 23:37:09 +01:00
|
|
|
/// Respond to current request with a packet. Can be called from an IO handler for incoming packet.
|
2017-11-13 14:37:08 +01:00
|
|
|
fn respond(&mut self, packet_id: PacketId, data: Vec<u8>) -> Result<(), Error>;
|
2016-10-25 18:40:01 +02:00
|
|
|
/// Send a packet to a peer using specified protocol.
|
2019-02-13 09:20:33 +01:00
|
|
|
fn send(&mut self, peer_id: PeerId, packet_id: SyncPacket, data: Vec<u8>) -> Result<(), Error>;
|
2016-01-10 23:37:09 +01:00
|
|
|
/// Get the blockchain
|
2019-07-09 10:04:20 +02:00
|
|
|
fn chain(&self) -> &dyn BlockChainClient;
|
2016-09-06 15:31:13 +02:00
|
|
|
/// Get the snapshot service.
|
2019-07-09 10:04:20 +02:00
|
|
|
fn snapshot_service(&self) -> &dyn SnapshotService;
|
2019-08-16 14:45:52 +02:00
|
|
|
/// Get the private state wrapper
|
|
|
|
fn private_state(&self) -> Option<Arc<PrivateStateDB>>;
|
2019-02-07 15:27:09 +01:00
|
|
|
/// Returns peer version identifier
|
|
|
|
fn peer_version(&self, peer_id: PeerId) -> ClientVersion {
|
|
|
|
ClientVersion::from(peer_id.to_string())
|
2016-01-14 19:03:48 +01:00
|
|
|
}
|
Snapshot restoration overhaul (#11219)
* Comments and todos
Use `snapshot_sync` as logging target
* fix compilation
* More todos, more logs
* Fix picking snapshot peer: prefer the one with the highest block number
More docs, comments, todos
* Adjust WAIT_PEERS_TIMEOUT to be a multiple of MAINTAIN_SYNC_TIMER to try to fix snapshot startup problems
Docs, todos, comments
* Tabs
* Formatting
* Don't build new rlp::EMPTY_LIST_RLP instances
* Dial down debug logging
* Don't warn about missing hashes in the manifest: it's normal
Log client version on peer connect
* Cleanup
* Do not skip snapshots further away than 30k block from the highest block seen
Currently we look for peers that seed snapshots that are close to the highest block seen on the network (where "close" means withing 30k blocks). When a node starts up we wait for some time (5sec, increased here to 10sec) to let peers connect and if we have found a suitable peer to sync a snapshot from at the end of that delay, we start the download; if none is found and --warp-barrier is used we stall, otherwise we start a slow-sync.
When looking for a suitable snapshot, we use the highest block seen on the network to check if a peer has a snapshot that is within 30k blocks of that highest block number. This means that in a situation where all available snapshots are older than that, we will often fail to start a snapshot at all. What's worse is that the longer we delay starting a snapshot sync (to let more peers connect, in the hope of finding a good snapshot), the more likely we are to have seen a high block and thus the more likely we become to accept a snapshot.
This commit removes this comparison with the highest blocknumber criteria entirely and picks the best snapshot we find in 10sec.
* lockfile
* Add a `ChunkType::Dupe` variant so that we do not disconnect a peer if they happen to send us a duplicate chunk (just ignore the chunk and keep going)
Resolve some documentation todos, add more
* tweak log message
* Don't warp sync twice
Check if our own block is beyond the given warp barrier (can happen after we've completed a warp sync but are not quite yet synced up to the tip) and if so, don't sync.
More docs, resolve todos.
Dial down some `sync` debug level logging to trace
* Avoid iterating over all snapshot block/state hashes to find the next work item
Use a HashSet instead of a Vec and remove items from the set as chunks are processed. Calculate and store the total number of chunks in the `Snapshot` struct instead of counting pending chunks each time.
* Address review grumbles
* Log correct number of bytes written to disk
* Revert ChunkType::Dup change
* whitespace grumble
* Cleanup debugging code
* Fix docs
* Fix import and a typo
* Fix test impl
* Use `indexmap::IndexSet` to ensure chunk hashes are accessed in order
* Revert increased SNAPSHOT_MANIFEST_TIMEOUT: 5sec should be enough
2019-10-31 16:07:21 +01:00
|
|
|
/// Returns the peer enode string
|
|
|
|
fn peer_enode(&self, peer_id: PeerId) -> Option<String>;
|
2016-10-12 20:18:59 +02:00
|
|
|
/// Returns information on p2p session
|
|
|
|
fn peer_session_info(&self, peer_id: PeerId) -> Option<SessionInfo>;
|
2016-10-24 16:24:35 +02:00
|
|
|
/// Maximum mutually supported version of a gien protocol.
|
|
|
|
fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8;
|
2016-03-17 14:56:19 +01:00
|
|
|
/// Returns if the chain block queue empty
|
|
|
|
fn is_chain_queue_empty(&self) -> bool {
|
2018-11-28 11:30:05 +01:00
|
|
|
self.chain().is_queue_empty()
|
2016-03-17 14:56:19 +01:00
|
|
|
}
|
2016-06-17 16:01:33 +02:00
|
|
|
/// Check if the session is expired
|
|
|
|
fn is_expired(&self) -> bool;
|
2016-10-18 18:16:00 +02:00
|
|
|
/// Return sync overlay
|
|
|
|
fn chain_overlay(&self) -> &RwLock<HashMap<BlockNumber, Bytes>>;
|
2019-01-04 19:58:21 +01:00
|
|
|
/// Returns the size the payload shouldn't exceed
|
|
|
|
fn payload_soft_limit(&self) -> usize;
|
2016-01-09 18:50:45 +01:00
|
|
|
}
|
|
|
|
|
2016-01-13 15:10:48 +01:00
|
|
|
/// Wraps `NetworkContext` and the blockchain client
|
2018-03-05 11:56:35 +01:00
|
|
|
pub struct NetSyncIo<'s> {
|
2019-07-09 10:04:20 +02:00
|
|
|
network: &'s dyn NetworkContext,
|
|
|
|
chain: &'s dyn BlockChainClient,
|
|
|
|
snapshot_service: &'s dyn SnapshotService,
|
2016-10-18 18:16:00 +02:00
|
|
|
chain_overlay: &'s RwLock<HashMap<BlockNumber, Bytes>>,
|
2019-08-16 14:45:52 +02:00
|
|
|
private_state: Option<Arc<PrivateStateDB>>,
|
2016-01-09 18:50:45 +01:00
|
|
|
}
|
|
|
|
|
2018-03-05 11:56:35 +01:00
|
|
|
impl<'s> NetSyncIo<'s> {
|
2016-01-13 15:10:48 +01:00
|
|
|
/// Creates a new instance from the `NetworkContext` and the blockchain client reference.
|
2019-07-09 10:04:20 +02:00
|
|
|
pub fn new(network: &'s dyn NetworkContext,
|
|
|
|
chain: &'s dyn BlockChainClient,
|
|
|
|
snapshot_service: &'s dyn SnapshotService,
|
2019-08-16 14:45:52 +02:00
|
|
|
chain_overlay: &'s RwLock<HashMap<BlockNumber, Bytes>>,
|
|
|
|
private_state: Option<Arc<PrivateStateDB>>) -> NetSyncIo<'s> {
|
2016-01-09 18:50:45 +01:00
|
|
|
NetSyncIo {
|
2019-08-16 14:45:52 +02:00
|
|
|
network,
|
|
|
|
chain,
|
|
|
|
snapshot_service,
|
|
|
|
chain_overlay,
|
|
|
|
private_state,
|
2016-01-09 18:50:45 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-05 11:56:35 +01:00
|
|
|
impl<'s> SyncIo for NetSyncIo<'s> {
|
2016-01-14 19:03:48 +01:00
|
|
|
fn disable_peer(&mut self, peer_id: PeerId) {
|
|
|
|
self.network.disable_peer(peer_id);
|
2016-01-09 18:50:45 +01:00
|
|
|
}
|
|
|
|
|
2016-02-02 14:54:46 +01:00
|
|
|
fn disconnect_peer(&mut self, peer_id: PeerId) {
|
|
|
|
self.network.disconnect_peer(peer_id);
|
|
|
|
}
|
|
|
|
|
2017-11-13 14:37:08 +01:00
|
|
|
fn respond(&mut self, packet_id: PacketId, data: Vec<u8>) -> Result<(), Error>{
|
2016-01-09 18:50:45 +01:00
|
|
|
self.network.respond(packet_id, data)
|
|
|
|
}
|
|
|
|
|
2019-02-13 09:20:33 +01:00
|
|
|
fn send(&mut self, peer_id: PeerId, packet_id: SyncPacket, data: Vec<u8>) -> Result<(), Error>{
|
|
|
|
self.network.send_protocol(packet_id.protocol(), peer_id, packet_id.id(), data)
|
2016-10-25 18:40:01 +02:00
|
|
|
}
|
|
|
|
|
2019-07-09 10:04:20 +02:00
|
|
|
fn chain(&self) -> &dyn BlockChainClient {
|
2016-01-09 18:50:45 +01:00
|
|
|
self.chain
|
|
|
|
}
|
2016-01-14 19:03:48 +01:00
|
|
|
|
2019-07-09 10:04:20 +02:00
|
|
|
fn snapshot_service(&self) -> &dyn SnapshotService {
|
2016-09-06 15:31:13 +02:00
|
|
|
self.snapshot_service
|
|
|
|
}
|
|
|
|
|
2019-08-16 14:45:52 +02:00
|
|
|
fn private_state(&self) -> Option<Arc<PrivateStateDB>> {
|
|
|
|
self.private_state.clone()
|
|
|
|
}
|
|
|
|
|
Snapshot restoration overhaul (#11219)
* Comments and todos
Use `snapshot_sync` as logging target
* fix compilation
* More todos, more logs
* Fix picking snapshot peer: prefer the one with the highest block number
More docs, comments, todos
* Adjust WAIT_PEERS_TIMEOUT to be a multiple of MAINTAIN_SYNC_TIMER to try to fix snapshot startup problems
Docs, todos, comments
* Tabs
* Formatting
* Don't build new rlp::EMPTY_LIST_RLP instances
* Dial down debug logging
* Don't warn about missing hashes in the manifest: it's normal
Log client version on peer connect
* Cleanup
* Do not skip snapshots further away than 30k block from the highest block seen
Currently we look for peers that seed snapshots that are close to the highest block seen on the network (where "close" means withing 30k blocks). When a node starts up we wait for some time (5sec, increased here to 10sec) to let peers connect and if we have found a suitable peer to sync a snapshot from at the end of that delay, we start the download; if none is found and --warp-barrier is used we stall, otherwise we start a slow-sync.
When looking for a suitable snapshot, we use the highest block seen on the network to check if a peer has a snapshot that is within 30k blocks of that highest block number. This means that in a situation where all available snapshots are older than that, we will often fail to start a snapshot at all. What's worse is that the longer we delay starting a snapshot sync (to let more peers connect, in the hope of finding a good snapshot), the more likely we are to have seen a high block and thus the more likely we become to accept a snapshot.
This commit removes this comparison with the highest blocknumber criteria entirely and picks the best snapshot we find in 10sec.
* lockfile
* Add a `ChunkType::Dupe` variant so that we do not disconnect a peer if they happen to send us a duplicate chunk (just ignore the chunk and keep going)
Resolve some documentation todos, add more
* tweak log message
* Don't warp sync twice
Check if our own block is beyond the given warp barrier (can happen after we've completed a warp sync but are not quite yet synced up to the tip) and if so, don't sync.
More docs, resolve todos.
Dial down some `sync` debug level logging to trace
* Avoid iterating over all snapshot block/state hashes to find the next work item
Use a HashSet instead of a Vec and remove items from the set as chunks are processed. Calculate and store the total number of chunks in the `Snapshot` struct instead of counting pending chunks each time.
* Address review grumbles
* Log correct number of bytes written to disk
* Revert ChunkType::Dup change
* whitespace grumble
* Cleanup debugging code
* Fix docs
* Fix import and a typo
* Fix test impl
* Use `indexmap::IndexSet` to ensure chunk hashes are accessed in order
* Revert increased SNAPSHOT_MANIFEST_TIMEOUT: 5sec should be enough
2019-10-31 16:07:21 +01:00
|
|
|
fn peer_version(&self, peer_id: PeerId) -> ClientVersion {
|
|
|
|
self.network.peer_client_version(peer_id)
|
2016-01-14 19:03:48 +01:00
|
|
|
}
|
2016-06-17 16:01:33 +02:00
|
|
|
|
Snapshot restoration overhaul (#11219)
* Comments and todos
Use `snapshot_sync` as logging target
* fix compilation
* More todos, more logs
* Fix picking snapshot peer: prefer the one with the highest block number
More docs, comments, todos
* Adjust WAIT_PEERS_TIMEOUT to be a multiple of MAINTAIN_SYNC_TIMER to try to fix snapshot startup problems
Docs, todos, comments
* Tabs
* Formatting
* Don't build new rlp::EMPTY_LIST_RLP instances
* Dial down debug logging
* Don't warn about missing hashes in the manifest: it's normal
Log client version on peer connect
* Cleanup
* Do not skip snapshots further away than 30k block from the highest block seen
Currently we look for peers that seed snapshots that are close to the highest block seen on the network (where "close" means withing 30k blocks). When a node starts up we wait for some time (5sec, increased here to 10sec) to let peers connect and if we have found a suitable peer to sync a snapshot from at the end of that delay, we start the download; if none is found and --warp-barrier is used we stall, otherwise we start a slow-sync.
When looking for a suitable snapshot, we use the highest block seen on the network to check if a peer has a snapshot that is within 30k blocks of that highest block number. This means that in a situation where all available snapshots are older than that, we will often fail to start a snapshot at all. What's worse is that the longer we delay starting a snapshot sync (to let more peers connect, in the hope of finding a good snapshot), the more likely we are to have seen a high block and thus the more likely we become to accept a snapshot.
This commit removes this comparison with the highest blocknumber criteria entirely and picks the best snapshot we find in 10sec.
* lockfile
* Add a `ChunkType::Dupe` variant so that we do not disconnect a peer if they happen to send us a duplicate chunk (just ignore the chunk and keep going)
Resolve some documentation todos, add more
* tweak log message
* Don't warp sync twice
Check if our own block is beyond the given warp barrier (can happen after we've completed a warp sync but are not quite yet synced up to the tip) and if so, don't sync.
More docs, resolve todos.
Dial down some `sync` debug level logging to trace
* Avoid iterating over all snapshot block/state hashes to find the next work item
Use a HashSet instead of a Vec and remove items from the set as chunks are processed. Calculate and store the total number of chunks in the `Snapshot` struct instead of counting pending chunks each time.
* Address review grumbles
* Log correct number of bytes written to disk
* Revert ChunkType::Dup change
* whitespace grumble
* Cleanup debugging code
* Fix docs
* Fix import and a typo
* Fix test impl
* Use `indexmap::IndexSet` to ensure chunk hashes are accessed in order
* Revert increased SNAPSHOT_MANIFEST_TIMEOUT: 5sec should be enough
2019-10-31 16:07:21 +01:00
|
|
|
fn peer_enode(&self, peer_id: PeerId) -> Option<String> {
|
|
|
|
self.network.session_info(peer_id).and_then(|info| {
|
|
|
|
info.id.map(|node_id| {
|
|
|
|
format!("enode:://{}@{}", node_id, info.remote_address)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
fn peer_session_info(&self, peer_id: PeerId) -> Option<SessionInfo> {
|
|
|
|
self.network.session_info(peer_id)
|
2016-06-17 16:01:33 +02:00
|
|
|
}
|
2016-09-06 15:31:13 +02:00
|
|
|
|
2016-10-24 16:24:35 +02:00
|
|
|
fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8 {
|
|
|
|
self.network.protocol_version(*protocol, peer_id).unwrap_or(0)
|
2016-09-06 15:31:13 +02:00
|
|
|
}
|
2016-12-10 14:20:34 +01:00
|
|
|
|
Snapshot restoration overhaul (#11219)
* Comments and todos
Use `snapshot_sync` as logging target
* fix compilation
* More todos, more logs
* Fix picking snapshot peer: prefer the one with the highest block number
More docs, comments, todos
* Adjust WAIT_PEERS_TIMEOUT to be a multiple of MAINTAIN_SYNC_TIMER to try to fix snapshot startup problems
Docs, todos, comments
* Tabs
* Formatting
* Don't build new rlp::EMPTY_LIST_RLP instances
* Dial down debug logging
* Don't warn about missing hashes in the manifest: it's normal
Log client version on peer connect
* Cleanup
* Do not skip snapshots further away than 30k block from the highest block seen
Currently we look for peers that seed snapshots that are close to the highest block seen on the network (where "close" means withing 30k blocks). When a node starts up we wait for some time (5sec, increased here to 10sec) to let peers connect and if we have found a suitable peer to sync a snapshot from at the end of that delay, we start the download; if none is found and --warp-barrier is used we stall, otherwise we start a slow-sync.
When looking for a suitable snapshot, we use the highest block seen on the network to check if a peer has a snapshot that is within 30k blocks of that highest block number. This means that in a situation where all available snapshots are older than that, we will often fail to start a snapshot at all. What's worse is that the longer we delay starting a snapshot sync (to let more peers connect, in the hope of finding a good snapshot), the more likely we are to have seen a high block and thus the more likely we become to accept a snapshot.
This commit removes this comparison with the highest blocknumber criteria entirely and picks the best snapshot we find in 10sec.
* lockfile
* Add a `ChunkType::Dupe` variant so that we do not disconnect a peer if they happen to send us a duplicate chunk (just ignore the chunk and keep going)
Resolve some documentation todos, add more
* tweak log message
* Don't warp sync twice
Check if our own block is beyond the given warp barrier (can happen after we've completed a warp sync but are not quite yet synced up to the tip) and if so, don't sync.
More docs, resolve todos.
Dial down some `sync` debug level logging to trace
* Avoid iterating over all snapshot block/state hashes to find the next work item
Use a HashSet instead of a Vec and remove items from the set as chunks are processed. Calculate and store the total number of chunks in the `Snapshot` struct instead of counting pending chunks each time.
* Address review grumbles
* Log correct number of bytes written to disk
* Revert ChunkType::Dup change
* whitespace grumble
* Cleanup debugging code
* Fix docs
* Fix import and a typo
* Fix test impl
* Use `indexmap::IndexSet` to ensure chunk hashes are accessed in order
* Revert increased SNAPSHOT_MANIFEST_TIMEOUT: 5sec should be enough
2019-10-31 16:07:21 +01:00
|
|
|
fn is_expired(&self) -> bool {
|
|
|
|
self.network.is_expired()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn chain_overlay(&self) -> &RwLock<HashMap<BlockNumber, Bytes>> {
|
|
|
|
self.chain_overlay
|
2016-12-10 14:20:34 +01:00
|
|
|
}
|
2019-01-04 19:58:21 +01:00
|
|
|
|
|
|
|
fn payload_soft_limit(&self) -> usize {
|
|
|
|
self.network.payload_soft_limit()
|
|
|
|
}
|
2016-01-09 18:50:45 +01:00
|
|
|
}
|