grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into auth-bft

Browse Source
This commit is contained in:
keorn
2016-11-21 12:17:00 +00:00
parent e69be670de 21b2b4ac27
commit 6e0bd4072b
85 changed files with 3081 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
sync/src/api.rs
View File

@@ -15,13 +15,13 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::sync::Arc;
use std::collections::HashMap;
use std::collections::{HashMap, BTreeMap};
use std::io;
use util::Bytes;
use network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId, ProtocolId,
NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, NetworkError,
AllowIP as NetworkAllowIP};
use util::{U256, H256};
use util::{U256, H256, H512};
use io::{TimerToken};
use ethcore::client::{BlockChainClient, ChainNotify};
use ethcore::snapshot::SnapshotService;
@@ -82,6 +82,16 @@ pub trait SyncProvider: Send + Sync {
/// Get the enode if available.
fn enode(&self) -> Option<String>;
/// Returns propagation count for pending transactions.
fn transactions_stats(&self) -> BTreeMap<H256, TransactionStats>;
}
/// Transaction stats
#[derive(Debug, Binary)]
pub struct TransactionStats {
pub first_seen: u64,
pub propagated_to: BTreeMap<H512, usize>,
}
/// Peer connection information
@@ -165,6 +175,14 @@ impl SyncProvider for EthSync {
fn enode(&self) -> Option<String> {
self.network.external_url()
}
fn transactions_stats(&self) -> BTreeMap<H256, TransactionStats> {
let sync = self.handler.sync.read();
sync.transactions_stats()
.iter()
.map(|(hash, stats)| (*hash, stats.into()))
.collect()
}
}
struct SyncProtocolHandler {

31
sync/src/block_sync.rs
View File

@@ -34,6 +34,7 @@ const MAX_RECEPITS_TO_REQUEST: usize = 128;
const SUBCHAIN_SIZE: u64 = 256;
const MAX_ROUND_PARENTS: usize = 32;
const MAX_PARALLEL_SUBCHAIN_DOWNLOAD: usize = 5;
const MAX_REORG_BLOCKS: u64 = 20;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
/// Downloader state
@@ -262,7 +263,8 @@ impl BlockDownloader {
State::Blocks => {
let count = headers.len();
// At least one of the heades must advance the subchain. Otherwise they are all useless.
if !any_known {
if count == 0 || !any_known {
trace!(target: "sync", "No useful headers");
return Err(BlockDownloaderImportError::Useless);
}
self.blocks.insert_headers(headers);
@@ -340,14 +342,21 @@ impl BlockDownloader {
self.last_imported_hash = p.clone();
trace!(target: "sync", "Searching common header from the last round {} ({})", self.last_imported_block, self.last_imported_hash);
} else {
match io.chain().block_hash(BlockID::Number(self.last_imported_block - 1)) {
Some(h) => {
self.last_imported_block -= 1;
self.last_imported_hash = h;
trace!(target: "sync", "Searching common header in the blockchain {} ({})", self.last_imported_block, self.last_imported_hash);
}
None => {
debug!(target: "sync", "Could not revert to previous block, last: {} ({})", self.last_imported_block, self.last_imported_hash);
let best = io.chain().chain_info().best_block_number;
if best > self.last_imported_block && best - self.last_imported_block > MAX_REORG_BLOCKS {
debug!(target: "sync", "Could not revert to previous ancient block, last: {} ({})", self.last_imported_block, self.last_imported_hash);
self.reset();
} else {
match io.chain().block_hash(BlockID::Number(self.last_imported_block - 1)) {
Some(h) => {
self.last_imported_block -= 1;
self.last_imported_hash = h;
trace!(target: "sync", "Searching common header in the blockchain {} ({})", self.last_imported_block, self.last_imported_hash);
}
None => {
debug!(target: "sync", "Could not revert to previous block, last: {} ({})", self.last_imported_block, self.last_imported_hash);
self.reset();
}
}
}
}
@@ -362,7 +371,9 @@ impl BlockDownloader {
match self.state {
State::Idle => {
self.start_sync_round(io);
return self.request_blocks(io, num_active_peers);
if self.state == State::ChainHead {
return self.request_blocks(io, num_active_peers);
}
},
State::ChainHead => {
if num_active_peers < MAX_PARALLEL_SUBCHAIN_DOWNLOAD {

113
sync/src/chain.rs
View File

@@ -104,6 +104,7 @@ use block_sync::{BlockDownloader, BlockRequest, BlockDownloaderImportError as Do
use snapshot::{Snapshot, ChunkType};
use rand::{thread_rng, Rng};
use api::{PeerInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID};
use transactions_stats::{TransactionsStats, Stats as TransactionStats};
known_heap_size!(0, PeerInfo);
@@ -259,7 +260,7 @@ enum ForkConfirmation {
Unconfirmed,
/// Peers chain is too short to confirm the fork.
TooShort,
/// Fork is confurmed.
/// Fork is confirmed.
Confirmed,
}
@@ -349,6 +350,8 @@ pub struct ChainSync {
handshaking_peers: HashMap<PeerId, u64>,
/// Sync start timestamp. Measured when first peer is connected
sync_start_time: Option<u64>,
/// Transactions propagation statistics
transactions_stats: TransactionsStats,
}
type RlpResponseResult = Result<Option<(PacketId, RlpStream)>, PacketDecodeError>;
@@ -371,6 +374,7 @@ impl ChainSync {
fork_block: config.fork_block,
snapshot: Snapshot::new(),
sync_start_time: None,
transactions_stats: TransactionsStats::default(),
};
sync.update_targets(chain);
sync
@@ -419,6 +423,11 @@ impl ChainSync {
.collect()
}
/// Returns transactions propagation statistics
pub fn transactions_stats(&self) -> &H256FastMap<TransactionStats> {
self.transactions_stats.stats()
}
/// Abort all sync activity
pub fn abort(&mut self, io: &mut SyncIo) {
self.reset_and_continue(io);
@@ -1146,6 +1155,7 @@ impl ChainSync {
let have_latest = io.chain().block_status(BlockID::Hash(peer_latest)) != BlockStatus::Unknown;
if !have_latest && (higher_difficulty || force || self.state == SyncState::NewBlocks) {
// check if got new blocks to download
trace!(target: "sync", "Syncing with {}, force={}, td={:?}, our td={}, state={:?}", peer_id, force, peer_difficulty, syncing_difficulty, self.state);
if let Some(request) = self.new_blocks.request_blocks(io, num_active_peers) {
self.request_blocks(io, peer_id, request, BlockSet::NewBlocks);
if self.state == SyncState::Idle {
@@ -1868,7 +1878,7 @@ impl ChainSync {
/// propagates new transactions to all peers
pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize {
// Early out of nobody to send to.
// Early out if nobody to send to.
if self.peers.is_empty() {
return 0;
}
@@ -1885,38 +1895,53 @@ impl ChainSync {
packet.out()
};
// Clear old transactions from stats
self.transactions_stats.retain(&all_transactions_hashes);
// sqrt(x)/x scaled to max u32
let fraction = (self.peers.len() as f64).powf(-0.5).mul(u32::max_value() as f64).round() as u32;
let small = self.peers.len() < MIN_PEERS_PROPAGATION;
let block_number = io.chain().chain_info().best_block_number;
let lucky_peers = self.peers.iter_mut()
.filter(|_| small || ::rand::random::<u32>() < fraction)
.take(MAX_PEERS_PROPAGATION)
.filter_map(|(peer_id, mut peer_info)| {
// Send all transactions
if peer_info.last_sent_transactions.is_empty() {
peer_info.last_sent_transactions = all_transactions_hashes.clone();
return Some((*peer_id, all_transactions_rlp.clone()));
}
// Get hashes of all transactions to send to this peer
let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions).cloned().collect::<HashSet<_>>();
if to_send.is_empty() {
return None;
}
// Construct RLP
let mut packet = RlpStream::new_list(to_send.len());
for tx in &transactions {
if to_send.contains(&tx.hash()) {
packet.append(tx);
let lucky_peers = {
let stats = &mut self.transactions_stats;
self.peers.iter_mut()
.filter(|_| small || ::rand::random::<u32>() < fraction)
.take(MAX_PEERS_PROPAGATION)
.filter_map(|(peer_id, mut peer_info)| {
// Send all transactions
if peer_info.last_sent_transactions.is_empty() {
// update stats
for hash in &all_transactions_hashes {
let id = io.peer_session_info(*peer_id).and_then(|info| info.id);
stats.propagated(*hash, id, block_number);
}
peer_info.last_sent_transactions = all_transactions_hashes.clone();
return Some((*peer_id, all_transactions_rlp.clone()));
}
}
peer_info.last_sent_transactions = all_transactions_hashes.clone();
Some((*peer_id, packet.out()))
})
.collect::<Vec<_>>();
// Get hashes of all transactions to send to this peer
let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions).cloned().collect::<HashSet<_>>();
if to_send.is_empty() {
return None;
}
// Construct RLP
let mut packet = RlpStream::new_list(to_send.len());
for tx in &transactions {
if to_send.contains(&tx.hash()) {
packet.append(tx);
// update stats
let id = io.peer_session_info(*peer_id).and_then(|info| info.id);
stats.propagated(tx.hash(), id, block_number);
}
}
peer_info.last_sent_transactions = all_transactions_hashes.clone();
Some((*peer_id, packet.out()))
})
.collect::<Vec<_>>()
};
// Send RLPs
let sent = lucky_peers.len();
@@ -1966,9 +1991,6 @@ impl ChainSync {
trace!(target: "sync", "Bad blocks in the queue, restarting");
self.restart(io);
}
for peer_info in self.peers.values_mut() {
peer_info.last_sent_transactions.clear();
}
}
}
@@ -2294,18 +2316,23 @@ mod tests {
let peer_count = sync.propagate_new_transactions(&mut io);
// Try to propagate same transactions for the second time
let peer_count2 = sync.propagate_new_transactions(&mut io);
// Even after new block transactions should not be propagated twice
sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[]);
// Try to propagate same transactions for the third time
let peer_count3 = sync.propagate_new_transactions(&mut io);
// 1 message should be send
assert_eq!(1, io.queue.len());
// 1 peer should be updated but only once
assert_eq!(1, peer_count);
assert_eq!(0, peer_count2);
assert_eq!(0, peer_count3);
// TRANSACTIONS_PACKET
assert_eq!(0x02, io.queue[0].packet_id);
}
#[test]
fn propagates_transactions_again_after_new_block() {
fn propagates_new_transactions_after_new_block() {
let mut client = TestBlockChainClient::new();
client.add_blocks(100, EachBlockWith::Uncle);
client.insert_transaction_to_queue();
@@ -2314,15 +2341,14 @@ mod tests {
let ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &ss, &mut queue, None);
let peer_count = sync.propagate_new_transactions(&mut io);
io.chain.insert_transaction_to_queue();
// New block import should trigger propagation.
sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[]);
// Try to propagate same transactions for the second time
let peer_count2 = sync.propagate_new_transactions(&mut io);
// 2 message should be send
assert_eq!(2, io.queue.len());
// 1 peer should be updated twice
// 1 peer should receive the message
assert_eq!(1, peer_count);
assert_eq!(1, peer_count2);
// TRANSACTIONS_PACKET
assert_eq!(0x02, io.queue[0].packet_id);
assert_eq!(0x02, io.queue[1].packet_id);
@@ -2361,6 +2387,21 @@ mod tests {
assert_eq!(0x02, io.queue[1].packet_id);
}
#[test]
fn should_maintain_transations_propagation_stats() {
let mut client = TestBlockChainClient::new();
client.add_blocks(100, EachBlockWith::Uncle);
client.insert_transaction_to_queue();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
let mut queue = VecDeque::new();
let ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &ss, &mut queue, None);
sync.propagate_new_transactions(&mut io);
let stats = sync.transactions_stats();
assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.")
}
#[test]
fn handles_peer_new_block_malformed() {
let mut client = TestBlockChainClient::new();

3
sync/src/lib.rs
View File

@@ -52,6 +52,7 @@ mod block_sync;
mod sync_io;
mod infinity;
mod snapshot;
mod transactions_stats;
#[cfg(test)]
mod tests;
@@ -62,7 +63,7 @@ mod api {
}
pub use api::{EthSync, SyncProvider, SyncClient, NetworkManagerClient, ManageNetwork, SyncConfig,
ServiceConfiguration, NetworkConfiguration, PeerInfo, AllowIP};
ServiceConfiguration, NetworkConfiguration, PeerInfo, AllowIP, TransactionStats};
pub use chain::{SyncStatus, SyncState};
pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError};

26
sync/src/tests/chain.rs
View File

@@ -79,14 +79,14 @@ fn empty_blocks() {
fn forked() {
::env_logger::init().ok();
let mut net = TestNet::new(3);
net.peer_mut(0).chain.add_blocks(300, EachBlockWith::Uncle);
net.peer_mut(1).chain.add_blocks(300, EachBlockWith::Uncle);
net.peer_mut(2).chain.add_blocks(300, EachBlockWith::Uncle);
net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Nothing); //fork
net.peer_mut(1).chain.add_blocks(200, EachBlockWith::Uncle);
net.peer_mut(2).chain.add_blocks(200, EachBlockWith::Uncle);
net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); //fork between 1 and 2
net.peer_mut(2).chain.add_blocks(10, EachBlockWith::Nothing);
net.peer_mut(0).chain.add_blocks(30, EachBlockWith::Uncle);
net.peer_mut(1).chain.add_blocks(30, EachBlockWith::Uncle);
net.peer_mut(2).chain.add_blocks(30, EachBlockWith::Uncle);
net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Nothing); //fork
net.peer_mut(1).chain.add_blocks(20, EachBlockWith::Uncle);
net.peer_mut(2).chain.add_blocks(20, EachBlockWith::Uncle);
net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); //fork between 1 and 2
net.peer_mut(2).chain.add_blocks(1, EachBlockWith::Nothing);
// peer 1 has the best chain of 601 blocks
let peer1_chain = net.peer(1).chain.numbers.read().clone();
net.sync();
@@ -102,12 +102,12 @@ fn forked_with_misbehaving_peer() {
let mut net = TestNet::new(3);
// peer 0 is on a totally different chain with higher total difficulty
net.peer_mut(0).chain = TestBlockChainClient::new_with_extra_data(b"fork".to_vec());
net.peer_mut(0).chain.add_blocks(500, EachBlockWith::Nothing);
net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Nothing);
net.peer_mut(2).chain.add_blocks(100, EachBlockWith::Nothing);
net.peer_mut(0).chain.add_blocks(50, EachBlockWith::Nothing);
net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Nothing);
net.peer_mut(2).chain.add_blocks(10, EachBlockWith::Nothing);
net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Nothing);
net.peer_mut(2).chain.add_blocks(200, EachBlockWith::Uncle);
net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Nothing);
net.peer_mut(2).chain.add_blocks(20, EachBlockWith::Uncle);
// peer 1 should sync to peer 2, others should not change
let peer0_chain = net.peer(0).chain.numbers.read().clone();
let peer2_chain = net.peer(2).chain.numbers.read().clone();

134
sync/src/transactions_stats.rs Normal file
View File

@@ -0,0 +1,134 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use api::TransactionStats;
use std::collections::{HashSet, HashMap};
use util::{H256, H512};
use util::hash::H256FastMap;
type NodeId = H512;
type BlockNumber = u64;
#[derive(Debug, PartialEq, Clone)]
pub struct Stats {
first_seen: BlockNumber,
propagated_to: HashMap<NodeId, usize>,
}
impl Stats {
pub fn new(number: BlockNumber) -> Self {
Stats {
first_seen: number,
propagated_to: Default::default(),
}
}
}
impl<'a> From<&'a Stats> for TransactionStats {
fn from(other: &'a Stats) -> Self {
TransactionStats {
first_seen: other.first_seen,
propagated_to: other.propagated_to
.iter()
.map(|(hash, size)| (*hash, *size))
.collect(),
}
}
}
#[derive(Debug, Default)]
pub struct TransactionsStats {
pending_transactions: H256FastMap<Stats>,
}
impl TransactionsStats {
/// Increases number of propagations to given `enodeid`.
pub fn propagated(&mut self, hash: H256, enode_id: Option<NodeId>, current_block_num: BlockNumber) {
let enode_id = enode_id.unwrap_or_default();
let mut stats = self.pending_transactions.entry(hash).or_insert_with(|| Stats::new(current_block_num));
let mut count = stats.propagated_to.entry(enode_id).or_insert(0);
*count = count.saturating_add(1);
}
/// Returns propagation stats for given hash or `None` if hash is not known.
#[cfg(test)]
pub fn get(&self, hash: &H256) -> Option<&Stats> {
self.pending_transactions.get(hash)
}
pub fn stats(&self) -> &H256FastMap<Stats> {
&self.pending_transactions
}
/// Retains only transactions present in given `HashSet`.
pub fn retain(&mut self, hashes: &HashSet<H256>) {
let to_remove = self.pending_transactions.keys()
.filter(|hash| !hashes.contains(hash))
.cloned()
.collect::<Vec<_>>();
for hash in to_remove {
self.pending_transactions.remove(&hash);
}
}
}
#[cfg(test)]
mod tests {
use std::collections::{HashMap, HashSet};
use super::{Stats, TransactionsStats};
#[test]
fn should_keep_track_of_propagations() {
// given
let mut stats = TransactionsStats::default();
let hash = 5.into();
let enodeid1 = 2.into();
let enodeid2 = 5.into();
// when
stats.propagated(hash, Some(enodeid1), 5);
stats.propagated(hash, Some(enodeid1), 10);
stats.propagated(hash, Some(enodeid2), 15);
// then
let stats = stats.get(&hash);
assert_eq!(stats, Some(&Stats {
first_seen: 5,
propagated_to: hash_map![
enodeid1 => 2,
enodeid2 => 1
]
}));
}
#[test]
fn should_remove_hash_from_tracking() {
// given
let mut stats = TransactionsStats::default();
let hash = 5.into();
let enodeid1 = 5.into();
stats.propagated(hash, Some(enodeid1), 10);
// when
stats.retain(&HashSet::new());
// then
let stats = stats.get(&hash);
assert_eq!(stats, None);
}
}