grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Prometheus, heavy memory calls removed (#27)

Browse Source
This commit is contained in:
rakita
2020-09-14 16:08:57 +02:00
committed by GitHub
parent dd38573a28
commit aecc6fc862
55 changed files with 3953 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
ethcore/sync/src/api.rs
View File

@@ -30,8 +30,8 @@ use std::{
};
use chain::{
ChainSyncApi, SyncStatus as EthSyncStatus, ETH_PROTOCOL_VERSION_62, ETH_PROTOCOL_VERSION_63,
PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2,
ChainSyncApi, SyncState, SyncStatus as EthSyncStatus, ETH_PROTOCOL_VERSION_62,
ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2,
};
use ethcore::{
client::{BlockChainClient, ChainMessageType, ChainNotify, NewBlocks},
@@ -42,12 +42,17 @@ use ethkey::Secret;
use io::TimerToken;
use network::IpFilter;
use parking_lot::{Mutex, RwLock};
use stats::{prometheus, prometheus_counter, prometheus_gauge, PrometheusMetrics};
use std::{
net::{AddrParseError, SocketAddr},
str::FromStr,
};
use sync_io::NetSyncIo;
use types::{transaction::UnverifiedTransaction, BlockNumber};
use types::{
creation_status::CreationStatus, restoration_status::RestorationStatus,
transaction::UnverifiedTransaction, BlockNumber,
};
/// Parity sync protocol
pub const PAR_PROTOCOL: ProtocolId = *b"par";
@@ -120,7 +125,7 @@ impl Default for SyncConfig {
}
/// Current sync status
pub trait SyncProvider: Send + Sync {
pub trait SyncProvider: Send + Sync + PrometheusMetrics {
/// Get sync status
fn status(&self) -> EthSyncStatus;
@@ -311,6 +316,110 @@ impl SyncProvider for EthSync {
}
}
impl PrometheusMetrics for EthSync {
fn prometheus_metrics(&self, r: &mut prometheus::Registry) {
let scalar = |b| if b { 1i64 } else { 0i64 };
let sync_status = self.status();
prometheus_gauge(r,
"sync_status",
"WaitingPeers(0), SnapshotManifest(1), SnapshotData(2), SnapshotWaiting(3), Blocks(4), Idle(5), Waiting(6), NewBlocks(7)",
match self.eth_handler.sync.status().state {
SyncState::WaitingPeers => 0,
SyncState::SnapshotManifest => 1,
SyncState::SnapshotData => 2,
SyncState::SnapshotWaiting => 3,
SyncState::Blocks => 4,
SyncState::Idle => 5,
SyncState::Waiting => 6,
SyncState::NewBlocks => 7,
});
for (key, value) in sync_status.item_sizes.iter() {
prometheus_gauge(
r,
&key,
format!("Total item number of {}", key).as_str(),
*value as i64,
);
}
prometheus_gauge(
r,
"net_peers",
"Total number of connected peers",
sync_status.num_peers as i64,
);
prometheus_gauge(
r,
"net_active_peers",
"Total number of active peers",
sync_status.num_active_peers as i64,
);
prometheus_counter(
r,
"sync_blocks_recieved",
"Number of blocks downloaded so far",
sync_status.blocks_received as i64,
);
prometheus_counter(
r,
"sync_blocks_total",
"Total number of blocks for the sync process",
sync_status.blocks_total as i64,
);
prometheus_gauge(
r,
"sync_blocks_highest",
"Highest block number in the download queue",
sync_status.highest_block_number.unwrap_or(0) as i64,
);
prometheus_gauge(
r,
"snapshot_download_active",
"1 if downloading snapshots",
scalar(sync_status.is_snapshot_syncing()),
);
prometheus_gauge(
r,
"snapshot_download_chunks",
"Snapshot chunks",
sync_status.num_snapshot_chunks as i64,
);
prometheus_gauge(
r,
"snapshot_download_chunks_done",
"Snapshot chunks downloaded",
sync_status.snapshot_chunks_done as i64,
);
let restoration = self.eth_handler.snapshot_service.restoration_status();
let creation = self.eth_handler.snapshot_service.creation_status();
prometheus_gauge(
r,
"snapshot_create_block",
"First block of the current snapshot creation",
if let CreationStatus::Ongoing { block_number } = creation {
block_number as i64
} else {
0
},
);
prometheus_gauge(
r,
"snapshot_restore_block",
"First block of the current snapshot restoration",
if let RestorationStatus::Ongoing { block_number, .. } = restoration {
block_number as i64
} else {
0
},
);
}
}
const PEERS_TIMER: TimerToken = 0;
const MAINTAIN_SYNC_TIMER: TimerToken = 1;
const CONTINUE_SYNC_TIMER: TimerToken = 2;

14
ethcore/sync/src/block_sync.rs
View File

@@ -24,14 +24,13 @@ use ethcore::{
},
};
use ethereum_types::H256;
use heapsize::HeapSizeOf;
use network::{client_version::ClientCapabilities, PeerId};
use rlp::{self, Rlp};
use std::cmp;
///
/// Blockchain downloader
///
use std::collections::{HashSet, VecDeque};
use std::collections::{BTreeMap, HashSet, VecDeque};
use sync_io::SyncIo;
use types::BlockNumber;
@@ -218,9 +217,14 @@ impl BlockDownloader {
self.state = State::Blocks;
}
/// Returns used heap memory size.
pub fn heap_size(&self) -> usize {
self.blocks.heap_size() + self.round_parents.heap_size_of_children()
/// Returns number if items in structures
pub fn get_sizes(&self, sizes: &mut BTreeMap<String, usize>) {
let prefix = format!("{}_", self.block_set.to_string());
self.blocks.get_sizes(sizes, &prefix);
sizes.insert(
format!("{}{}", prefix, "round_parents"),
self.round_parents.len(),
);
}
fn reset_to_block(&mut self, start_hash: &H256, start_number: BlockNumber) {

41
ethcore/sync/src/blocks.rs
View File

@@ -21,7 +21,7 @@ use hash::{keccak, KECCAK_EMPTY_LIST_RLP, KECCAK_NULL_RLP};
use heapsize::HeapSizeOf;
use network;
use rlp::{DecoderError, Rlp, RlpStream};
use std::collections::{hash_map, HashMap, HashSet};
use std::collections::{hash_map, BTreeMap, HashMap, HashSet};
use triehash_ethereum::ordered_trie_root;
use types::{header::Header as BlockHeader, transaction::UnverifiedTransaction};
@@ -414,14 +414,37 @@ impl BlockCollection {
self.heads.len()
}
/// Return used heap size.
pub fn heap_size(&self) -> usize {
self.heads.heap_size_of_children()
+ self.blocks.heap_size_of_children()
+ self.parents.heap_size_of_children()
+ self.header_ids.heap_size_of_children()
+ self.downloading_headers.heap_size_of_children()
+ self.downloading_bodies.heap_size_of_children()
/// Return number of items size.
pub fn get_sizes(&self, sizes: &mut BTreeMap<String, usize>, insert_prefix: &str) {
sizes.insert(format!("{}{}", insert_prefix, "heads"), self.heads.len());
sizes.insert(format!("{}{}", insert_prefix, "blocks"), self.blocks.len());
sizes.insert(
format!("{}{}", insert_prefix, "parents_len"),
self.parents.len(),
);
sizes.insert(
format!("{}{}", insert_prefix, "header_ids_len"),
self.header_ids.len(),
);
sizes.insert(
format!("{}{}", insert_prefix, "downloading_headers_len"),
self.downloading_headers.len(),
);
sizes.insert(
format!("{}{}", insert_prefix, "downloading_bodies_len"),
self.downloading_bodies.len(),
);
if self.need_receipts {
sizes.insert(
format!("{}{}", insert_prefix, "downloading_receipts_len"),
self.downloading_receipts.len(),
);
sizes.insert(
format!("{}{}", insert_prefix, "receipt_ids_len"),
self.receipt_ids.len(),
);
}
}
/// Check if given block hash is marked as being downloaded.

2
ethcore/sync/src/chain/handler.rs
View File

@@ -599,7 +599,7 @@ impl SyncHandler {
}
// check service status
let status = io.snapshot_service().status();
let status = io.snapshot_service().restoration_status();
match status {
RestorationStatus::Inactive | RestorationStatus::Failed => {
trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id);

35
ethcore/sync/src/chain/mod.rs
View File

@@ -105,7 +105,6 @@ use ethcore::{
use ethereum_types::{H256, U256};
use fastmap::{H256FastMap, H256FastSet};
use hash::keccak;
use heapsize::HeapSizeOf;
use network::{self, client_version::ClientVersion, PeerId};
use parking_lot::{Mutex, RwLock, RwLockWriteGuard};
use rand::Rng;
@@ -214,7 +213,7 @@ pub enum SyncState {
}
/// Syncing status and statistics
#[derive(Clone, Copy)]
#[derive(Clone)]
pub struct SyncStatus {
/// State
pub state: SyncState,
@@ -236,14 +235,14 @@ pub struct SyncStatus {
pub num_peers: usize,
/// Total number of active peers.
pub num_active_peers: usize,
/// Heap memory used in bytes.
pub mem_used: usize,
/// Snapshot chunks
pub num_snapshot_chunks: usize,
/// Snapshot chunks downloaded
pub snapshot_chunks_done: usize,
/// Last fully downloaded and imported ancient block number (if any).
pub last_imported_old_block_number: Option<BlockNumber>,
/// Internal structure item numbers
pub item_sizes: BTreeMap<String, usize>,
}
impl SyncStatus {
@@ -297,6 +296,16 @@ pub enum BlockSet {
/// Missing old blocks
OldBlocks,
}
impl BlockSet {
pub fn to_string(&self) -> &'static str {
match *self {
Self::NewBlocks => "new_blocks",
Self::OldBlocks => "old_blocks",
}
}
}
#[derive(Clone, Eq, PartialEq)]
pub enum ForkConfirmation {
/// Fork block confirmation pending.
@@ -708,6 +717,12 @@ impl ChainSync {
/// Returns synchonization status
pub fn status(&self) -> SyncStatus {
let last_imported_number = self.new_blocks.last_imported_block_number();
let mut item_sizes = BTreeMap::<String, usize>::new();
self.old_blocks
.as_ref()
.map_or((), |d| d.get_sizes(&mut item_sizes));
self.new_blocks.get_sizes(&mut item_sizes);
SyncStatus {
state: self.state.clone(),
protocol_version: ETH_PROTOCOL_VERSION_63.0,
@@ -738,9 +753,7 @@ impl ChainSync {
.count(),
num_snapshot_chunks: self.snapshot.total_chunks(),
snapshot_chunks_done: self.snapshot.done_chunks(),
mem_used: self.new_blocks.heap_size()
+ self.old_blocks.as_ref().map_or(0, |d| d.heap_size())
+ self.peers.heap_size_of_children(),
item_sizes: item_sizes,
}
}
@@ -1108,7 +1121,7 @@ impl ChainSync {
}
},
SyncState::SnapshotData => {
match io.snapshot_service().status() {
match io.snapshot_service().restoration_status() {
RestorationStatus::Ongoing { state_chunks_done, block_chunks_done, .. } => {
// Initialize the snapshot if not already done
self.snapshot.initialize(io.snapshot_service());
@@ -1309,13 +1322,13 @@ impl ChainSync {
self.state = SyncState::Blocks;
self.continue_sync(io);
}
SyncState::SnapshotData => match io.snapshot_service().status() {
SyncState::SnapshotData => match io.snapshot_service().restoration_status() {
RestorationStatus::Inactive | RestorationStatus::Failed => {
self.state = SyncState::SnapshotWaiting;
}
RestorationStatus::Initializing { .. } | RestorationStatus::Ongoing { .. } => (),
},
SyncState::SnapshotWaiting => match io.snapshot_service().status() {
SyncState::SnapshotWaiting => match io.snapshot_service().restoration_status() {
RestorationStatus::Inactive => {
trace!(target:"sync", "Snapshot restoration is complete");
self.restart(io);
@@ -1541,7 +1554,7 @@ pub mod tests {
blocks_received: 0,
num_peers: 0,
num_active_peers: 0,
mem_used: 0,
item_sizes: BTreeMap::new(),
num_snapshot_chunks: 0,
snapshot_chunks_done: 0,
last_imported_old_block_number: None,

1
ethcore/sync/src/lib.rs
View File

@@ -36,6 +36,7 @@ extern crate parity_bytes as bytes;
extern crate parking_lot;
extern crate rand;
extern crate rlp;
extern crate stats;
extern crate triehash_ethereum;
#[cfg(test)]

9
ethcore/sync/src/tests/snapshot.rs
View File

@@ -18,7 +18,7 @@ use super::helpers::*;
use bytes::Bytes;
use ethcore::{
client::EachBlockWith,
snapshot::{ManifestData, RestorationStatus, SnapshotService},
snapshot::{CreationStatus, ManifestData, RestorationStatus, SnapshotService},
};
use ethereum_types::H256;
use hash::keccak;
@@ -101,7 +101,11 @@ impl SnapshotService for TestSnapshotService {
self.chunks.get(&hash).cloned()
}
fn status(&self) -> RestorationStatus {
fn creation_status(&self) -> CreationStatus {
CreationStatus::Inactive
}
fn restoration_status(&self) -> RestorationStatus {
match *self.restoration_manifest.lock() {
Some(ref manifest)
if self.state_restoration_chunks.lock().len() == manifest.state_hashes.len()
@@ -111,6 +115,7 @@ impl SnapshotService for TestSnapshotService {
RestorationStatus::Inactive
}
Some(ref manifest) => RestorationStatus::Ongoing {
block_number: 0,
state_chunks: manifest.state_hashes.len() as u32,
block_chunks: manifest.block_hashes.len() as u32,
state_chunks_done: self.state_restoration_chunks.lock().len() as u32,