grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Prometheus, heavy memory calls removed (#27)

Browse Source
This commit is contained in:
rakita
2020-09-14 16:08:57 +02:00
committed by GitHub
parent dd38573a28
commit aecc6fc862
55 changed files with 3953 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
ethcore/src/client/client.rs
View File

@@ -85,6 +85,7 @@ use snapshot::{self, io as snapshot_io, SnapshotClient};
use spec::Spec;
use state::{self, State};
use state_db::StateDB;
use stats::{prometheus, prometheus_counter, prometheus_gauge, PrometheusMetrics};
use trace::{
self, Database as TraceDatabase, ImportRequest as TraceImportRequest, LocalizedTrace, TraceDB,
};
@@ -118,8 +119,8 @@ pub struct ClientReport {
pub transactions_applied: usize,
/// How much gas has been processed so far.
pub gas_processed: U256,
/// Memory used by state DB
pub state_db_mem: usize,
/// Internal structure item sizes
pub item_sizes: BTreeMap<String, usize>,
}
impl ClientReport {
@@ -135,13 +136,9 @@ impl<'a> ::std::ops::Sub<&'a ClientReport> for ClientReport {
type Output = Self;
fn sub(mut self, other: &'a ClientReport) -> Self {
let higher_mem = ::std::cmp::max(self.state_db_mem, other.state_db_mem);
let lower_mem = ::std::cmp::min(self.state_db_mem, other.state_db_mem);
self.blocks_imported -= other.blocks_imported;
self.transactions_applied -= other.transactions_applied;
self.gas_processed = self.gas_processed - other.gas_processed;
self.state_db_mem = higher_mem - lower_mem;
self
}
@@ -1245,7 +1242,7 @@ impl Client {
/// Get the report.
pub fn report(&self) -> ClientReport {
let mut report = self.report.read().clone();
report.state_db_mem = self.state_db.read().mem_used();
self.state_db.read().get_sizes(&mut report.item_sizes);
report
}
@@ -3183,6 +3180,163 @@ impl IoChannelQueue {
}
}
impl PrometheusMetrics for Client {
fn prometheus_metrics(&self, r: &mut prometheus::Registry) {
// gas, tx & blocks
let report = self.report();
for (key, value) in report.item_sizes.iter() {
prometheus_gauge(
r,
&key,
format!("Total item number of {}", key).as_str(),
*value as i64,
);
}
prometheus_counter(
r,
"import_gas",
"Gas processed",
report.gas_processed.as_u64() as i64,
);
prometheus_counter(
r,
"import_blocks",
"Blocks imported",
report.blocks_imported as i64,
);
prometheus_counter(
r,
"import_txs",
"Transactions applied",
report.transactions_applied as i64,
);
let state_db = self.state_db.read();
prometheus_gauge(
r,
"statedb_cache_size",
"State DB cache size",
state_db.cache_size() as i64,
);
// blockchain cache
let blockchain_cache_info = self.blockchain_cache_info();
prometheus_gauge(
r,
"blockchaincache_block_details",
"BlockDetails cache size",
blockchain_cache_info.block_details as i64,
);
prometheus_gauge(
r,
"blockchaincache_block_recipts",
"Block receipts size",
blockchain_cache_info.block_receipts as i64,
);
prometheus_gauge(
r,
"blockchaincache_blocks",
"Blocks cache size",
blockchain_cache_info.blocks as i64,
);
prometheus_gauge(
r,
"blockchaincache_txaddrs",
"Transaction addresses cache size",
blockchain_cache_info.transaction_addresses as i64,
);
prometheus_gauge(
r,
"blockchaincache_size",
"Total blockchain cache size",
blockchain_cache_info.total() as i64,
);
// chain info
let chain = self.chain_info();
let gap = chain
.ancient_block_number
.map(|x| U256::from(x + 1))
.and_then(|first| {
chain
.first_block_number
.map(|last| (first, U256::from(last)))
});
if let Some((first, last)) = gap {
prometheus_gauge(
r,
"chain_warpsync_gap_first",
"Warp sync gap, first block",
first.as_u64() as i64,
);
prometheus_gauge(
r,
"chain_warpsync_gap_last",
"Warp sync gap, last block",
last.as_u64() as i64,
);
}
prometheus_gauge(
r,
"chain_block",
"Best block number",
chain.best_block_number as i64,
);
// prunning info
let prunning = self.pruning_info();
prometheus_gauge(
r,
"prunning_earliest_chain",
"The first block which everything can be served after",
prunning.earliest_chain as i64,
);
prometheus_gauge(
r,
"prunning_earliest_state",
"The first block where state requests may be served",
prunning.earliest_state as i64,
);
// queue info
let queue = self.queue_info();
prometheus_gauge(
r,
"queue_mem_used",
"Queue heap memory used in bytes",
queue.mem_used as i64,
);
prometheus_gauge(
r,
"queue_size_total",
"The total size of the queues",
queue.total_queue_size() as i64,
);
prometheus_gauge(
r,
"queue_size_unverified",
"Number of queued items pending verification",
queue.unverified_queue_size as i64,
);
prometheus_gauge(
r,
"queue_size_verified",
"Number of verified queued items pending import",
queue.verified_queue_size as i64,
);
prometheus_gauge(
r,
"queue_size_verifying",
"Number of items being verified",
queue.verifying_queue_size as i64,
);
}
}
#[cfg(test)]
mod tests {
use blockchain::{BlockProvider, ExtrasInsert};

5
ethcore/src/client/test_client.rs
View File

@@ -72,6 +72,7 @@ use miner::{self, Miner, MinerService};
use spec::Spec;
use state::StateInfo;
use state_db::StateDB;
use stats::{prometheus, PrometheusMetrics};
use trace::LocalizedTrace;
use verification::queue::{kind::blocks::Unverified, QueueInfo};
@@ -1114,3 +1115,7 @@ impl super::traits::EngineClient for TestBlockChainClient {
BlockChainClient::block_header(self, id)
}
}
impl PrometheusMetrics for TestBlockChainClient {
fn prometheus_metrics(&self, _r: &mut prometheus::Registry) {}
}

4
ethcore/src/snapshot/mod.rs
View File

@@ -64,8 +64,8 @@ pub use self::{
watcher::Watcher,
};
pub use types::{
basic_account::BasicAccount, restoration_status::RestorationStatus,
snapshot_manifest::ManifestData,
basic_account::BasicAccount, creation_status::CreationStatus,
restoration_status::RestorationStatus, snapshot_manifest::ManifestData,
};
pub mod io;

26
ethcore/src/snapshot/service.rs
View File

@@ -30,7 +30,8 @@ use std::{
use super::{
io::{LooseReader, LooseWriter, SnapshotReader, SnapshotWriter},
ManifestData, Rebuilder, RestorationStatus, SnapshotService, StateRebuilder, MAX_CHUNK_SIZE,
CreationStatus, ManifestData, Rebuilder, RestorationStatus, SnapshotService, StateRebuilder,
MAX_CHUNK_SIZE,
};
use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler};
@@ -271,6 +272,7 @@ pub struct Service {
client: Arc<dyn SnapshotClient>,
progress: super::Progress,
taking_snapshot: AtomicBool,
taking_snapshot_at: AtomicUsize,
restoring_snapshot: AtomicBool,
}
@@ -292,6 +294,7 @@ impl Service {
client: params.client,
progress: Default::default(),
taking_snapshot: AtomicBool::new(false),
taking_snapshot_at: AtomicUsize::new(0),
restoring_snapshot: AtomicBool::new(false),
};
@@ -522,6 +525,9 @@ impl Service {
return Ok(());
}
self.taking_snapshot_at
.store(num as usize, Ordering::SeqCst);
info!("Taking snapshot at #{}", num);
self.progress.reset();
@@ -629,6 +635,7 @@ impl Service {
self.restoring_snapshot.store(true, Ordering::SeqCst);
let block_number = manifest.block_number;
// Import previous chunks, continue if it fails
self.import_prev_chunks(&mut res, manifest).ok();
@@ -636,6 +643,7 @@ impl Service {
let mut restoration_status = self.status.lock();
if let RestorationStatus::Initializing { .. } = *restoration_status {
*restoration_status = RestorationStatus::Ongoing {
block_number,
state_chunks: state_chunks as u32,
block_chunks: block_chunks as u32,
state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32,
@@ -774,7 +782,7 @@ impl Service {
is_state: bool,
) -> Result<(), Error> {
let (result, db) = {
match self.status() {
match self.restoration_status() {
RestorationStatus::Inactive | RestorationStatus::Failed => {
trace!(target: "snapshot", "Tried to restore chunk {:x} while inactive or failed", hash);
return Ok(());
@@ -881,7 +889,17 @@ impl SnapshotService for Service {
}
}
fn status(&self) -> RestorationStatus {
fn creation_status(&self) -> CreationStatus {
if self.taking_snapshot.load(Ordering::SeqCst) {
CreationStatus::Ongoing {
block_number: self.taking_snapshot_at.load(Ordering::SeqCst) as u32,
}
} else {
CreationStatus::Inactive
}
}
fn restoration_status(&self) -> RestorationStatus {
let mut cur_status = self.status.lock();
match *cur_status {
@@ -1003,7 +1021,7 @@ mod tests {
assert!(service.manifest().is_none());
assert!(service.chunk(Default::default()).is_none());
assert_eq!(service.status(), RestorationStatus::Inactive);
assert_eq!(service.restoration_status(), RestorationStatus::Inactive);
let manifest = ManifestData {
version: 2,

10
ethcore/src/snapshot/tests/service.rs
View File

@@ -95,7 +95,7 @@ fn restored_is_equivalent() {
service.feed_block_chunk(hash, &chunk);
}
assert_eq!(service.status(), RestorationStatus::Inactive);
assert_eq!(service.restoration_status(), RestorationStatus::Inactive);
for x in 0..NUM_BLOCKS {
let block1 = client.block(BlockId::Number(x as u64)).unwrap();
@@ -265,7 +265,7 @@ fn keep_ancient_blocks() {
service.feed_state_chunk(*hash, &chunk);
}
match service.status() {
match service.restoration_status() {
RestorationStatus::Inactive => (),
RestorationStatus::Failed => panic!("Snapshot Restoration has failed."),
RestorationStatus::Ongoing { .. } => panic!("Snapshot Restoration should be done."),
@@ -334,7 +334,7 @@ fn recover_aborted_recovery() {
service.feed_state_chunk(*hash, &chunk);
}
match service.status() {
match service.restoration_status() {
RestorationStatus::Ongoing {
block_chunks_done,
state_chunks_done,
@@ -352,7 +352,7 @@ fn recover_aborted_recovery() {
// And try again!
service.init_restore(manifest.clone(), true).unwrap();
match service.status() {
match service.restoration_status() {
RestorationStatus::Ongoing {
block_chunks_done,
state_chunks_done,
@@ -371,7 +371,7 @@ fn recover_aborted_recovery() {
// And try again!
service.init_restore(manifest.clone(), true).unwrap();
match service.status() {
match service.restoration_status() {
RestorationStatus::Ongoing {
block_chunks_done,
state_chunks_done,

7
ethcore/src/snapshot/traits.rs
View File

@@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
use super::{ManifestData, RestorationStatus};
use super::{CreationStatus, ManifestData, RestorationStatus};
use bytes::Bytes;
use ethereum_types::H256;
@@ -37,7 +37,10 @@ pub trait SnapshotService: Sync + Send {
fn chunk(&self, hash: H256) -> Option<Bytes>;
/// Ask the snapshot service for the restoration status.
fn status(&self) -> RestorationStatus;
fn restoration_status(&self) -> RestorationStatus;
/// Ask the snapshot service for the creation status.
fn creation_status(&self) -> CreationStatus;
/// Begin snapshot restoration.
/// If restoration in-progress, this will reset it.

20
ethcore/src/state_db.rs
View File

@@ -17,7 +17,7 @@
//! State database abstraction. For more info, see the doc for `StateDB`
use std::{
collections::{HashSet, VecDeque},
collections::{BTreeMap, HashSet, VecDeque},
io,
sync::Arc,
};
@@ -394,13 +394,17 @@ impl StateDB {
}
/// Heap size used.
pub fn mem_used(&self) -> usize {
// TODO: account for LRU-cache overhead; this is a close approximation.
self.db.mem_used() + {
let accounts = self.account_cache.lock().accounts.len();
let code_size = self.code_cache.lock().current_size();
code_size + accounts * ::std::mem::size_of::<Option<Account>>()
}
pub fn get_sizes(&self, sizes: &mut BTreeMap<String, usize>) {
self.db.get_sizes(sizes);
sizes.insert(
String::from("account_cache_len"),
self.account_cache.lock().accounts.len(),
);
sizes.insert(
String::from("code_cache_size"),
self.code_cache.lock().current_size(),
);
}
/// Returns underlying `JournalDB`.