grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improved metrics (#240)

Browse Source 
Added db metrics (kvdb_bytes_read, kvdb_bytes_written, kvdb_reads, kvdb_writes)
Added --metrics-prefix=[prefix]
This commit is contained in:
adria0.eth
2021-03-03 22:44:35 +01:00
committed by GitHub
parent ba011eba15
commit 0fcb102f03
34 changed files with 440 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
crates/ethcore/src/client/client.rs
View File

@@ -33,12 +33,12 @@ use blockchain::{
};
use bytes::{Bytes, ToPretty};
use call_contract::CallContract;
use db::{DBTransaction, DBValue, KeyValueDB};
use error::Error;
use ethcore_miner::pool::VerifiedTransaction;
use ethereum_types::{Address, H256, H264, U256};
use hash::keccak;
use itertools::Itertools;
use kvdb::{DBTransaction, DBValue, KeyValueDB};
use parking_lot::{Mutex, RwLock};
use rand::OsRng;
use rlp::{PayloadInfo, Rlp};
@@ -88,7 +88,7 @@ use snapshot::{self, io as snapshot_io, SnapshotClient};
use spec::Spec;
use state::{self, State};
use state_db::StateDB;
use stats::{prometheus, prometheus_counter, prometheus_gauge, PrometheusMetrics};
use stats::{PrometheusMetrics, PrometheusRegistry};
use trace::{
self, Database as TraceDatabase, ImportRequest as TraceImportRequest, LocalizedTrace, TraceDB,
};
@@ -3236,41 +3236,36 @@ impl IoChannelQueue {
}
impl PrometheusMetrics for Client {
fn prometheus_metrics(&self, r: &mut prometheus::Registry) {
fn prometheus_metrics(&self, r: &mut PrometheusRegistry) {
// gas, tx & blocks
let report = self.report();
for (key, value) in report.item_sizes.iter() {
prometheus_gauge(
r,
r.register_gauge(
&key,
format!("Total item number of {}", key).as_str(),
*value as i64,
);
}
prometheus_counter(
r,
r.register_counter(
"import_gas",
"Gas processed",
report.gas_processed.as_u64() as i64,
);
prometheus_counter(
r,
r.register_counter(
"import_blocks",
"Blocks imported",
report.blocks_imported as i64,
);
prometheus_counter(
r,
r.register_counter(
"import_txs",
"Transactions applied",
report.transactions_applied as i64,
);
let state_db = self.state_db.read();
prometheus_gauge(
r,
r.register_gauge(
"statedb_cache_size",
"State DB cache size",
state_db.cache_size() as i64,
@@ -3278,32 +3273,27 @@ impl PrometheusMetrics for Client {
// blockchain cache
let blockchain_cache_info = self.blockchain_cache_info();
prometheus_gauge(
r,
r.register_gauge(
"blockchaincache_block_details",
"BlockDetails cache size",
blockchain_cache_info.block_details as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"blockchaincache_block_recipts",
"Block receipts size",
blockchain_cache_info.block_receipts as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"blockchaincache_blocks",
"Blocks cache size",
blockchain_cache_info.blocks as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"blockchaincache_txaddrs",
"Transaction addresses cache size",
blockchain_cache_info.transaction_addresses as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"blockchaincache_size",
"Total blockchain cache size",
blockchain_cache_info.total() as i64,
@@ -3321,22 +3311,19 @@ impl PrometheusMetrics for Client {
.map(|last| (first, U256::from(last)))
});
if let Some((first, last)) = gap {
prometheus_gauge(
r,
r.register_gauge(
"chain_warpsync_gap_first",
"Warp sync gap, first block",
first.as_u64() as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"chain_warpsync_gap_last",
"Warp sync gap, last block",
last.as_u64() as i64,
);
}
prometheus_gauge(
r,
r.register_gauge(
"chain_block",
"Best block number",
chain.best_block_number as i64,
@@ -3344,14 +3331,12 @@ impl PrometheusMetrics for Client {
// prunning info
let prunning = self.pruning_info();
prometheus_gauge(
r,
r.register_gauge(
"prunning_earliest_chain",
"The first block which everything can be served after",
prunning.earliest_chain as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"prunning_earliest_state",
"The first block where state requests may be served",
prunning.earliest_state as i64,
@@ -3359,36 +3344,34 @@ impl PrometheusMetrics for Client {
// queue info
let queue = self.queue_info();
prometheus_gauge(
r,
r.register_gauge(
"queue_mem_used",
"Queue heap memory used in bytes",
queue.mem_used as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"queue_size_total",
"The total size of the queues",
queue.total_queue_size() as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"queue_size_unverified",
"Number of queued items pending verification",
queue.unverified_queue_size as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"queue_size_verified",
"Number of verified queued items pending import",
queue.verified_queue_size as i64,
);
prometheus_gauge(
r,
r.register_gauge(
"queue_size_verifying",
"Number of items being verified",
queue.verifying_queue_size as i64,
);
// database info
self.db.read().key_value().prometheus_metrics(r);
}
}

5
crates/ethcore/src/client/evm_test_client.rs
View File

@@ -25,7 +25,6 @@ use executive;
use factory::{self, Factories};
use journaldb;
use kvdb::{self, KeyValueDB};
use kvdb_memorydb;
use pod_state;
use spec;
use state;
@@ -181,7 +180,7 @@ impl<'a> EvmTestClient<'a> {
spec: &'a spec::Spec,
factories: &Factories,
) -> Result<state::State<state_db::StateDB>, EvmTestError> {
let db = Arc::new(kvdb_memorydb::create(
let db = Arc::new(ethcore_db::InMemoryWithMetrics::create(
db::NUM_COLUMNS.expect("We use column-based DB; qed"),
));
let journal_db =
@@ -211,7 +210,7 @@ impl<'a> EvmTestClient<'a> {
factories: &Factories,
pod_state: pod_state::PodState,
) -> Result<state::State<state_db::StateDB>, EvmTestError> {
let db = Arc::new(kvdb_memorydb::create(
let db = Arc::new(ethcore_db::InMemoryWithMetrics::create(
db::NUM_COLUMNS.expect("We use column-based DB; qed"),
));
let journal_db =

7
crates/ethcore/src/client/test_client.rs
View File

@@ -34,7 +34,6 @@ use ethtrie;
use hash::keccak;
use itertools::Itertools;
use kvdb::DBValue;
use kvdb_memorydb;
use parking_lot::RwLock;
use rlp::RlpStream;
use rustc_hex::FromHex;
@@ -75,7 +74,7 @@ use miner::{self, Miner, MinerService};
use spec::Spec;
use state::StateInfo;
use state_db::StateDB;
use stats::{prometheus, PrometheusMetrics};
use stats::{PrometheusMetrics, PrometheusRegistry};
use trace::LocalizedTrace;
use verification::queue::{kind::blocks::Unverified, QueueInfo};
@@ -409,7 +408,7 @@ impl TestBlockChainClient {
/// Get temporary db state1
pub fn get_temp_state_db() -> StateDB {
let db = kvdb_memorydb::create(NUM_COLUMNS.unwrap_or(0));
let db = ethcore_db::InMemoryWithMetrics::create(NUM_COLUMNS.unwrap_or(0));
let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, COL_STATE);
StateDB::new(journal_db, 1024 * 1024)
}
@@ -1132,5 +1131,5 @@ impl super::traits::EngineClient for TestBlockChainClient {
}
impl PrometheusMetrics for TestBlockChainClient {
fn prometheus_metrics(&self, _r: &mut prometheus::Registry) {}
fn prometheus_metrics(&self, _r: &mut PrometheusRegistry) {}
}

2
crates/ethcore/src/lib.rs
View File

@@ -73,8 +73,6 @@ extern crate ethcore_accounts as accounts;
extern crate ethcore_stratum;
#[cfg(feature = "json-tests")]
extern crate globset;
#[cfg(any(test, feature = "test-helpers"))]
extern crate kvdb_memorydb;
#[cfg(any(test, feature = "kvdb-rocksdb"))]
extern crate kvdb_rocksdb;
#[cfg(test)]

2
crates/ethcore/src/snapshot/consensus/authority.rs
View File

@@ -32,9 +32,9 @@ use snapshot::{Error, ManifestData, Progress};
use blockchain::{BlockChain, BlockChainDB, BlockProvider};
use bytes::Bytes;
use db::KeyValueDB;
use ethereum_types::{H256, U256};
use itertools::{Itertools, Position};
use kvdb::KeyValueDB;
use rlp::{Rlp, RlpStream};
use types::{
encoded, header::Header, ids::BlockId, receipt::TypedReceipt, transaction::TypedTransaction,

2
crates/ethcore/src/snapshot/consensus/work.rs
View File

@@ -32,9 +32,9 @@ use std::{
use blockchain::{BlockChain, BlockChainDB, BlockProvider};
use bytes::Bytes;
use db::KeyValueDB;
use engines::EthEngine;
use ethereum_types::H256;
use kvdb::KeyValueDB;
use rand::OsRng;
use rlp::{Rlp, RlpStream};
use snapshot::{block::AbridgedBlock, Error, ManifestData, Progress};

2
crates/ethcore/src/snapshot/mod.rs
View File

@@ -35,12 +35,12 @@ use engines::EthEngine;
use types::{header::Header, ids::BlockId};
use bytes::Bytes;
use db::{DBValue, KeyValueDB};
use ethereum_types::H256;
use ethtrie::{TrieDB, TrieDBMut};
use hash_db::HashDB;
use journaldb::{self, Algorithm, JournalDB};
use keccak_hasher::KeccakHasher;
use kvdb::{DBValue, KeyValueDB};
use num_cpus;
use parking_lot::Mutex;
use rlp::{Rlp, RlpStream};

28
crates/ethcore/src/snapshot/tests/state.rs
View File

@@ -82,8 +82,11 @@ fn snap_and_restore() {
let db_path = tempdir.path().join("db");
let db = {
let new_db = Arc::new(Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap());
let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent);
let new_db = Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap();
let new_db_with_metrics: Arc<dyn ethcore_db::KeyValueDB> =
Arc::new(ethcore_db::DatabaseWithMetrics::new(new_db));
let mut rebuilder =
StateRebuilder::new(new_db_with_metrics.clone(), Algorithm::OverlayRecent);
let reader = PackedReader::new(&snap_file).unwrap().unwrap();
let flag = AtomicBool::new(true);
@@ -98,7 +101,7 @@ fn snap_and_restore() {
assert_eq!(rebuilder.state_root(), state_root);
rebuilder.finalize(1000, H256::default()).unwrap();
new_db
new_db_with_metrics
};
let new_db = journaldb::new(db, Algorithm::OverlayRecent, ::db::COL_STATE);
@@ -163,10 +166,11 @@ fn get_code_from_prev_chunk() {
let tempdir = TempDir::new("").unwrap();
let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
let new_db = Arc::new(Database::open(&db_cfg, tempdir.path().to_str().unwrap()).unwrap());
let new_db = Database::open(&db_cfg, tempdir.path().to_str().unwrap()).unwrap();
let new_db_with_metrics = Arc::new(db::DatabaseWithMetrics::new(new_db));
{
let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent);
let mut rebuilder =
StateRebuilder::new(new_db_with_metrics.clone(), Algorithm::OverlayRecent);
let flag = AtomicBool::new(true);
rebuilder.feed(&chunk1, &flag).unwrap();
@@ -175,7 +179,11 @@ fn get_code_from_prev_chunk() {
rebuilder.finalize(1000, H256::random()).unwrap();
}
let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ::db::COL_STATE);
let state_db = journaldb::new(
new_db_with_metrics,
Algorithm::OverlayRecent,
::db::COL_STATE,
);
assert_eq!(state_db.earliest_era(), Some(1000));
}
@@ -214,8 +222,10 @@ fn checks_flag() {
let tempdir = TempDir::new("").unwrap();
let db_path = tempdir.path().join("db");
{
let new_db = Arc::new(Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap());
let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent);
let new_db = Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap();
let new_db_with_metrics = Arc::new(db::DatabaseWithMetrics::new(new_db));
let mut rebuilder =
StateRebuilder::new(new_db_with_metrics.clone(), Algorithm::OverlayRecent);
let reader = PackedReader::new(&snap_file).unwrap().unwrap();
let flag = AtomicBool::new(false);

2
crates/ethcore/src/spec/spec.rs
View File

@@ -970,7 +970,7 @@ impl Spec {
let factories = Default::default();
let mut db = journaldb::new(
Arc::new(kvdb_memorydb::create(0)),
Arc::new(db::InMemoryWithMetrics::create(0)),
journaldb::Algorithm::Archive,
None,
);

23
crates/ethcore/src/test_helpers.rs
View File

@@ -23,12 +23,12 @@ use blockchain::{
};
use blooms_db;
use bytes::Bytes;
use db::KeyValueDB;
use ethereum_types::{Address, H256, U256};
use ethkey::KeyPair;
use evm::Factory as EvmFactory;
use hash::keccak;
use io::IoChannel;
use kvdb::KeyValueDB;
use kvdb_rocksdb::{self, Database, DatabaseConfig};
use parking_lot::RwLock;
use rlp::{self, RlpStream};
@@ -350,6 +350,10 @@ impl BlockChainDB for TestBlockChainDB {
}
}
impl stats::PrometheusMetrics for TestBlockChainDB {
fn prometheus_metrics(&self, _: &mut stats::PrometheusRegistry) {}
}
/// Creates new test instance of `BlockChainDB`
pub fn new_db() -> Arc<dyn BlockChainDB> {
let blooms_dir = TempDir::new("").unwrap();
@@ -360,7 +364,9 @@ pub fn new_db() -> Arc<dyn BlockChainDB> {
trace_blooms: blooms_db::Database::open(trace_blooms_dir.path()).unwrap(),
_blooms_dir: blooms_dir,
_trace_blooms_dir: trace_blooms_dir,
key_value: Arc::new(::kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap())),
key_value: Arc::new(ethcore_db::InMemoryWithMetrics::create(
::db::NUM_COLUMNS.unwrap(),
)),
};
Arc::new(db)
@@ -374,13 +380,13 @@ pub fn new_temp_db(tempdir: &Path) -> Arc<dyn BlockChainDB> {
let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
let key_value_db = Database::open(&db_config, key_value_dir.to_str().unwrap()).unwrap();
let key_value_db_with_metrics = ethcore_db::DatabaseWithMetrics::new(key_value_db);
let db = TestBlockChainDB {
blooms: blooms_db::Database::open(blooms_dir.path()).unwrap(),
trace_blooms: blooms_db::Database::open(trace_blooms_dir.path()).unwrap(),
_blooms_dir: blooms_dir,
_trace_blooms_dir: trace_blooms_dir,
key_value: Arc::new(key_value_db),
key_value: Arc::new(key_value_db_with_metrics),
};
Arc::new(db)
@@ -413,13 +419,14 @@ pub fn restoration_db_handler(
&self.trace_blooms
}
}
impl stats::PrometheusMetrics for RestorationDB {
fn prometheus_metrics(&self, _: &mut stats::PrometheusRegistry) {}
}
impl BlockChainDBHandler for RestorationDBHandler {
fn open(&self, db_path: &Path) -> io::Result<Arc<dyn BlockChainDB>> {
let key_value = Arc::new(kvdb_rocksdb::Database::open(
&self.config,
&db_path.to_string_lossy(),
)?);
let key_value = kvdb_rocksdb::Database::open(&self.config, &db_path.to_string_lossy())?;
let key_value = Arc::new(db::DatabaseWithMetrics::new(key_value));
let blooms_path = db_path.join("blooms");
let trace_blooms_path = db_path.join("trace_blooms");
fs::create_dir_all(&blooms_path)?;