informant: display I/O stats (#11523)
* informant: collect I/O stats for state_db * informat: debug i/o log * informat: remove unused cache hit ratio * Cargo.lock: cargo update -p librocksdb-sys * [deps]: upgrade kvdb-rocksdb to 0.6 * Update ethcore/types/src/client_types.rs Co-Authored-By: David <dvdplm@gmail.com> Co-authored-by: David <dvdplm@gmail.com>
This commit is contained in:
Andronik Ordian
GitHub
parent
3231454bb1
commit
70c4ed7fa0
100
Cargo.lock
generated
100
Cargo.lock
generated
@ -313,23 +313,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.49.2"
|
||||
version = "0.53.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "846a1fba6535362a01487ef6b10f0275faa12e5c5d835c5c1c627aabc46ccbd6"
|
||||
checksum = "6bb26d6a69a335b8cb0e7c7e9775cd5666611dc50a37177c3f2cedcfc040e8c8"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"cfg-if",
|
||||
"clang-sys",
|
||||
"clap",
|
||||
"env_logger 0.6.2",
|
||||
"fxhash",
|
||||
"env_logger 0.7.1",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"log",
|
||||
"peeking_take_while",
|
||||
"proc-macro2 0.4.20",
|
||||
"quote 0.6.8",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"which",
|
||||
]
|
||||
@ -534,11 +535,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.3.6"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
|
||||
checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
|
||||
dependencies = [
|
||||
"nom",
|
||||
"nom 5.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -568,9 +569,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "0.28.1"
|
||||
version = "0.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
|
||||
checksum = "f92986241798376849e1a007827041fed9bb36195822c2049d18e174420e0534"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
@ -1089,6 +1090,19 @@ dependencies = [
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "error-chain"
|
||||
version = "0.12.0"
|
||||
@ -1217,7 +1231,7 @@ dependencies = [
|
||||
"keccak-hash",
|
||||
"kvdb",
|
||||
"kvdb-memorydb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"machine",
|
||||
@ -1583,7 +1597,7 @@ dependencies = [
|
||||
"ethcore-sync",
|
||||
"ethereum-types",
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"log",
|
||||
"snapshot",
|
||||
"spec",
|
||||
@ -2166,9 +2180,9 @@ checksum = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.1.1"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e"
|
||||
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
|
||||
dependencies = [
|
||||
"quick-error",
|
||||
]
|
||||
@ -2599,6 +2613,25 @@ dependencies = [
|
||||
"smallvec 1.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kvdb-rocksdb"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fecd50b14a534125228d7039951f92aaff742aff151c04546347aba4d3b4fbc"
|
||||
dependencies = [
|
||||
"fs-swap",
|
||||
"interleaved-ordered",
|
||||
"kvdb",
|
||||
"log",
|
||||
"num_cpus",
|
||||
"owning_ref 0.4.0",
|
||||
"parity-util-mem",
|
||||
"parking_lot 0.10.0",
|
||||
"regex",
|
||||
"rocksdb",
|
||||
"smallvec 1.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
@ -2636,9 +2669,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "librocksdb-sys"
|
||||
version = "6.2.4"
|
||||
version = "6.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be"
|
||||
checksum = "4e3b727e2dd20ec2fb7ed93f23d9fd5328a0871185485ebdaff007b47d3e27e4"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
@ -2855,7 +2888,7 @@ name = "migration-rocksdb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"log",
|
||||
"maplit",
|
||||
"tempdir",
|
||||
@ -3019,6 +3052,16 @@ dependencies = [
|
||||
"version_check 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "5.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"version_check 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "null-engine"
|
||||
version = "0.1.0"
|
||||
@ -3169,7 +3212,7 @@ dependencies = [
|
||||
"jsonrpc-core",
|
||||
"keccak-hash",
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"log",
|
||||
"migration-rocksdb",
|
||||
"node-filter",
|
||||
@ -3468,7 +3511,7 @@ dependencies = [
|
||||
"jsonrpc-server-utils",
|
||||
"keccak-hash",
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.5.0",
|
||||
"lazy_static",
|
||||
"libsecp256k1",
|
||||
"log",
|
||||
@ -4321,6 +4364,12 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hex"
|
||||
version = "1.0.0"
|
||||
@ -4631,7 +4680,7 @@ dependencies = [
|
||||
"keccak-hash",
|
||||
"keccak-hasher 0.1.1",
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"num_cpus",
|
||||
@ -4677,7 +4726,7 @@ dependencies = [
|
||||
"keccak-hash",
|
||||
"keccak-hasher 0.1.1",
|
||||
"kvdb",
|
||||
"kvdb-rocksdb",
|
||||
"kvdb-rocksdb 0.6.0",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"parity-bytes",
|
||||
@ -5783,16 +5832,15 @@ version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164"
|
||||
dependencies = [
|
||||
"nom",
|
||||
"nom 4.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "2.0.1"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
|
||||
checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"libc",
|
||||
]
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ journaldb = { path = "util/journaldb" }
|
||||
jsonrpc-core = "14.0.3"
|
||||
keccak-hash = "0.4.0"
|
||||
kvdb = "0.4.0"
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
log = "0.4"
|
||||
migration-rocksdb = { path = "util/migration-rocksdb" }
|
||||
node-filter = { path = "ethcore/node-filter" }
|
||||
|
||||
@ -33,7 +33,7 @@ journaldb = { path = "../util/journaldb" }
|
||||
keccak-hash = "0.4.0"
|
||||
kvdb = "0.4.0"
|
||||
kvdb-memorydb = { version = "0.4.0", optional = true }
|
||||
kvdb-rocksdb = { version = "0.5.0", optional = true }
|
||||
kvdb-rocksdb = { version = "0.6.0", optional = true }
|
||||
lazy_static = { version = "1.3", optional = true }
|
||||
log = "0.4"
|
||||
machine = { path = "./machine" }
|
||||
@ -78,7 +78,7 @@ ethjson = { path = "../json", features = ["test-helpers"] }
|
||||
parity-crypto = { version = "0.5.0", features = ["publickey"] }
|
||||
fetch = { path = "../util/fetch" }
|
||||
kvdb-memorydb = "0.4.0"
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
lazy_static = "1.3"
|
||||
machine = { path = "./machine", features = ["test-helpers"] }
|
||||
parity-runtime = "0.1.1"
|
||||
|
||||
@ -23,5 +23,5 @@ trace-time = "0.1"
|
||||
[dev-dependencies]
|
||||
ethcore = { path = "..", features = ["test-helpers"] }
|
||||
ethcore-db = { path = "../db" }
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
tempdir = "0.3"
|
||||
|
||||
@ -53,7 +53,7 @@ ethabi-contract = "9.0.0"
|
||||
ethabi-derive = "9.0.1"
|
||||
ethcore = { path = "..", features = ["test-helpers"] }
|
||||
ethkey = { path = "../../accounts/ethkey" }
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
lazy_static = { version = "1.3" }
|
||||
spec = { path = "../spec" }
|
||||
tempdir = "0.3"
|
||||
|
||||
@ -24,7 +24,7 @@ journaldb = { path = "../../../util/journaldb" }
|
||||
keccak-hash = "0.4.0"
|
||||
keccak-hasher = { path = "../../../util/keccak-hasher" }
|
||||
kvdb = "0.4.0"
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
log = "0.4.8"
|
||||
parking_lot = "0.10.0"
|
||||
parity-crypto = { version = "0.5.0", features = ["publickey"] }
|
||||
|
||||
@ -109,7 +109,7 @@ use types::{
|
||||
BlockNumber,
|
||||
call_analytics::CallAnalytics,
|
||||
chain_notify::{ChainMessageType, ChainRoute, NewBlocks},
|
||||
client_types::{ClientReport, Mode, StateResult},
|
||||
client_types::{ClientReport, IoStats, Mode, StateResult},
|
||||
encoded,
|
||||
engines::{
|
||||
epoch::{PendingTransition, Transition as EpochTransition},
|
||||
@ -1096,7 +1096,19 @@ impl Client {
|
||||
/// Get the report.
|
||||
pub fn report(&self) -> ClientReport {
|
||||
let mut report = self.report.read().clone();
|
||||
report.state_db_mem = self.state_db.read().mem_used();
|
||||
let state_db = self.state_db.read();
|
||||
report.state_db_mem = state_db.mem_used();
|
||||
let io_stats = state_db.journal_db().io_stats();
|
||||
report.io_stats = IoStats {
|
||||
transactions: io_stats.transactions,
|
||||
reads: io_stats.reads,
|
||||
cache_reads: io_stats.cache_reads,
|
||||
writes: io_stats.writes,
|
||||
bytes_read: io_stats.bytes_read,
|
||||
cache_read_bytes: io_stats.cache_read_bytes,
|
||||
bytes_written: io_stats.bytes_written,
|
||||
};
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
|
||||
@ -62,6 +62,27 @@ pub struct ClientReport {
|
||||
pub gas_processed: U256,
|
||||
/// Memory used by state DB
|
||||
pub state_db_mem: usize,
|
||||
/// I/O statistics for the state DB.
|
||||
pub io_stats: IoStats,
|
||||
}
|
||||
|
||||
/// I/O statistics.
|
||||
#[derive(Default, Debug, Clone, Eq, PartialEq)]
|
||||
pub struct IoStats {
|
||||
/// Number of transaction.
|
||||
pub transactions: u64,
|
||||
/// Number of read operations.
|
||||
pub reads: u64,
|
||||
/// Number of reads resulted in a read from cache.
|
||||
pub cache_reads: u64,
|
||||
/// Number of write operations.
|
||||
pub writes: u64,
|
||||
/// Number of bytes read.
|
||||
pub bytes_read: u64,
|
||||
/// Number of bytes read from cache.
|
||||
pub cache_read_bytes: u64,
|
||||
/// Number of bytes write.
|
||||
pub bytes_written: u64,
|
||||
}
|
||||
|
||||
impl ClientReport {
|
||||
|
||||
@ -234,7 +234,7 @@ impl<K: Kind, C> VerificationQueue<K, C> {
|
||||
let number_of_threads = if scale_verifiers {
|
||||
max_verifiers
|
||||
} else {
|
||||
cmp::min(default_amount, max_verifiers)
|
||||
default_amount
|
||||
};
|
||||
|
||||
let state = Arc::new((Mutex::new(State::Work(default_amount)), Condvar::new()));
|
||||
|
||||
@ -257,6 +257,14 @@ impl<T: InformantData> Informant<T> {
|
||||
(diffed, full_report)
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "io_stats",
|
||||
"{} reads, {} writes, {} transactions",
|
||||
client_report.io_stats.reads,
|
||||
client_report.io_stats.writes,
|
||||
client_report.io_stats.transactions,
|
||||
);
|
||||
|
||||
let Report {
|
||||
importing,
|
||||
chain_info,
|
||||
|
||||
@ -102,6 +102,10 @@ impl JournalDB for ArchiveDB {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn io_stats(&self) -> kvdb::IoStats {
|
||||
self.backing.io_stats(kvdb::IoStatsKind::SincePrevious)
|
||||
}
|
||||
|
||||
fn mem_used(&self) -> usize {
|
||||
self.overlay.malloc_size_of()
|
||||
}
|
||||
|
||||
@ -322,6 +322,10 @@ impl JournalDB for EarlyMergeDB {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn io_stats(&self) -> kvdb::IoStats {
|
||||
self.backing.io_stats(kvdb::IoStatsKind::SincePrevious)
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none()
|
||||
}
|
||||
|
||||
@ -45,6 +45,9 @@ pub trait JournalDB: HashDB<KeccakHasher, DBValue> {
|
||||
/// Returns heap memory size used
|
||||
fn mem_used(&self) -> usize;
|
||||
|
||||
/// Returns the I/O statistics.
|
||||
fn io_stats(&self) -> kvdb::IoStats;
|
||||
|
||||
/// Returns the size of journalled state in memory.
|
||||
/// This function has a considerable speed requirement --
|
||||
/// it must be fast enough to call several times per block imported.
|
||||
|
||||
@ -244,6 +244,10 @@ impl JournalDB for OverlayRecentDB {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn io_stats(&self) -> kvdb::IoStats {
|
||||
self.backing.io_stats(kvdb::IoStatsKind::SincePrevious)
|
||||
}
|
||||
|
||||
fn mem_used(&self) -> usize {
|
||||
let mut ops = new_malloc_size_ops();
|
||||
let mut mem = self.transaction_overlay.size_of(&mut ops);
|
||||
|
||||
@ -98,6 +98,10 @@ impl JournalDB for RefCountedDB {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
fn io_stats(&self) -> kvdb::IoStats {
|
||||
self.backing.io_stats(kvdb::IoStatsKind::SincePrevious)
|
||||
}
|
||||
|
||||
fn mem_used(&self) -> usize {
|
||||
let mut ops = new_malloc_size_ops();
|
||||
self.inserts.size_of(&mut ops) + self.removes.size_of(&mut ops)
|
||||
|
||||
@ -8,7 +8,7 @@ license = "GPL-3.0"
|
||||
[dependencies]
|
||||
log = "0.4"
|
||||
kvdb = "0.4.0"
|
||||
kvdb-rocksdb = "0.5.0"
|
||||
kvdb-rocksdb = "0.6.0"
|
||||
|
||||
[dev-dependencies]
|
||||
tempdir = "0.3"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user