From 7a857a24aed9ee55f8ddbabf37899924120d53e9 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 17 Feb 2017 16:18:31 +0100 Subject: [PATCH] use new histogram/corpus --- Cargo.lock | 9 +++++++++ ethcore/Cargo.toml | 1 + ethcore/src/client/traits.rs | 35 ++++++++++------------------------ ethcore/src/lib.rs | 1 + ethcore/src/tests/client.rs | 11 +++++------ rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/helpers/dispatch.rs | 8 ++++++-- rpc/src/v1/impls/parity.rs | 2 +- rpc/src/v1/types/histogram.rs | 7 +++---- util/stats/src/lib.rs | 21 +++++++++++--------- 11 files changed, 50 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c51c85ee0..8119910b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,6 +393,7 @@ dependencies = [ "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "stats 0.1.0", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -615,6 +616,7 @@ dependencies = [ "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", + "stats 0.1.0", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2129,6 +2131,13 @@ name = "stable-heap" version = "0.1.0" source = "git+https://github.com/carllerche/stable-heap?rev=3c5cd1ca47#3c5cd1ca4706f167a1de85658b5af0d6d3e65165" +[[package]] +name = "stats" +version = "0.1.0" +dependencies = [ + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.3.0" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 442f8b785..c8a1c7fb5 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -43,6 +43,7 @@ rlp = { path = "../util/rlp" } ethcore-stratum = { path = "../stratum" } ethcore-bloom-journal = { path = "../util/bloom" } hardware-wallet = { path = "../hw" } +stats = { path = "../util/stats" } [dependencies.hyper] git = "https://github.com/ethcore/hyper" diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index dce708b3a..6e1ea9d31 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -16,7 +16,6 @@ use std::collections::BTreeMap; use util::{U256, Address, H256, H2048, Bytes, Itertools}; -use util::stats::Histogram; use blockchain::TreeRoute; use verification::queue::QueueInfo as BlockQueueInfo; use block::{OpenBlock, SealedBlock}; @@ -212,38 +211,24 @@ pub trait BlockChainClient : Sync + Send { fn ready_transactions(&self) -> Vec; /// Sorted list of transaction gas prices from at least last sample_size blocks. - fn gas_price_corpus(&self, sample_size: usize) -> Vec { + fn gas_price_corpus(&self, sample_size: usize) -> ::stats::Corpus { let mut h = self.chain_info().best_block_hash; let mut corpus = Vec::new(); while corpus.is_empty() { for _ in 0..sample_size { - let block = self.block(BlockId::Hash(h)).expect("h is either the best_block_hash or an ancestor; qed"); - let header = block.header_view(); - if header.number() == 0 { - corpus.sort(); - return corpus; + let block = match self.block(BlockId::Hash(h)) { + Some(block) => block, + None => return corpus.into(), + }; + + if block.number() == 0 { + return corpus.into(); } block.transaction_views().iter().foreach(|t| corpus.push(t.gas_price())); - h = header.parent_hash().clone(); + h = block.parent_hash().clone(); } } - corpus.sort(); - corpus - } - - /// Calculate median gas price from recent blocks if they have any transactions. - fn gas_price_median(&self, sample_size: usize) -> Option { - let corpus = self.gas_price_corpus(sample_size); - corpus.get(corpus.len() / 2).cloned() - } - - /// Get the gas price distribution based on recent blocks if they have any transactions. - fn gas_price_histogram(&self, sample_size: usize, bucket_number: usize) -> Option { - let raw_corpus = self.gas_price_corpus(sample_size); - let raw_len = raw_corpus.len(); - // Throw out outliers. - let (corpus, _) = raw_corpus.split_at(raw_len - raw_len / 40); - Histogram::new(corpus, bucket_number) + corpus.into() } /// Get the preferred network ID to sign on diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index be5247340..3a56db51b 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -106,6 +106,7 @@ extern crate lru_cache; extern crate ethcore_stratum; extern crate ethabi; extern crate hardware_wallet; +extern crate stats; #[macro_use] extern crate log; diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index 6c2c02c2d..e239eec5b 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -27,7 +27,6 @@ use miner::Miner; use rlp::View; use spec::Spec; use views::BlockView; -use util::stats::Histogram; use ethkey::{KeyPair, Secret}; use transaction::{PendingTransaction, Transaction, Action, Condition}; use miner::MinerService; @@ -208,11 +207,11 @@ fn can_collect_garbage() { fn can_generate_gas_price_median() { let client_result = generate_dummy_client_with_data(3, 1, slice_into![1, 2, 3]); let client = client_result.reference(); - assert_eq!(Some(U256::from(2)), client.gas_price_median(3)); + assert_eq!(Some(&U256::from(2)), client.gas_price_corpus(3).median()); let client_result = generate_dummy_client_with_data(4, 1, slice_into![1, 4, 3, 2]); let client = client_result.reference(); - assert_eq!(Some(U256::from(3)), client.gas_price_median(4)); + assert_eq!(Some(&U256::from(3)), client.gas_price_corpus(3).median()); } #[test] @@ -220,8 +219,8 @@ fn can_generate_gas_price_histogram() { let client_result = generate_dummy_client_with_data(20, 1, slice_into![6354,8593,6065,4842,7845,7002,689,4958,4250,6098,5804,4320,643,8895,2296,8589,7145,2000,2512,1408]); let client = client_result.reference(); - let hist = client.gas_price_histogram(20, 5).unwrap(); - let correct_hist = Histogram { bucket_bounds: vec_into![643, 2294, 3945, 5596, 7247, 8898], counts: vec![4,2,4,6,4] }; + let hist = client.gas_price_corpus(20).histogram(5).unwrap(); + let correct_hist = ::stats::Histogram { bucket_bounds: vec_into![643, 2294, 3945, 5596, 7247, 8898], counts: vec![4,2,4,6,4] }; assert_eq!(hist, correct_hist); } @@ -230,7 +229,7 @@ fn empty_gas_price_histogram() { let client_result = generate_dummy_client_with_data(20, 0, slice_into![]); let client = client_result.reference(); - assert!(client.gas_price_histogram(20, 5).is_none()); + assert!(client.gas_price_corpus(20).histogram(5).is_none()); } #[test] diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 8b8f9ecd7..91058b990 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -39,6 +39,7 @@ rlp = { path = "../util/rlp" } fetch = { path = "../util/fetch" } parity-reactor = { path = "../util/reactor" } clippy = { version = "0.0.103", optional = true} +stats = { path = "../util/stats" } [features] dev = ["clippy", "ethcore/dev", "ethcore-util/dev", "ethsync/dev"] diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 030a03702..201f41c22 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -44,6 +44,7 @@ extern crate futures; extern crate order_stat; extern crate parity_updater as updater; extern crate parity_reactor; +extern crate stats; #[macro_use] extern crate log; diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index e8fbf9b76..64f5b69af 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -21,11 +21,12 @@ use std::ops::Deref; use std::sync::{Arc, Weak}; use futures::{future, Future, BoxFuture}; +use light::cache::Cache as LightDataCache; use light::client::LightChainClient; use light::on_demand::{request, OnDemand}; use light::TransactionQueue as LightTransactionQueue; use rlp::{self, Stream}; -use util::{Address, H520, H256, U256, Uint, Bytes, RwLock}; +use util::{Address, H520, H256, U256, Uint, Bytes, Mutex, RwLock}; use util::sha3::Hashable; use ethkey::Signature; @@ -162,6 +163,7 @@ pub struct LightDispatcher { sync: Arc, client: Arc, on_demand: Arc, + cache: Arc>, transaction_queue: Arc>, } @@ -173,12 +175,14 @@ impl LightDispatcher { sync: Arc, client: Arc, on_demand: Arc, + cache: Arc>, transaction_queue: Arc>, ) -> Self { LightDispatcher { sync: sync, client: client, on_demand: on_demand, + cache: cache, transaction_queue: transaction_queue, } } @@ -453,7 +457,7 @@ fn decrypt(accounts: &AccountProvider, address: Address, msg: Bytes, password: S pub fn default_gas_price(client: &C, miner: &M) -> U256 where C: MiningBlockChainClient, M: MinerService { - client.gas_price_median(100).unwrap_or_else(|| miner.sensible_gas_price()) + client.gas_price_corpus(100).median().cloned().unwrap_or_else(|| miner.sensible_gas_price()) } /// Convert RPC confirmation payload to signer confirmation payload. diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 431c34e84..8dbb4578f 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -236,7 +236,7 @@ impl Parity for ParityClient where } fn gas_price_histogram(&self) -> Result { - take_weak!(self.client).gas_price_histogram(100, 10).ok_or_else(errors::not_enough_data).map(Into::into) + take_weak!(self.client).gas_price_corpus(100).histogram(10).ok_or_else(errors::not_enough_data).map(Into::into) } fn unsigned_transactions_count(&self) -> Result { diff --git a/rpc/src/v1/types/histogram.rs b/rpc/src/v1/types/histogram.rs index 6cec96469..55d8ae835 100644 --- a/rpc/src/v1/types/histogram.rs +++ b/rpc/src/v1/types/histogram.rs @@ -17,7 +17,6 @@ //! Gas prices histogram. use v1::types::U256; -use util::stats; /// Values of RPC settings. #[derive(Serialize, Deserialize)] @@ -27,11 +26,11 @@ pub struct Histogram { #[serde(rename="bucketBounds")] pub bucket_bounds: Vec, /// Transacion counts for each bucket. - pub counts: Vec, + pub counts: Vec, } -impl From for Histogram { - fn from(h: stats::Histogram) -> Self { +impl From<::stats::Histogram<::util::U256>> for Histogram { + fn from(h: ::stats::Histogram<::util::U256>) -> Self { Histogram { bucket_bounds: h.bucket_bounds.into_iter().map(Into::into).collect(), counts: h.counts diff --git a/util/stats/src/lib.rs b/util/stats/src/lib.rs index ccfca525b..01c988232 100644 --- a/util/stats/src/lib.rs +++ b/util/stats/src/lib.rs @@ -17,14 +17,14 @@ //! Statistical functions and helpers. use std::iter::FromIterator; -use std::ops::{Add, Sub, Div}; +use std::ops::{Add, Sub, Deref, Div}; #[macro_use] extern crate log; /// Sorted corpus of data. #[derive(Debug, Clone, PartialEq)] -pub struct Corpus(Vec); +pub struct Corpus(Vec); impl From> for Corpus { fn from(mut data: Vec) -> Self { @@ -39,6 +39,12 @@ impl FromIterator for Corpus { } } +impl Deref for Corpus { + type Target = [T]; + + fn deref(&self) -> &[T] { &self.0[..] } +} + impl Corpus { /// Get the median element, if it exists. pub fn median(&self) -> Option<&T> { @@ -54,20 +60,17 @@ impl Corpus { pub fn len(&self) -> usize { self.0.len() } - - /// Split the corpus at a given point. - pub fn split_at(self, idx: usize) -> (Self, Self) { - let (left, right) = self.0.split_at(idx); - (Corpus(left), Corpus(right)) - } } impl Corpus where T: Add + Sub + Div + From { /// Create a histogram of this corpus if it at least spans the buckets. Bounds are left closed. + /// Excludes outliers. pub fn histogram(&self, bucket_number: usize) -> Option> { - Histogram::create(&self.0, bucket_number) + // TODO: get outliers properly. + let upto = self.len() - self.len() / 40; + Histogram::create(&self.0[..upto], bucket_number) } }