use new histogram/corpus

Cargo.lock

@@ -393,6 +393,7 @@ dependencies = [
  "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stats 0.1.0",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -615,6 +616,7 @@ dependencies = [
  "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stats 0.1.0",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2129,6 +2131,13 @@ name = "stable-heap"
 version = "0.1.0"
 source = "git+https://github.com/carllerche/stable-heap?rev=3c5cd1ca47#3c5cd1ca4706f167a1de85658b5af0d6d3e65165"
 
+[[package]]
+name = "stats"
+version = "0.1.0"
+dependencies = [
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "strsim"
 version = "0.3.0"

ethcore/Cargo.toml

@@ -43,6 +43,7 @@ rlp = { path = "../util/rlp" }
 ethcore-stratum = { path = "../stratum" }
 ethcore-bloom-journal = { path = "../util/bloom" }
 hardware-wallet = { path = "../hw" }
+stats = { path = "../util/stats" }
 
 [dependencies.hyper]
 git = "https://github.com/ethcore/hyper"

ethcore/src/client/traits.rs

@@ -16,7 +16,6 @@
 
 use std::collections::BTreeMap;
 use util::{U256, Address, H256, H2048, Bytes, Itertools};
-use util::stats::Histogram;
 use blockchain::TreeRoute;
 use verification::queue::QueueInfo as BlockQueueInfo;
 use block::{OpenBlock, SealedBlock};
@@ -212,38 +211,24 @@ pub trait BlockChainClient : Sync + Send {
 	fn ready_transactions(&self) -> Vec<PendingTransaction>;
 
 	/// Sorted list of transaction gas prices from at least last sample_size blocks.
-	fn gas_price_corpus(&self, sample_size: usize) -> Vec<U256> {
+	fn gas_price_corpus(&self, sample_size: usize) -> ::stats::Corpus<U256> {
 		let mut h = self.chain_info().best_block_hash;
 		let mut corpus = Vec::new();
 		while corpus.is_empty() {
 			for _ in 0..sample_size {
-				let block = self.block(BlockId::Hash(h)).expect("h is either the best_block_hash or an ancestor; qed");
+				let block = match self.block(BlockId::Hash(h)) {
-				let header = block.header_view();
+					Some(block) => block,
-				if header.number() == 0 {
+					None => return corpus.into(),
-					corpus.sort();
+				};
-					return corpus;
+
+				if block.number() == 0 {
+					return corpus.into();
 				}
 				block.transaction_views().iter().foreach(|t| corpus.push(t.gas_price()));
-				h = header.parent_hash().clone();
+				h = block.parent_hash().clone();
 			}
 		}
-		corpus.sort();
+		corpus.into()
-		corpus
-	}
-
-	/// Calculate median gas price from recent blocks if they have any transactions.
-	fn gas_price_median(&self, sample_size: usize) -> Option<U256> {
-		let corpus = self.gas_price_corpus(sample_size);
-		corpus.get(corpus.len() / 2).cloned()
-	}
-
-	/// Get the gas price distribution based on recent blocks if they have any transactions.
-	fn gas_price_histogram(&self, sample_size: usize, bucket_number: usize) -> Option<Histogram> {
-		let raw_corpus = self.gas_price_corpus(sample_size);
-		let raw_len = raw_corpus.len();
-		// Throw out outliers.
-		let (corpus, _) = raw_corpus.split_at(raw_len - raw_len / 40);
-		Histogram::new(corpus, bucket_number)
 	}
 
 	/// Get the preferred network ID to sign on

ethcore/src/lib.rs

@@ -106,6 +106,7 @@ extern crate lru_cache;
 extern crate ethcore_stratum;
 extern crate ethabi;
 extern crate hardware_wallet;
+extern crate stats;
 
 #[macro_use]
 extern crate log;

ethcore/src/tests/client.rs

@@ -27,7 +27,6 @@ use miner::Miner;
 use rlp::View;
 use spec::Spec;
 use views::BlockView;
-use util::stats::Histogram;
 use ethkey::{KeyPair, Secret};
 use transaction::{PendingTransaction, Transaction, Action, Condition};
 use miner::MinerService;
@@ -208,11 +207,11 @@ fn can_collect_garbage() {
 fn can_generate_gas_price_median() {
 	let client_result = generate_dummy_client_with_data(3, 1, slice_into![1, 2, 3]);
 	let client = client_result.reference();
-	assert_eq!(Some(U256::from(2)), client.gas_price_median(3));
+	assert_eq!(Some(&U256::from(2)), client.gas_price_corpus(3).median());
 
 	let client_result = generate_dummy_client_with_data(4, 1, slice_into![1, 4, 3, 2]);
 	let client = client_result.reference();
-	assert_eq!(Some(U256::from(3)), client.gas_price_median(4));
+	assert_eq!(Some(&U256::from(3)), client.gas_price_corpus(3).median());
 }
 
 #[test]
@@ -220,8 +219,8 @@ fn can_generate_gas_price_histogram() {
 	let client_result = generate_dummy_client_with_data(20, 1, slice_into![6354,8593,6065,4842,7845,7002,689,4958,4250,6098,5804,4320,643,8895,2296,8589,7145,2000,2512,1408]);
 	let client = client_result.reference();
 
-	let hist = client.gas_price_histogram(20, 5).unwrap();
+	let hist = client.gas_price_corpus(20).histogram(5).unwrap();
-	let correct_hist = Histogram { bucket_bounds: vec_into![643, 2294, 3945, 5596, 7247, 8898], counts: vec![4,2,4,6,4] };
+	let correct_hist = ::stats::Histogram { bucket_bounds: vec_into![643, 2294, 3945, 5596, 7247, 8898], counts: vec![4,2,4,6,4] };
 	assert_eq!(hist, correct_hist);
 }
 
@@ -230,7 +229,7 @@ fn empty_gas_price_histogram() {
 	let client_result = generate_dummy_client_with_data(20, 0, slice_into![]);
 	let client = client_result.reference();
 
-	assert!(client.gas_price_histogram(20, 5).is_none());
+	assert!(client.gas_price_corpus(20).histogram(5).is_none());
 }
 
 #[test]

rpc/Cargo.toml

@@ -39,6 +39,7 @@ rlp = { path = "../util/rlp" }
 fetch = { path = "../util/fetch" }
 parity-reactor = { path = "../util/reactor" }
 clippy = { version = "0.0.103", optional = true}
+stats = { path = "../util/stats" }
 
 [features]
 dev = ["clippy", "ethcore/dev", "ethcore-util/dev", "ethsync/dev"]

rpc/src/lib.rs

@@ -44,6 +44,7 @@ extern crate futures;
 extern crate order_stat;
 extern crate parity_updater as updater;
 extern crate parity_reactor;
+extern crate stats;
 
 #[macro_use]
 extern crate log;

rpc/src/v1/helpers/dispatch.rs

@@ -21,11 +21,12 @@ use std::ops::Deref;
 use std::sync::{Arc, Weak};
 
 use futures::{future, Future, BoxFuture};
+use light::cache::Cache as LightDataCache;
 use light::client::LightChainClient;
 use light::on_demand::{request, OnDemand};
 use light::TransactionQueue as LightTransactionQueue;
 use rlp::{self, Stream};
-use util::{Address, H520, H256, U256, Uint, Bytes, RwLock};
+use util::{Address, H520, H256, U256, Uint, Bytes, Mutex, RwLock};
 use util::sha3::Hashable;
 
 use ethkey::Signature;
@@ -162,6 +163,7 @@ pub struct LightDispatcher {
 	sync: Arc<LightSync>,
 	client: Arc<LightChainClient>,
 	on_demand: Arc<OnDemand>,
+	cache: Arc<Mutex<LightDataCache>>,
 	transaction_queue: Arc<RwLock<LightTransactionQueue>>,
 }
 
@@ -173,12 +175,14 @@ impl LightDispatcher {
 		sync: Arc<LightSync>,
 		client: Arc<LightChainClient>,
 		on_demand: Arc<OnDemand>,
+		cache: Arc<Mutex<LightDataCache>>,
 		transaction_queue: Arc<RwLock<LightTransactionQueue>>,
 	) -> Self {
 		LightDispatcher {
 			sync: sync,
 			client: client,
 			on_demand: on_demand,
+			cache: cache,
 			transaction_queue: transaction_queue,
 		}
 	}
@@ -453,7 +457,7 @@ fn decrypt(accounts: &AccountProvider, address: Address, msg: Bytes, password: S
 pub fn default_gas_price<C, M>(client: &C, miner: &M) -> U256
 	where C: MiningBlockChainClient, M: MinerService
 {
-	client.gas_price_median(100).unwrap_or_else(|| miner.sensible_gas_price())
+	client.gas_price_corpus(100).median().cloned().unwrap_or_else(|| miner.sensible_gas_price())
 }
 
 /// Convert RPC confirmation payload to signer confirmation payload.

rpc/src/v1/impls/parity.rs

@@ -236,7 +236,7 @@ impl<C, M, S: ?Sized, U> Parity for ParityClient<C, M, S, U> where
 	}
 
 	fn gas_price_histogram(&self) -> Result<Histogram, Error> {
-		take_weak!(self.client).gas_price_histogram(100, 10).ok_or_else(errors::not_enough_data).map(Into::into)
+		take_weak!(self.client).gas_price_corpus(100).histogram(10).ok_or_else(errors::not_enough_data).map(Into::into)
 	}
 
 	fn unsigned_transactions_count(&self) -> Result<usize, Error> {

rpc/src/v1/types/histogram.rs

@@ -17,7 +17,6 @@
 //! Gas prices histogram.
 
 use v1::types::U256;
-use util::stats;
 
 /// Values of RPC settings.
 #[derive(Serialize, Deserialize)]
@@ -27,11 +26,11 @@ pub struct Histogram {
 	#[serde(rename="bucketBounds")]
 	pub bucket_bounds: Vec<U256>,
 	/// Transacion counts for each bucket.
-	pub counts: Vec<u64>,
+	pub counts: Vec<usize>,
 }
 
-impl From<stats::Histogram> for Histogram {
+impl From<::stats::Histogram<::util::U256>> for Histogram {
-	fn from(h: stats::Histogram) -> Self {
+	fn from(h: ::stats::Histogram<::util::U256>) -> Self {
 		Histogram {
 			bucket_bounds: h.bucket_bounds.into_iter().map(Into::into).collect(),
 			counts: h.counts

util/stats/src/lib.rs

@@ -17,14 +17,14 @@
 //! Statistical functions and helpers.
 
 use std::iter::FromIterator;
-use std::ops::{Add, Sub, Div};
+use std::ops::{Add, Sub, Deref, Div};
 
 #[macro_use]
 extern crate log;
 
 /// Sorted corpus of data.
 #[derive(Debug, Clone, PartialEq)]
-pub struct Corpus<T: Ord>(Vec<T>);
+pub struct Corpus<T>(Vec<T>);
 
 impl<T: Ord> From<Vec<T>> for Corpus<T> {
 	fn from(mut data: Vec<T>) -> Self {
@@ -39,6 +39,12 @@ impl<T: Ord> FromIterator<T> for Corpus<T> {
 	}
 }
 
+impl<T> Deref for Corpus<T> {
+	type Target = [T];
+
+	fn deref(&self) -> &[T] { &self.0[..] }
+}
+
 impl<T: Ord> Corpus<T> {
 	/// Get the median element, if it exists.
 	pub fn median(&self) -> Option<&T> {
@@ -54,20 +60,17 @@ impl<T: Ord> Corpus<T> {
 	pub fn len(&self) -> usize {
 		self.0.len()
 	}
-
-	/// Split the corpus at a given point.
-	pub fn split_at(self, idx: usize) -> (Self, Self) {
-		let (left, right) = self.0.split_at(idx);
-		(Corpus(left), Corpus(right))
-	}
 }
 
 impl<T: Ord + Copy + ::std::fmt::Display> Corpus<T>
 	where T: Add<Output=T> + Sub<Output=T> + Div<Output=T> + From<usize>
 {
 	/// Create a histogram of this corpus if it at least spans the buckets. Bounds are left closed.
+	/// Excludes outliers.
 	pub fn histogram(&self, bucket_number: usize) -> Option<Histogram<T>> {
-		Histogram::create(&self.0, bucket_number)
+		// TODO: get outliers properly.
+		let upto = self.len() - self.len() / 40;
+		Histogram::create(&self.0[..upto], bucket_number)
 	}
 }