fetch gas price corpus from network when needed

Cargo.lock

@@ -427,7 +427,7 @@ dependencies = [
  "ethcore-rpc 1.6.0",
  "ethcore-util 1.6.0",
  "fetch 0.1.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)",
  "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "jsonrpc-http-server 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
@@ -536,12 +536,13 @@ dependencies = [
  "ethcore-ipc-codegen 1.6.0",
  "ethcore-network 1.6.0",
  "ethcore-util 1.6.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.1.0",
  "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "stats 0.1.0",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -601,7 +602,7 @@ dependencies = [
  "ethstore 0.1.0",
  "ethsync 1.6.0",
  "fetch 0.1.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "jsonrpc-http-server 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "jsonrpc-ipc-server 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
@@ -650,7 +651,7 @@ dependencies = [
  "ethcore-ipc-codegen 1.6.0",
  "ethcore-ipc-nano 1.6.0",
  "ethcore-util 1.6.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "jsonrpc-macros 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "jsonrpc-tcp-server 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
@@ -813,7 +814,7 @@ dependencies = [
 name = "fetch"
 version = "0.1.0"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -832,7 +833,7 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.1.6"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -844,7 +845,7 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1045,7 +1046,7 @@ name = "jsonrpc-core"
 version = "6.0.0"
 source = "git+https://github.com/ethcore/jsonrpc.git#86d7a89c85f324b5f6671315d9b71010ca995300"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1565,7 +1566,7 @@ dependencies = [
  "ethabi 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-util 1.6.0",
  "fetch 0.1.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1578,7 +1579,7 @@ dependencies = [
 name = "parity-reactor"
 version = "0.1.0"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-core 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1589,7 +1590,7 @@ dependencies = [
  "ethcore-rpc 1.6.0",
  "ethcore-signer 1.6.0",
  "ethcore-util 1.6.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1898,7 +1899,7 @@ dependencies = [
  "ethcore-bigint 0.1.2",
  "ethcore-rpc 1.6.0",
  "ethcore-util 1.6.0",
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-rpc-client 1.4.0",
  "rpassword 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2249,7 +2250,7 @@ name = "tokio-core"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2261,7 +2262,7 @@ name = "tokio-proto"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "net2 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2277,7 +2278,7 @@ name = "tokio-service"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2503,7 +2504,7 @@ dependencies = [
 "checksum ethabi 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d8f6cc4c1acd005f48e1d17b06a461adac8fb6eeeb331fbf19a0e656fba91cd"
 "checksum fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b1ee15a7050e5580b3712877157068ea713b245b080ff302ae2ca973cfcd9baa"
 "checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
-"checksum futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bad0a2ac64b227fdc10c254051ae5af542cf19c9328704fd4092f7914196897"
+"checksum futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "c1913eb7083840b1bbcbf9631b7fda55eaf35fe7ead13cca034e8946f9e2bc41"
 "checksum futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bb982bb25cd8fa5da6a8eb3a460354c984ff1113da82bcb4f0b0862b5795db82"
 "checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
 "checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518"

ethcore/light/Cargo.toml

@@ -23,6 +23,7 @@ smallvec = "0.3.1"
 futures = "0.1"
 rand = "0.3"
 itertools = "0.5"
+stats = { path = "../../util/stats" }
 
 [features]
 default = []

ethcore/light/src/cache.rs

@@ -24,6 +24,7 @@ use ethcore::encoded;
 use ethcore::header::BlockNumber;
 use ethcore::receipt::Receipt;
 
+use stats::Corpus;
 use time::{SteadyTime, Duration};
 use util::{U256, H256};
 use util::cache::MemoryLruCache;
@@ -66,7 +67,7 @@ pub struct Cache {
 	bodies: MemoryLruCache<H256, encoded::Body>,
 	receipts: MemoryLruCache<H256, Vec<Receipt>>,
 	chain_score: MemoryLruCache<H256, U256>,
-	corpus: Option<(Vec<U256>, SteadyTime)>,
+	corpus: Option<(Corpus<U256>, SteadyTime)>,
 	corpus_expiration: Duration,
 }
 
@@ -135,7 +136,7 @@ impl Cache {
 	}
 
 	/// Get gas price corpus, if recent enough.
-	pub fn gas_price_corpus(&self) -> Option<Vec<U256>> {
+	pub fn gas_price_corpus(&self) -> Option<Corpus<U256>> {
 		let now = SteadyTime::now();
 
 		self.corpus.as_ref().and_then(|&(ref corpus, ref tm)| {
@@ -148,7 +149,7 @@ impl Cache {
 	}
 
 	/// Set the cached gas price corpus.
-	pub fn set_gas_price_corpus(&mut self, corpus: Vec<U256>) {
+	pub fn set_gas_price_corpus(&mut self, corpus: Corpus<U256>) {
 		self.corpus = Some((corpus, SteadyTime::now()))
 	}
 }
@@ -162,8 +163,8 @@ mod tests {
 	fn corpus_inaccessible() {
 		let mut cache = Cache::new(Default::default(), Duration::hours(5));
 
-		cache.set_gas_price_corpus(vec![]);
+		cache.set_gas_price_corpus(vec![].into());
-		assert_eq!(cache.gas_price_corpus(), Some(vec![]));
+		assert_eq!(cache.gas_price_corpus(), Some(vec![].into()));
 
 		{
 			let corpus_time = &mut cache.corpus.as_mut().unwrap().1;

ethcore/light/src/client/header_chain.rs

@@ -241,6 +241,14 @@ impl HeaderChain {
 		self.block_header(BlockId::Latest).expect("Header for best block always stored; qed")
 	}
 
+	/// Get an iterator over a block and its ancestry.
+	pub fn ancestry_iter(&self, start: BlockId) -> AncestryIter {
+		AncestryIter {
+			next: self.block_header(start),
+			chain: self,
+		}
+	}
+
 	/// Get the nth CHT root, if it's been computed.
 	///
 	/// CHT root 0 is from block `1..2048`.
@@ -295,6 +303,25 @@ impl HeapSizeOf for HeaderChain {
 	}
 }
 
+/// Iterator over a block's ancestry.
+pub struct AncestryIter<'a> {
+	next: Option<encoded::Header>,
+	chain: &'a HeaderChain,
+}
+
+impl<'a> Iterator for AncestryIter<'a> {
+	type Item = encoded::Header;
+
+	fn next(&mut self) -> Option<encoded::Header> {
+		let next = self.next.take();
+		if let Some(p_hash) = next.as_ref().map(|hdr| hdr.parent_hash()) {
+			self.next = self.chain.block_header(BlockId::Hash(p_hash));
+		}
+
+		next
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::HeaderChain;

ethcore/light/src/client/mod.rs

@@ -33,7 +33,7 @@ use io::IoChannel;
 
 use util::{Bytes, H256, Mutex, RwLock};
 
-use self::header_chain::HeaderChain;
+use self::header_chain::{AncestryIter, HeaderChain};
 
 pub use self::service::Service;
 
@@ -62,6 +62,9 @@ pub trait LightChainClient: Send + Sync {
 	/// Get the best block header.
 	fn best_block_header(&self) -> encoded::Header;
 
+	/// Get an iterator over a block and its ancestry.
+	fn ancestry_iter<'a>(&'a self, start: BlockId) -> Box<Iterator<Item=encoded::Header> + 'a>;
+
 	/// Get the signing network ID.
 	fn signing_network_id(&self) -> Option<u64>;
 
@@ -167,6 +170,11 @@ impl Client {
 		self.chain.best_header()
 	}
 
+	/// Get an iterator over a block and its ancestry.
+	pub fn ancestry_iter(&self, start: BlockId) -> AncestryIter {
+		self.chain.ancestry_iter(start)
+	}
+
 	/// Get the signing network id.
 	pub fn signing_network_id(&self) -> Option<u64> {
 		self.engine.signing_network_id(&self.latest_env_info())
@@ -269,6 +277,10 @@ impl LightChainClient for Client {
 		Client::best_block_header(self)
 	}
 
+	fn ancestry_iter<'a>(&'a self, start: BlockId) -> Box<Iterator<Item=encoded::Header> + 'a> {
+		Box::new(Client::ancestry_iter(self, start))
+	}
+
 	fn signing_network_id(&self) -> Option<u64> {
 		Client::signing_network_id(self)
 	}

ethcore/light/src/lib.rs

@@ -72,6 +72,7 @@ extern crate time;
 extern crate futures;
 extern crate rand;
 extern crate itertools;
+extern crate stats;
 
 #[cfg(feature = "ipc")]
 extern crate ethcore_ipc as ipc;

rpc/src/v1/helpers/dispatch.rs

@@ -20,17 +20,18 @@ use std::fmt::Debug;
 use std::ops::Deref;
 use std::sync::{Arc, Weak};
 
-use futures::{future, Future, BoxFuture};
+use futures::{future, stream, Future, Stream, BoxFuture};
 use light::cache::Cache as LightDataCache;
 use light::client::LightChainClient;
 use light::on_demand::{request, OnDemand};
 use light::TransactionQueue as LightTransactionQueue;
-use rlp::{self, Stream};
+use rlp::{self, Stream as StreamRlp};
 use util::{Address, H520, H256, U256, Uint, Bytes, Mutex, RwLock};
 use util::sha3::Hashable;
 
 use ethkey::Signature;
 use ethsync::LightSync;
+use ethcore::ids::BlockId;
 use ethcore::miner::MinerService;
 use ethcore::client::MiningBlockChainClient;
 use ethcore::transaction::{Action, SignedTransaction, PendingTransaction, Transaction};
@@ -192,20 +193,75 @@ impl Dispatcher for LightDispatcher {
 	fn fill_optional_fields(&self, request: TransactionRequest, default_sender: Address)
 		-> BoxFuture<FilledTransactionRequest, Error>
 	{
-		let request = request;
+		const GAS_PRICE_SAMPLE_SIZE: usize = 100;
-		let gas_limit = self.client.best_block_header().gas_limit();
+		const DEFAULT_GAS_PRICE: U256 = U256([0, 0, 0, 21_000_000]);
 
-		future::ok(FilledTransactionRequest {
+		let gas_limit = self.client.best_block_header().gas_limit();
-			from: request.from.unwrap_or(default_sender),
+		let request_gas_price = request.gas_price.clone();
-			used_default_from: request.from.is_none(),
+
-			to: request.to,
+		let with_gas_price = move |gas_price| {
-			nonce: request.nonce,
+			let request = request;
-			gas_price: request.gas_price.unwrap_or_else(|| 21_000_000.into()), // TODO: fetch corpus from network.
+			FilledTransactionRequest {
-			gas: request.gas.unwrap_or_else(|| gas_limit / 3.into()),
+				from: request.from.unwrap_or(default_sender),
-			value: request.value.unwrap_or_else(|| 0.into()),
+				used_default_from: request.from.is_none(),
-			data: request.data.unwrap_or_else(Vec::new),
+				to: request.to,
-			condition: request.condition,
+				nonce: request.nonce,
-		}).boxed()
+				gas_price: gas_price,
+				gas: request.gas.unwrap_or_else(|| gas_limit / 3.into()),
+				value: request.value.unwrap_or_else(|| 0.into()),
+				data: request.data.unwrap_or_else(Vec::new),
+				condition: request.condition,
+			}
+		};
+
+		// fast path for gas price supplied or cached corpus.
+		let known_price = request_gas_price.or_else(||
+			self.cache.lock().gas_price_corpus().and_then(|corp| corp.median().cloned())
+		);
+
+		match known_price {
+			Some(gas_price) => future::ok(with_gas_price(gas_price)).boxed(),
+			None => {
+				let cache = self.cache.clone();
+				let gas_price_res = self.sync.with_context(|ctx| {
+
+					// get some recent headers with gas used,
+					// and request each of the blocks from the network.
+					let block_futures = self.client.ancestry_iter(BlockId::Latest)
+						.filter(|hdr| hdr.gas_used() != U256::default())
+						.take(GAS_PRICE_SAMPLE_SIZE)
+						.map(request::Body::new)
+						.map(|req| self.on_demand.block(ctx, req));
+
+					// as the blocks come in, collect gas prices into a vector
+					stream::futures_unordered(block_futures)
+						.fold(Vec::new(), |mut v, block| {
+							for t in block.transaction_views().iter() {
+								v.push(t.gas_price())
+							}
+
+							future::ok(v)
+						})
+						.map(move |v| {
+							// produce a corpus from the vector, cache it, and return
+							// the median as the intended gas price.
+							let corpus: ::stats::Corpus<_> = v.into();
+							cache.lock().set_gas_price_corpus(corpus.clone());
+
+
+							corpus.median().cloned().unwrap_or(DEFAULT_GAS_PRICE)
+						})
+						.map_err(|_| errors::no_light_peers())
+				});
+
+				// attempt to fetch the median, but fall back to a hardcoded
+				// value in case of weak corpus or disconnected network.
+				match gas_price_res {
+					Some(res) => res.map(with_gas_price).boxed(),
+					None => future::ok(with_gas_price(DEFAULT_GAS_PRICE)).boxed()
+				}
+			}
+		}
 	}
 
 	fn sign(&self, accounts: Arc<AccountProvider>, filled: FilledTransactionRequest, password: SignWith)