grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Better support for eth_getLogs in light mode (#9186)

Browse Source 
* Light client on-demand request for headers range.

* Cache headers in HeaderWithAncestors response.

Also fulfills request locally if all headers are in cache.

* LightFetch::logs fetches missing headers on demand.

* LightFetch::logs limit the number of headers requested at a time.

* LightFetch::logs refactor header fetching logic.

* Enforce limit on header range length in light client logs request.

* Fix light request tests after struct change.

* Respond to review comments.
This commit is contained in:
Jim Posen
2018-08-25 14:06:01 -07:00
committed by Afri Schoedon
parent 7abe9ec4cc
commit 9ed43230ca
9 changed files with 418 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
rpc/src/v1/helpers/errors.rs
View File

@@ -101,6 +101,14 @@ pub fn request_rejected_limit() -> Error {
}
}
pub fn request_rejected_param_limit(limit: u64, items_desc: &str) -> Error {
Error {
code: ErrorCode::ServerError(codes::REQUEST_REJECTED_LIMIT),
message: format!("Requested data size exceeds limit of {} {}.", limit, items_desc),
data: None,
}
}
pub fn account<T: fmt::Debug>(error: &str, details: T) -> Error {
Error {
code: ErrorCode::ServerError(codes::ACCOUNT_ERROR),

269
rpc/src/v1/helpers/light_fetch.rs
View File

@@ -16,6 +16,7 @@
//! Helpers for fetching blockchain data either from the light client or the network.
use std::cmp;
use std::sync::Arc;
use ethcore::basic_account::BasicAccount;
@@ -31,7 +32,7 @@ use jsonrpc_macros::Trailing;
use light::cache::Cache;
use light::client::LightChainClient;
use light::cht;
use light::{cht, MAX_HEADERS_PER_REQUEST};
use light::on_demand::{
request, OnDemand, HeaderRef, Request as OnDemandRequest,
Response as OnDemandResponse, ExecutionResult,
@@ -42,6 +43,7 @@ use sync::LightSync;
use ethereum_types::{U256, Address};
use hash::H256;
use parking_lot::Mutex;
use fastmap::H256FastMap;
use transaction::{Action, Transaction as EthTransaction, SignedTransaction, LocalizedTransaction};
use v1::helpers::{CallRequest as CallRequestHelper, errors, dispatch};
@@ -299,78 +301,67 @@ impl LightFetch {
use std::collections::BTreeMap;
use jsonrpc_core::futures::stream::{self, Stream};
// early exit for "to" block before "from" block.
let best_number = self.client.chain_info().best_block_number;
let block_number = |id| match id {
BlockId::Earliest => Some(0),
BlockId::Latest => Some(best_number),
BlockId::Hash(h) => self.client.block_header(BlockId::Hash(h)).map(|hdr| hdr.number()),
BlockId::Number(x) => Some(x),
};
const MAX_BLOCK_RANGE: u64 = 1000;
let (from_block_number, from_block_header) = match self.client.block_header(filter.from_block) {
Some(from) => (from.number(), from),
None => return Either::A(future::err(errors::unknown_block())),
};
let fetcher = self.clone();
self.headers_range_by_block_id(filter.from_block, filter.to_block, MAX_BLOCK_RANGE)
.and_then(move |mut headers| {
if headers.is_empty() {
return Either::A(future::ok(Vec::new()));
}
match block_number(filter.to_block) {
Some(to) if to < from_block_number || from_block_number > best_number
=> return Either::A(future::ok(Vec::new())),
Some(_) => (),
_ => return Either::A(future::err(errors::unknown_block())),
}
let on_demand = &fetcher.on_demand;
let maybe_future = self.sync.with_context(move |ctx| {
// find all headers which match the filter, and fetch the receipts for each one.
// match them with their numbers for easy sorting later.
let bit_combos = filter.bloom_possibilities();
let receipts_futures: Vec<_> = self.client.ancestry_iter(filter.to_block)
.take_while(|ref hdr| hdr.number() != from_block_number)
.chain(Some(from_block_header))
.filter(|ref hdr| {
let hdr_bloom = hdr.log_bloom();
bit_combos.iter().any(|bloom| hdr_bloom.contains_bloom(bloom))
})
.map(|hdr| (hdr.number(), hdr.hash(), request::BlockReceipts(hdr.into())))
.map(|(num, hash, req)| self.on_demand.request(ctx, req).expect(NO_INVALID_BACK_REFS).map(move |x| (num, hash, x)))
.collect();
let maybe_future = fetcher.sync.with_context(move |ctx| {
// find all headers which match the filter, and fetch the receipts for each one.
// match them with their numbers for easy sorting later.
let bit_combos = filter.bloom_possibilities();
let receipts_futures: Vec<_> = headers.drain(..)
.filter(|ref hdr| {
let hdr_bloom = hdr.log_bloom();
bit_combos.iter().any(|bloom| hdr_bloom.contains_bloom(bloom))
})
.map(|hdr| (hdr.number(), hdr.hash(), request::BlockReceipts(hdr.into())))
.map(|(num, hash, req)| on_demand.request(ctx, req).expect(NO_INVALID_BACK_REFS).map(move |x| (num, hash, x)))
.collect();
// as the receipts come in, find logs within them which match the filter.
// insert them into a BTreeMap to maintain order by number and block index.
stream::futures_unordered(receipts_futures)
.fold(BTreeMap::new(), move |mut matches, (num, hash, receipts)| {
let mut block_index = 0;
for (transaction_index, receipt) in receipts.into_iter().enumerate() {
for (transaction_log_index, log) in receipt.logs.into_iter().enumerate() {
if filter.matches(&log) {
matches.insert((num, block_index), Log {
address: log.address.into(),
topics: log.topics.into_iter().map(Into::into).collect(),
data: log.data.into(),
block_hash: Some(hash.into()),
block_number: Some(num.into()),
// No way to easily retrieve transaction hash, so let's just skip it.
transaction_hash: None,
transaction_index: Some(transaction_index.into()),
log_index: Some(block_index.into()),
transaction_log_index: Some(transaction_log_index.into()),
log_type: "mined".into(),
removed: false,
});
// as the receipts come in, find logs within them which match the filter.
// insert them into a BTreeMap to maintain order by number and block index.
stream::futures_unordered(receipts_futures)
.fold(BTreeMap::new(), move |mut matches, (num, hash, receipts)| {
let mut block_index = 0;
for (transaction_index, receipt) in receipts.into_iter().enumerate() {
for (transaction_log_index, log) in receipt.logs.into_iter().enumerate() {
if filter.matches(&log) {
matches.insert((num, block_index), Log {
address: log.address.into(),
topics: log.topics.into_iter().map(Into::into).collect(),
data: log.data.into(),
block_hash: Some(hash.into()),
block_number: Some(num.into()),
// No way to easily retrieve transaction hash, so let's just skip it.
transaction_hash: None,
transaction_index: Some(transaction_index.into()),
log_index: Some(block_index.into()),
transaction_log_index: Some(transaction_log_index.into()),
log_type: "mined".into(),
removed: false,
});
}
block_index += 1;
}
}
block_index += 1;
}
}
future::ok(matches)
}) // and then collect them into a vector.
.map(|matches| matches.into_iter().map(|(_, v)| v).collect())
.map_err(errors::on_demand_cancel)
});
future::ok(matches)
}) // and then collect them into a vector.
.map(|matches| matches.into_iter().map(|(_, v)| v).collect())
.map_err(errors::on_demand_cancel)
});
match maybe_future {
Some(fut) => Either::B(Either::A(fut)),
None => Either::B(Either::B(future::err(errors::network_disabled()))),
}
match maybe_future {
Some(fut) => Either::B(Either::A(fut)),
None => Either::B(Either::B(future::err(errors::network_disabled()))),
}
})
}
// Get a transaction by hash. also returns the index in the block.
@@ -448,6 +439,150 @@ impl LightFetch {
None => Box::new(future::err(errors::network_disabled())) as Box<Future<Item = _, Error = _> + Send>
}
}
fn headers_range_by_block_id(
&self,
from_block: BlockId,
to_block: BlockId,
max: u64
) -> impl Future<Item = Vec<encoded::Header>, Error = Error> {
let fetch_hashes = [from_block, to_block].iter()
.filter_map(|block_id| match block_id {
BlockId::Hash(hash) => Some(hash.clone()),
_ => None,
})
.collect::<Vec<_>>();
let best_number = self.client.chain_info().best_block_number;
let fetcher = self.clone();
self.headers_by_hash(&fetch_hashes[..]).and_then(move |mut header_map| {
let (from_block_num, to_block_num) = {
let block_number = |id| match id {
&BlockId::Earliest => 0,
&BlockId::Latest => best_number,
&BlockId::Hash(ref h) =>
header_map.get(h).map(|hdr| hdr.number())
.expect("from_block and to_block headers are fetched by hash; this closure is only called on from_block and to_block; qed"),
&BlockId::Number(x) => x,
};
(block_number(&from_block), block_number(&to_block))
};
if to_block_num < from_block_num {
// early exit for "to" block before "from" block.
return Either::A(future::err(errors::filter_block_not_found(to_block)));
} else if to_block_num - from_block_num >= max {
return Either::A(future::err(errors::request_rejected_param_limit(max, "blocks")));
}
let to_header_hint = match to_block {
BlockId::Hash(ref h) => header_map.remove(h),
_ => None,
};
let headers_fut = fetcher.headers_range(from_block_num, to_block_num, to_header_hint);
Either::B(headers_fut.map(move |headers| {
// Validate from_block if it's a hash
let last_hash = headers.last().map(|hdr| hdr.hash());
match (last_hash, from_block) {
(Some(h1), BlockId::Hash(h2)) if h1 != h2 => Vec::new(),
_ => headers,
}
}))
})
}
fn headers_by_hash(&self, hashes: &[H256]) -> impl Future<Item = H256FastMap<encoded::Header>, Error = Error> {
let mut refs = H256FastMap::with_capacity_and_hasher(hashes.len(), Default::default());
let mut reqs = Vec::with_capacity(hashes.len());
for hash in hashes {
refs.entry(*hash).or_insert_with(|| {
self.make_header_requests(BlockId::Hash(*hash), &mut reqs)
.expect("make_header_requests never fails for BlockId::Hash; qed")
});
}
self.send_requests(reqs, move |res| {
let headers = refs.drain()
.map(|(hash, header_ref)| {
let hdr = extract_header(&res, header_ref)
.expect("these responses correspond to requests that header_ref belongs to; \
qed");
(hash, hdr)
})
.collect();
headers
})
}
fn headers_range(
&self,
from_number: u64,
to_number: u64,
to_header_hint: Option<encoded::Header>
) -> impl Future<Item = Vec<encoded::Header>, Error = Error> {
let range_length = (to_number - from_number + 1) as usize;
let mut headers: Vec<encoded::Header> = Vec::with_capacity(range_length);
let iter_start = match to_header_hint {
Some(hdr) => {
let block_id = BlockId::Hash(hdr.parent_hash());
headers.push(hdr);
block_id
}
None => BlockId::Number(to_number),
};
headers.extend(self.client.ancestry_iter(iter_start)
.take_while(|hdr| hdr.number() >= from_number));
let fetcher = self.clone();
future::loop_fn(headers, move |mut headers| {
let remaining = range_length - headers.len();
if remaining == 0 {
return Either::A(future::ok(future::Loop::Break(headers)));
}
let mut reqs: Vec<request::Request> = Vec::with_capacity(2);
let start_hash = if let Some(hdr) = headers.last() {
hdr.parent_hash().into()
} else {
let cht_root = cht::block_to_cht_number(to_number)
.and_then(|cht_num| fetcher.client.cht_root(cht_num as usize));
let cht_root = match cht_root {
Some(cht_root) => cht_root,
None => return Either::A(future::err(errors::unknown_block())),
};
let header_proof = request::HeaderProof::new(to_number, cht_root)
.expect("HeaderProof::new is Some(_) if cht::block_to_cht_number() is Some(_); \
this would return above if block_to_cht_number returned None; qed");
let idx = reqs.len();
let hash_ref = Field::back_ref(idx, 0);
reqs.push(header_proof.into());
hash_ref
};
let max = cmp::min(remaining as u64, MAX_HEADERS_PER_REQUEST);
reqs.push(request::HeaderWithAncestors {
block_hash: start_hash,
ancestor_count: max - 1,
}.into());
Either::B(fetcher.send_requests(reqs, |mut res| {
match res.last_mut() {
Some(&mut OnDemandResponse::HeaderWithAncestors(ref mut res_headers)) =>
headers.extend(res_headers.drain(..)),
_ => panic!("reqs has at least one entry; each request maps to a response; qed"),
};
future::Loop::Continue(headers)
}))
})
}
}
#[derive(Clone)]