Include total difficulty in CHTs and hide implementation details from consumers (#4428)

* CHT builder and prover

* use CHT abstraction in provider

* hide CHT internals from header chain

* fix itertools conflict by updating all to 0.5

* cht proof checker, use it in on_demand
This commit is contained in:
Robert Habermeier 2017-02-06 17:21:35 +01:00 committed by Arkadiy Paronyan
parent 127baed385
commit 4172a5369c
12 changed files with 261 additions and 95 deletions

18
Cargo.lock generated
View File

@ -300,6 +300,11 @@ name = "dtoa"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "either"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "elastic-array"
version = "0.6.0"
@ -531,6 +536,7 @@ dependencies = [
"ethcore-network 1.6.0",
"ethcore-util 1.6.0",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.1.0",
@ -666,7 +672,7 @@ dependencies = [
"ethcore-bloom-journal 0.1.0",
"ethcore-devtools 1.6.0",
"heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -734,7 +740,7 @@ dependencies = [
"ethcore-util 1.6.0",
"ethcrypto 0.1.0",
"ethkey 0.2.0",
"itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -990,8 +996,11 @@ dependencies = [
[[package]]
name = "itertools"
version = "0.4.13"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itoa"
@ -2502,6 +2511,7 @@ dependencies = [
"checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76"
"checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d"
"checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f"
"checksum either 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2b503c86dad62aaf414ecf2b8c527439abedb3f8d812537f0b12bfd6f32a91"
"checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "<none>"
"checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5"
"checksum eth-secp256k1 0.5.6 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
@ -2524,7 +2534,7 @@ dependencies = [
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
"checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484"
"checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c"
"checksum itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "086e1fa5fe48840b1cfdef3a20c7e3115599f8d5c4c87ef32a794a7cdd184d76"
"checksum itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d95557e7ba6b71377b0f2c3b3ae96c53f1b75a926a6901a500f557a370af730a"
"checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
"checksum itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91fd9dc2c587067de817fec4ad355e3818c3d893a78cab32a0a474c7a15bb8d5"
"checksum jsonrpc-core 5.1.0 (git+https://github.com/ethcore/jsonrpc.git)" = "<none>"

View File

@ -22,6 +22,7 @@ time = "0.1"
smallvec = "0.3.1"
futures = "0.1"
rand = "0.3"
itertools = "0.5"
[features]
default = []

View File

@ -12,10 +12,154 @@
// GNU General Public License for more details.
//! Canonical hash trie definitions and helper functions.
//!
//! Each CHT is a trie mapping block numbers to canonical hashes and total difficulty.
//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in
//! favor the the trie root. When the "ancient" blocks need to be accessed, we simply
//! request an inclusion proof of a specific block number against the trie with the
//! root has. A correct proof implies that the claimed block is identical to the one
//! we discarded.
use ethcore::ids::BlockId;
use util::{Bytes, H256, U256, HashDB, MemoryDB};
use util::trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder};
use rlp::{Stream, RlpStream, UntrustedRlp, View};
// encode a key.
macro_rules! key {
($num: expr) => { ::rlp::encode(&$num) }
}
macro_rules! val {
($hash: expr, $td: expr) => {{
let mut stream = RlpStream::new_list(2);
stream.append(&$hash).append(&$td);
stream.drain()
}}
}
/// The size of each CHT.
pub const SIZE: u64 = 2048;
/// A canonical hash trie. This is generic over any database it can query.
/// See module docs for more details.
#[derive(Debug, Clone)]
pub struct CHT<DB: HashDB> {
db: DB,
root: H256, // the root of this CHT.
number: u64,
}
impl<DB: HashDB> CHT<DB> {
/// Query the root of the CHT.
pub fn root(&self) -> H256 { self.root }
/// Query the number of the CHT.
pub fn number(&self) -> u64 { self.number }
/// Generate an inclusion proof for the entry at a specific block.
/// Nodes before level `from_level` will be omitted.
/// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request.
pub fn prove(&self, num: u64, from_level: u32) -> trie::Result<Option<Vec<Bytes>>> {
if block_to_cht_number(num) != Some(self.number) { return Ok(None) }
let mut recorder = Recorder::with_depth(from_level);
let t = TrieDB::new(&self.db, &self.root)?;
t.get_with(&key!(num), &mut recorder)?;
Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect()))
}
}
/// Block information necessary to build a CHT.
pub struct BlockInfo {
/// The block's hash.
pub hash: H256,
/// The block's parent's hash.
pub parent_hash: H256,
/// The block's total difficulty.
pub total_difficulty: U256,
}
/// Build an in-memory CHT from a closure which provides necessary information
/// about blocks. If the fetcher ever fails to provide the info, the CHT
/// will not be generated.
pub fn build<F>(cht_num: u64, mut fetcher: F) -> Option<CHT<MemoryDB>>
where F: FnMut(BlockId) -> Option<BlockInfo>
{
let mut db = MemoryDB::new();
// start from the last block by number and work backwards.
let last_num = start_number(cht_num + 1) - 1;
let mut id = BlockId::Number(last_num);
let mut root = H256::default();
{
let mut t = TrieDBMut::new(&mut db, &mut root);
for blk_num in (0..SIZE).map(|n| last_num - n) {
let info = match fetcher(id) {
Some(info) => info,
None => return None,
};
id = BlockId::Hash(info.parent_hash);
t.insert(&key!(blk_num), &val!(info.hash, info.total_difficulty))
.expect("fresh in-memory database is infallible; qed");
}
}
Some(CHT {
db: db,
root: root,
number: cht_num,
})
}
/// Compute a CHT root from an iterator of (hash, td) pairs. Fails if shorter than
/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`.
/// Discards the trie's nodes.
pub fn compute_root<I>(cht_num: u64, iterable: I) -> Option<H256>
where I: IntoIterator<Item=(H256, U256)>
{
let mut v = Vec::with_capacity(SIZE as usize);
let start_num = start_number(cht_num) as usize;
for (i, (h, td)) in iterable.into_iter().take(SIZE as usize).enumerate() {
v.push((key!(i + start_num).to_vec(), val!(h, td).to_vec()))
}
if v.len() == SIZE as usize {
Some(::util::triehash::trie_root(v))
} else {
None
}
}
/// Check a proof for a CHT.
/// Given a set of a trie nodes, a number to query, and a trie root,
/// verify the given trie branch and extract the canonical hash and total difficulty.
// TODO: better support for partially-checked queries.
pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> {
let mut db = MemoryDB::new();
for node in proof { db.insert(&node[..]); }
let res = match TrieDB::new(&db, &root) {
Err(_) => return None,
Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| {
let rlp = UntrustedRlp::new(val);
rlp.val_at::<H256>(0)
.and_then(|h| rlp.val_at::<U256>(1).map(|td| (h, td)))
.ok()
})
};
match res {
Ok(Some(Some((hash, td)))) => Some((hash, td)),
_ => None,
}
}
/// Convert a block number to a CHT number.
/// Returns `None` for `block_num` == 0, `Some` otherwise.
pub fn block_to_cht_number(block_num: u64) -> Option<u64> {
@ -37,6 +181,12 @@ pub fn start_number(cht_num: u64) -> u64 {
#[cfg(test)]
mod tests {
#[test]
fn size_is_lt_usize() {
// to ensure safe casting on the target platform.
assert!(::cht::SIZE < usize::max_value() as u64)
}
#[test]
fn block_to_cht_number() {
assert!(::cht::block_to_cht_number(0).is_none());

View File

@ -173,26 +173,34 @@ impl HeaderChain {
// produce next CHT root if it's time.
let earliest_era = *candidates.keys().next().expect("at least one era just created; qed");
if earliest_era + HISTORY + cht::SIZE <= number {
let mut values = Vec::with_capacity(cht::SIZE as usize);
{
let cht_num = cht::block_to_cht_number(earliest_era)
.expect("fails only for number == 0; genesis never imported; qed");
debug_assert_eq!(cht_num as usize, self.cht_roots.lock().len());
let mut headers = self.headers.write();
for i in (0..cht::SIZE).map(|x| x + earliest_era) {
let cht_root = {
let mut i = earliest_era;
// iterable function which removes the candidates as it goes
// along. this will only be called until the CHT is complete.
let iter = || {
let era_entry = candidates.remove(&i)
.expect("all eras are sequential with no gaps; qed");
i += 1;
for ancient in &era_entry.candidates {
headers.remove(&ancient.hash);
}
values.push((
::rlp::encode(&i).to_vec(),
::rlp::encode(&era_entry.canonical_hash).to_vec(),
));
}
}
let canon = &era_entry.candidates[0];
(canon.hash, canon.total_difficulty)
};
cht::compute_root(cht_num, ::itertools::repeat_call(iter))
.expect("fails only when too few items; this is checked; qed")
};
let cht_root = ::util::triehash::trie_root(values);
debug!(target: "chain", "Produced CHT {} root: {:?}", (earliest_era - 1) % cht::SIZE, cht_root);
debug!(target: "chain", "Produced CHT {} root: {:?}", cht_num, cht_root);
self.cht_roots.lock().push(cht_root);
}

View File

@ -68,6 +68,7 @@ extern crate smallvec;
extern crate time;
extern crate futures;
extern crate rand;
extern crate itertools;
#[cfg(feature = "ipc")]
extern crate ethcore_ipc as ipc;

View File

@ -29,7 +29,7 @@ use futures::sync::oneshot;
use network::PeerId;
use net::{Handler, Status, Capabilities, Announcement, EventContext, BasicContext, ReqId};
use util::{Bytes, RwLock};
use util::{Bytes, RwLock, U256};
use types::les_request::{self as les_request, Request as LesRequest};
pub mod request;
@ -79,7 +79,7 @@ struct Peer {
// Attempted request info and sender to put received value.
enum Pending {
HeaderByNumber(request::HeaderByNumber, Sender<encoded::Header>), // num + CHT root
HeaderByNumber(request::HeaderByNumber, Sender<(encoded::Header, U256)>), // num + CHT root
HeaderByHash(request::HeaderByHash, Sender<encoded::Header>),
Block(request::Body, Sender<encoded::Block>),
BlockReceipts(request::BlockReceipts, Sender<Vec<Receipt>>),
@ -105,14 +105,15 @@ impl Default for OnDemand {
impl OnDemand {
/// Request a header by block number and CHT root hash.
pub fn header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber) -> Response<encoded::Header> {
/// Returns the header and the total difficulty.
pub fn header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber) -> Response<(encoded::Header, U256)> {
let (sender, receiver) = oneshot::channel();
self.dispatch_header_by_number(ctx, req, sender);
Response(receiver)
}
// dispatch the request, completing the request if no peers available.
fn dispatch_header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber, sender: Sender<encoded::Header>) {
fn dispatch_header_by_number(&self, ctx: &BasicContext, req: request::HeaderByNumber, sender: Sender<(encoded::Header, U256)>) {
let num = req.num;
let cht_num = match ::cht::block_to_cht_number(req.num) {
Some(cht_num) => cht_num,

View File

@ -21,7 +21,7 @@ use ethcore::encoded;
use ethcore::receipt::Receipt;
use rlp::{RlpStream, Stream, UntrustedRlp, View};
use util::{Address, Bytes, HashDB, H256};
use util::{Address, Bytes, HashDB, H256, U256};
use util::memorydb::MemoryDB;
use util::sha3::Hashable;
use util::trie::{Trie, TrieDB, TrieError};
@ -66,24 +66,16 @@ pub struct HeaderByNumber {
impl HeaderByNumber {
/// Check a response with a header and cht proof.
pub fn check_response(&self, header: &[u8], proof: &[Bytes]) -> Result<encoded::Header, Error> {
use util::trie::{Trie, TrieDB};
// check the proof
let mut db = MemoryDB::new();
for node in proof { db.insert(&node[..]); }
let key = ::rlp::encode(&self.num);
let expected_hash: H256 = match TrieDB::new(&db, &self.cht_root).and_then(|t| t.get(&*key))? {
Some(val) => ::rlp::decode(&val),
None => return Err(Error::BadProof)
pub fn check_response(&self, header: &[u8], proof: &[Bytes]) -> Result<(encoded::Header, U256), Error> {
let (expected_hash, td) = match ::cht::check_proof(proof, self.num, self.cht_root) {
Some((expected_hash, td)) => (expected_hash, td),
None => return Err(Error::BadProof),
};
// and compare the hash to the found header.
let found_hash = header.sha3();
match expected_hash == found_hash {
true => Ok(encoded::Header::new(header.to_vec())),
true => Ok((encoded::Header::new(header.to_vec()), td)),
false => Err(Error::WrongHash(expected_hash, found_hash)),
}
}
@ -191,51 +183,44 @@ impl Account {
mod tests {
use super::*;
use util::{MemoryDB, Address, H256, FixedHash};
use util::trie::{Trie, TrieMut, TrieDB, SecTrieDB, TrieDBMut, SecTrieDBMut};
use util::trie::{Trie, TrieMut, SecTrieDB, SecTrieDBMut};
use util::trie::recorder::Recorder;
use ethcore::client::{BlockChainClient, TestBlockChainClient, EachBlockWith};
use ethcore::header::Header;
use ethcore::encoded;
use ethcore::receipt::Receipt;
#[test]
fn check_header_by_number() {
let mut root = H256::default();
let mut db = MemoryDB::new();
let mut header = Header::new();
header.set_number(10_000);
header.set_extra_data(b"test_header".to_vec());
use ::cht;
{
let mut trie = TrieDBMut::new(&mut db, &mut root);
for i in (0..2048u64).map(|x| x + 8192) {
let hash = if i == 10_000 {
header.hash()
} else {
H256::random()
};
trie.insert(&*::rlp::encode(&i), &*::rlp::encode(&hash)).unwrap();
}
}
let test_client = TestBlockChainClient::new();
test_client.add_blocks(10500, EachBlockWith::Nothing);
let proof = {
let trie = TrieDB::new(&db, &root).unwrap();
let key = ::rlp::encode(&10_000u64);
let mut recorder = Recorder::new();
trie.get_with(&*key, &mut recorder).unwrap().unwrap();
recorder.drain().into_iter().map(|r| r.data).collect::<Vec<_>>()
let cht = {
let fetcher = |id| {
let hdr = test_client.block_header(id).unwrap();
let td = test_client.block_total_difficulty(id).unwrap();
Some(cht::BlockInfo {
hash: hdr.hash(),
parent_hash: hdr.parent_hash(),
total_difficulty: td,
})
};
cht::build(cht::block_to_cht_number(10_000).unwrap(), fetcher).unwrap()
};
let proof = cht.prove(10_000, 0).unwrap().unwrap();
let req = HeaderByNumber {
num: 10_000,
cht_root: root,
cht_root: cht.root(),
};
let raw_header = ::rlp::encode(&header);
let raw_header = test_client.block_header(::ethcore::ids::BlockId::Number(10_000)).unwrap();
assert!(req.check_response(&*raw_header, &proof[..]).is_ok());
assert!(req.check_response(&raw_header.into_inner(), &proof[..]).is_ok());
}
#[test]

View File

@ -23,6 +23,8 @@ use ethcore::transaction::PendingTransaction;
use ethcore::ids::BlockId;
use ethcore::encoded;
use cht::{self, BlockInfo};
use util::{Bytes, H256};
use request;
@ -227,48 +229,54 @@ impl<T: ProvingBlockChainClient + ?Sized> Provider for T {
}
fn header_proof(&self, req: request::HeaderProof) -> Option<(encoded::Header, Vec<Bytes>)> {
use util::MemoryDB;
use util::trie::{Trie, TrieMut, TrieDB, TrieDBMut, Recorder};
if Some(req.cht_number) != ::cht::block_to_cht_number(req.block_number) {
if Some(req.cht_number) != cht::block_to_cht_number(req.block_number) {
debug!(target: "les_provider", "Requested CHT number mismatch with block number.");
return None;
}
let mut memdb = MemoryDB::new();
let mut root = H256::default();
let mut needed_hdr = None;
{
let mut t = TrieDBMut::new(&mut memdb, &mut root);
let start_num = ::cht::start_number(req.cht_number);
for i in (0..::cht::SIZE).map(|x| x + start_num) {
match self.block_header(BlockId::Number(i)) {
None => return None,
Some(hdr) => {
t.insert(
&*::rlp::encode(&i),
&*::rlp::encode(&hdr.hash()),
).expect("fresh in-memory database is infallible; qed");
if i == req.block_number { needed_hdr = Some(hdr) }
// build the CHT, caching the requested header as we pass through it.
let cht = {
let block_info = |id| {
let hdr = self.block_header(id);
let td = self.block_total_difficulty(id);
match (hdr, td) {
(Some(hdr), Some(td)) => {
let info = BlockInfo {
hash: hdr.hash(),
parent_hash: hdr.parent_hash(),
total_difficulty: td,
};
if hdr.number() == req.block_number {
needed_hdr = Some(hdr);
}
Some(info)
}
_ => None,
}
};
match cht::build(req.cht_number, block_info) {
Some(cht) => cht,
None => return None, // incomplete CHT.
}
};
let needed_hdr = needed_hdr.expect("`needed_hdr` always set in loop, number checked before; qed");
let mut recorder = Recorder::with_depth(req.from_level);
let t = TrieDB::new(&memdb, &root)
.expect("Same DB and root as just produced by TrieDBMut; qed");
if let Err(e) = t.get_with(&*::rlp::encode(&req.block_number), &mut recorder) {
// prove our result.
match cht.prove(req.block_number, req.from_level) {
Ok(Some(proof)) => Some((needed_hdr, proof)),
Ok(None) => None,
Err(e) => {
debug!(target: "les_provider", "Error looking up number in freshly-created CHT: {}", e);
return None;
None
}
}
// TODO: cache calculated CHT if possible.
let proof = recorder.drain().into_iter().map(|x| x.data).collect();
Some((needed_hdr, proof))
}
fn ready_transactions(&self) -> Vec<PendingTransaction> {

View File

@ -552,7 +552,7 @@ impl BlockChainClient for TestBlockChainClient {
let mut adding = false;
let mut blocks = Vec::new();
for (_, hash) in numbers_read.iter().sort_by(|tuple1, tuple2| tuple1.0.cmp(tuple2.0)) {
for (_, hash) in numbers_read.iter().sorted_by(|tuple1, tuple2| tuple1.0.cmp(tuple2.0)) {
if hash == to {
if adding {
blocks.push(hash.clone());

View File

@ -18,7 +18,7 @@ tiny-keccak = "1.0"
docopt = { version = "0.6", optional = true }
time = "0.1.34"
lazy_static = "0.2"
itertools = "0.4"
itertools = "0.5"
parking_lot = "0.3"
ethcrypto = { path = "../ethcrypto" }
ethcore-util = { path = "../util" }

View File

@ -21,7 +21,7 @@ rust-crypto = "0.2.34"
elastic-array = { git = "https://github.com/ethcore/elastic-array" }
rlp = { path = "rlp" }
heapsize = { version = "0.3", features = ["unstable"] }
itertools = "0.4"
itertools = "0.5"
sha3 = { path = "sha3" }
clippy = { version = "0.0.103", optional = true}
ethcore-devtools = { path = "../devtools" }

View File

@ -77,7 +77,9 @@ pub fn ordered_trie_root<I>(input: I) -> H256
/// assert_eq!(trie_root(v), H256::from_str(root).unwrap());
/// }
/// ```
pub fn trie_root(input: Vec<(Vec<u8>, Vec<u8>)>) -> H256 {
pub fn trie_root<I>(input: I) -> H256
where I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>
{
let gen_input = input
// first put elements into btree to sort them and to remove duplicates
.into_iter()