Replace all Rlp usages with UntrustedRlp except for ethcore views (#8233)

* Replace Rlp with UntrustedRlp and unsafely unwrap

All Rlp methods return Result<_,DecoderError> now, so for this first
pass each will be marked with `expect("TODO")`. In the next pass we can
categorise figure out how to handle each case.

* Handle DecoderError for tendermint message

* Unwrap rlp results in TestBlockcChainClient

Rlp should be valid since created manually in tests

* Replace `use rlp::*` with explicit imports

* Remove rlp decode unwraps from light cli request

* Structured rlp encoding for curr best and latest in header chain

* Propogate decoder errors from send_packet

* Fix body uncles rlp index

* Use BodyView in sync and `expect` rlp errors

* Revert bbf28f removing original Rlp for this phase

This can be done again in the next phase, in order that we can leave the ethcore views unchanged

* Restore legacy Rlp and UntrustedRlp

Use legacy Rlp for ethcore views. Will redo replacing Rlp with UntrustedRlp in  a subsequent PR

* Fix tests

* Replace boilerplate Encodable/Decodable with derive

* Use BlockView instead of Rlp, remove unwrap

* Remove rlp test_cli unwraps by using BlockView instead of Rlp directly

* Remove unneccesary change to use borrowed hash

* Construct sync block using new_from_header_and_body

ethcore/src/blockchain/blockchain.rs

@@ -25,7 +25,7 @@ use heapsize::HeapSizeOf;
 use ethereum_types::{H256, Bloom, U256};
 use parking_lot::{Mutex, RwLock};
 use bytes::Bytes;
-use rlp::*;
+use rlp::{Rlp, RlpStream};
 use rlp_compress::{compress, decompress, blocks_swapper};
 use header::*;
 use transaction::*;

ethcore/src/client/test_client.rs

@@ -29,7 +29,7 @@ use journaldb;
 use kvdb::DBValue;
 use kvdb_memorydb;
 use bytes::Bytes;
-use rlp::*;
+use rlp::{UntrustedRlp, RlpStream};
 use ethkey::{Generator, Random};
 use transaction::{self, Transaction, LocalizedTransaction, PendingTransaction, SignedTransaction, Action};
 use blockchain::{TreeRoute, BlockReceipts};
@@ -66,6 +66,7 @@ use encoded;
 use engines::EthEngine;
 use trie;
 use state::StateInfo;
+use views::BlockView;
 
 /// Test client.
 pub struct TestBlockChainClient {
@@ -469,7 +470,7 @@ impl ChainInfo for TestBlockChainClient {
 impl BlockInfo for TestBlockChainClient {
 	fn block_header(&self, id: BlockId) -> Option<encoded::Header> {
 		self.block_hash(id)
-			.and_then(|hash| self.blocks.read().get(&hash).map(|r| Rlp::new(r).at(0).as_raw().to_vec()))
+			.and_then(|hash| self.blocks.read().get(&hash).map(|r| BlockView::new(r).header_rlp().as_raw().to_vec()))
 			.map(encoded::Header::new)
 	}
 
@@ -510,7 +511,7 @@ impl RegistryInfo for TestBlockChainClient {
 
 impl ImportBlock for TestBlockChainClient {
 	fn import_block(&self, b: Bytes) -> Result<H256, BlockImportError> {
-		let header = Rlp::new(&b).val_at::<BlockHeader>(0);
+		let header = BlockView::new(&b).header();
 		let h = header.hash();
 		let number: usize = header.number() as usize;
 		if number > self.blocks.read().len() {
@@ -519,7 +520,7 @@ impl ImportBlock for TestBlockChainClient {
 		if number > 0 {
 			match self.blocks.read().get(header.parent_hash()) {
 				Some(parent) => {
-					let parent = Rlp::new(parent).val_at::<BlockHeader>(0);
+					let parent = BlockView::new(parent).header();
 					if parent.number() != (header.number() - 1) {
 						panic!("Unexpected block parent");
 					}
@@ -544,7 +545,7 @@ impl ImportBlock for TestBlockChainClient {
 				while n > 0 && self.numbers.read()[&n] != parent_hash {
 					*self.numbers.write().get_mut(&n).unwrap() = parent_hash.clone();
 					n -= 1;
-					parent_hash = Rlp::new(&self.blocks.read()[&parent_hash]).val_at::<BlockHeader>(0).parent_hash().clone();
+					parent_hash = BlockView::new(&self.blocks.read()[&parent_hash]).header().parent_hash().clone();
 				}
 			}
 		}
@@ -683,9 +684,10 @@ impl BlockChainClient for TestBlockChainClient {
 
 	fn block_body(&self, id: BlockId) -> Option<encoded::Body> {
 		self.block_hash(id).and_then(|hash| self.blocks.read().get(&hash).map(|r| {
+			let block = BlockView::new(r);
 			let mut stream = RlpStream::new_list(2);
-			stream.append_raw(Rlp::new(r).at(1).as_raw(), 1);
-			stream.append_raw(Rlp::new(r).at(2).as_raw(), 1);
+			stream.append_raw(block.transactions_rlp().as_raw(), 1);
+			stream.append_raw(block.uncles_rlp().as_raw(), 1);
 			encoded::Body::new(stream.out())
 		}))
 	}

ethcore/src/encoded.rs

@@ -31,7 +31,7 @@ use views;
 use hash::keccak;
 use heapsize::HeapSizeOf;
 use ethereum_types::{H256, Bloom, U256, Address};
-use rlp::Rlp;
+use rlp::{Rlp, RlpStream};
 
 /// Owning header view.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -144,6 +144,9 @@ impl Body {
 
 // forwarders to borrowed view.
 impl Body {
+	/// Get raw rlp of transactions
+	pub fn transactions_rlp(&self) -> Rlp { self.view().transactions_rlp() }
+
 	/// Get a vector of all transactions.
 	pub fn transactions(&self) -> Vec<UnverifiedTransaction> { self.view().transactions() }
 
@@ -156,6 +159,9 @@ impl Body {
 	/// The hash of each transaction in the block.
 	pub fn transaction_hashes(&self) -> Vec<H256> { self.view().transaction_hashes() }
 
+	/// Get raw rlp of uncle headers
+	pub fn uncles_rlp(&self) -> Rlp { self.view().uncles_rlp() }
+
 	/// Decode uncle headers.
 	pub fn uncles(&self) -> Vec<FullHeader> { self.view().uncles() }
 
@@ -181,6 +187,15 @@ impl Block {
 	/// Create a new owning block view. The raw bytes passed in must be an rlp-encoded block.
 	pub fn new(raw: Vec<u8>) -> Self { Block(raw) }
 
+	/// Create a new owning block view by concatenating the encoded header and body
+	pub fn new_from_header_and_body(header: &views::HeaderView, body: &views::BodyView) -> Self {
+		let mut stream = RlpStream::new_list(3);
+		stream.append_raw(header.rlp().as_raw(), 1);
+		stream.append_raw(body.transactions_rlp().as_raw(), 1);
+		stream.append_raw(body.uncles_rlp().as_raw(), 1);
+		Block::new(stream.out())
+	}
+
 	/// Get a borrowed view of the whole block.
 	#[inline]
 	pub fn view(&self) -> views::BlockView { views::BlockView::new(&self.0) }

ethcore/src/engines/tendermint/message.rs

@@ -23,7 +23,7 @@ use bytes::Bytes;
 use super::{Height, View, BlockHash, Step};
 use error::Error;
 use header::Header;
-use rlp::{Rlp, UntrustedRlp, RlpStream, Encodable, Decodable, DecoderError};
+use rlp::{UntrustedRlp, RlpStream, Encodable, Decodable, DecoderError};
 use ethkey::{recover, public_to_address};
 use super::super::vote_collector::Message;
 
@@ -100,7 +100,7 @@ impl ConsensusMessage {
 
 	pub fn verify(&self) -> Result<Address, Error> {
 		let full_rlp = ::rlp::encode(self);
-		let block_info = Rlp::new(&full_rlp).at(1);
+		let block_info = UntrustedRlp::new(&full_rlp).at(1)?;
 		let public_key = recover(&self.signature.into(), &keccak(block_info.as_raw()))?;
 		Ok(public_to_address(&public_key))
 	}

ethcore/src/header.rs

@@ -23,7 +23,7 @@ use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP, keccak};
 use heapsize::HeapSizeOf;
 use ethereum_types::{H256, U256, Address, Bloom};
 use bytes::Bytes;
-use rlp::*;
+use rlp::{UntrustedRlp, RlpStream, Encodable, DecoderError, Decodable};
 
 pub use types::BlockNumber;
 

ethcore/src/state/account.rs

@@ -27,7 +27,7 @@ use bytes::{Bytes, ToPretty};
 use trie;
 use trie::{SecTrieDB, Trie, TrieFactory, TrieError};
 use pod_account::*;
-use rlp::*;
+use rlp::{RlpStream, encode};
 use lru_cache::LruCache;
 use basic_account::BasicAccount;
 

ethcore/src/trace/types/flat.rs

@@ -17,7 +17,7 @@
 //! Flat trace module
 
 use std::collections::VecDeque;
-use rlp::*;
+use rlp::{UntrustedRlp, RlpStream, Decodable, Encodable, DecoderError};
 use heapsize::HeapSizeOf;
 use ethereum_types::Bloom;
 use super::trace::{Action, Res};

ethcore/src/trace/types/trace.rs

@@ -18,7 +18,7 @@
 
 use ethereum_types::{U256, Address, Bloom, BloomInput};
 use bytes::Bytes;
-use rlp::*;
+use rlp::{UntrustedRlp, RlpStream, Encodable, DecoderError, Decodable};
 
 use vm::ActionParams;
 use evm::CallType;

ethcore/src/views/block.rs

@@ -91,6 +91,11 @@ impl<'a> BlockView<'a> {
 			}).collect()
 	}
 
+	/// Return the raw rlp for the transactions in the given block.
+	pub fn transactions_rlp(&self) -> Rlp<'a> {
+		self.rlp.at(1)
+	}
+
 	/// Return number of transactions in given block, without deserializing them.
 	pub fn transactions_count(&self) -> usize {
 		self.rlp.at(1).iter().count()
@@ -125,6 +130,11 @@ impl<'a> BlockView<'a> {
 		})
 	}
 
+	/// Returns raw rlp for the uncles in the given block
+	pub fn uncles_rlp(&self) -> Rlp<'a> {
+		self.rlp.at(2)
+	}
+
 	/// Return list of uncles of given block.
 	pub fn uncles(&self) -> Vec<Header> {
 		self.rlp.list_at(2)

ethcore/src/views/body.rs

@@ -68,11 +68,15 @@ impl<'a> BodyView<'a> {
 			}).collect()
 	}
 
+	/// Return the raw rlp for the transactions in the given block.
+	pub fn transactions_rlp(&self) -> Rlp<'a> {
+		self.rlp.at(0)
+	}
+
 	/// Return number of transactions in given block, without deserializing them.
 	pub fn transactions_count(&self) -> usize {
 		self.rlp.at(0).item_count()
 	}
-
 	/// Return List of transactions in given block.
 	pub fn transaction_views(&self) -> Vec<TransactionView<'a>> {
 		self.rlp.at(0).iter().map(TransactionView::new_from_rlp).collect()
@@ -99,6 +103,11 @@ impl<'a> BodyView<'a> {
 		})
 	}
 
+	/// Returns raw rlp for the uncles in the given block
+	pub fn uncles_rlp(&self) -> Rlp<'a> {
+		self.rlp.at(1)
+	}
+
 	/// Return list of uncles of given block.
 	pub fn uncles(&self) -> Vec<Header> {
 		self.rlp.list_at(1)