Fork choice and metadata framework for Engine (#8401)

* Add light client TODO item

* Move existing total-difficulty-based fork choice check to Engine

* Abstract total difficulty and block provider as Machine::BlockMetadata and Machine::BlockProvider

* Decouple "generate_metadata" logic to Engine

* Use fixed BlockMetadata and BlockProvider type for null and instantseal

In this way they can use total difficulty fork choice check

* Extend blockdetails with metadatas and finalized info

* Extra data update: mark_finalized and update_metadatas

* Check finalized block in Blockchain

* Fix a test constructor in verification mod

* Add total difficulty trait

* Fix type import

* Db migration to V13 with metadata column

* Address grumbles

* metadatas -> metadata

* Use generic type for update_metadata to avoid passing HashMap all around

* Remove metadata in blockdetails

* [WIP] Implement a generic metadata architecture

* [WIP] Metadata insertion logic in BlockChain

* typo: Value -> Self::Value

* [WIP] Temporarily remove Engine::is_new_best interface

So that we don't have too many type errors.

* [WIP] Fix more type errors

* [WIP] ExtendedHeader::PartialEq

* [WIP] Change metadata type Option<Vec<u8>> to Vec<u8>

* [WIP] Remove Metadata Error

* [WIP] Clean up error conversion

* [WIP] finalized -> is_finalized

* [WIP] Mark all fields in ExtrasInsert as pub

* [WIP] Remove unused import

* [WIP] Keep only local metadata info

* Mark metadata as optional

* [WIP] Revert metadata db change in BlockChain

* [WIP] Put finalization in unclosed state

* Use metadata interface in BlockDetail

* [WIP] Fix current build failures

* [WIP] Remove unused blockmetadata struct

* Remove DB migration info

* [WIP] Typo

* Use ExtendedHeader to implement fork choice check

* Implement is_new_best using Ancestry iterator

* Use expect instead of panic

* [WIP] Add ancestry Engine support via on_new_block

* Fix tests

* Emission of ancestry actions

* use_short_version should take account of metadata

* Engine::is_new_best -> Engine::fork_choice

* Use proper expect format as defined in #1026

* panic -> expect

* ancestry_header -> ancestry_with_metadata

* Boxed iterator -> &mut iterator

* Fix tests

* is_new_best -> primitive_fork_choice

* Document how fork_choice works

* Engine::fork_choice -> Engine::primitive_fork_choice

* comment: clarify types of finalization where Engine::primitive_fork_choice works

* Expose FinalizationInfo to Engine

* Fix tests due to merging

* Remove TotalDifficulty trait

* Do not pass FinalizationInfo to Engine

If there's finalized blocks in from route, choose the old branch without calling `Engine::fork_choice`.

* Fix compile

* Fix unused import

* Remove is_to_route_finalized

When no block reorg passes a finalized block, this variable is always false.

* Address format grumbles

* Fix docs: mark_finalized returns None if block hash is not found

`blockchain` mod does not yet have an Error type, so we still temporarily use None here.

* Fix inaccurate tree_route None expect description

ethcore/src/header.rs

@@ -34,6 +34,19 @@ enum Seal {
 	Without,
 }
 
+/// Extended block header, wrapping `Header` with finalized and total difficulty information.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ExtendedHeader {
+	/// The actual header.
+	pub header: Header,
+	/// Whether the block underlying this header is considered finalized.
+	pub is_finalized: bool,
+	/// The parent block difficulty.
+	pub parent_total_difficulty: U256,
+	/// The block metadata information.
+	pub metadata: Option<Vec<u8>>,
+}
+
 /// A block header.
 ///
 /// Reflects the specific RLP fields of a block in the chain with additional room for the seal
@@ -368,21 +381,48 @@ impl HeapSizeOf for Header {
 
 impl ::parity_machine::Header for Header {
 	fn bare_hash(&self) -> H256 { Header::bare_hash(self) }
-
 	fn hash(&self) -> H256 { Header::hash(self) }
-
 	fn seal(&self) -> &[Vec<u8>] { Header::seal(self) }
-
 	fn author(&self) -> &Address { Header::author(self) }
-
 	fn number(&self) -> BlockNumber { Header::number(self) }
 }
 
 impl ::parity_machine::ScoredHeader for Header {
+	type Value = U256;
+
 	fn score(&self) -> &U256 { self.difficulty() }
 	fn set_score(&mut self, score: U256) { self.set_difficulty(score) }
 }
 
+impl ::parity_machine::Header for ExtendedHeader {
+	fn bare_hash(&self) -> H256 { self.header.bare_hash() }
+	fn hash(&self) -> H256 { self.header.hash() }
+	fn seal(&self) -> &[Vec<u8>] { self.header.seal() }
+	fn author(&self) -> &Address { self.header.author() }
+	fn number(&self) -> BlockNumber { self.header.number() }
+}
+
+impl ::parity_machine::ScoredHeader for ExtendedHeader {
+	type Value = U256;
+
+	fn score(&self) -> &U256 { self.header.difficulty() }
+	fn set_score(&mut self, score: U256) { self.header.set_difficulty(score) }
+}
+
+impl ::parity_machine::TotalScoredHeader for ExtendedHeader {
+	type Value = U256;
+
+	fn total_score(&self) -> U256 { self.parent_total_difficulty + *self.header.difficulty() }
+}
+
+impl ::parity_machine::FinalizableHeader for ExtendedHeader {
+	fn is_finalized(&self) -> bool { self.is_finalized }
+}
+
+impl ::parity_machine::WithMetadataHeader for ExtendedHeader {
+	fn metadata(&self) -> Option<&[u8]> { self.metadata.as_ref().map(|v| v.as_ref()) }
+}
+
 #[cfg(test)]
 mod tests {
 	use rustc_hex::FromHex;